SERVER-67666 Allow watching system.buckets collections in full cluster change streams

2 files changed, 297 insertions, 2 deletions

diff --git a/jstests/change_streams/timeseries.js b/jstests/change_streams/timeseries.js
new file mode 100644
index 00000000000..558e2e85a7a
--- /dev/null
+++ b/jstests/change_streams/timeseries.js
@@ -0,0 +1,291 @@
+/**
+ * Basic test to make sure events from timeseries buckets collections look normal and don't get
+ * filtered.
+ * @tags: [
+ *     change_stream_does_not_expect_txns,
+ *     assumes_unsharded_collection,
+ *     requires_fcv_60,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/change_stream_util.js");  // For ChangeStreamTest.
+
+let testDB = db.getSiblingDB(jsTestName());
+testDB.dropDatabase();
+let dbName = testDB.getName();
+
+let coll = testDB[jsTestName()];
+let collName = coll.getName();
+let bucketsCollName = "system.buckets." + collName;
+
+let cst = new ChangeStreamTest(testDB);
+let curWithEvents = cst.startWatchingChanges({
+    pipeline: [
+        {$changeStream: {showExpandedEvents: true, showSystemEvents: true}},
+        {
+            $project: {
+                documentKey: 0,
+                "fullDocument._id": 0,
+                collectionUUID: 0,
+                "stateBeforeChange.collectionOptions.uuid": 0,
+                clusterTime: 0,
+                wallTime: 0
+            }
+        },
+        {$match: {operationType: {$regex: "(?!shard)", $options: "i"}}}
+    ],
+    collection: 1
+});
+let curNoEvents = testDB.watch([], {showExpandedEvents: true});
+
+assert.commandWorked(testDB.createCollection(
+    jsTestName(),
+    {timeseries: {timeField: "ts", metaField: "meta"}}));    // on buckets ns and view ns
+coll.createIndex({ts: 1, "meta.b": 1}, {name: "dropMe"});    // on buckets ns
+coll.insertOne({_id: 1, ts: new Date(1000), meta: {a: 1}});  // on buckets ns
+coll.insertOne({_id: 1, ts: new Date(1000), meta: {a: 1}});  // on buckets ns
+coll.update({"meta.a": 1}, {$set: {"meta.b": 2}});           // on buckets ns
+coll.remove({"meta.a": 1});                                  // on buckets ns
+// collMod granularity. on both buckets ns and view ns
+assert.commandWorked(testDB.runCommand({collMod: collName, timeseries: {granularity: "hours"}}));
+// collMod expiration. just on buckets ns
+assert.commandWorked(testDB.runCommand({collMod: collName, expireAfterSeconds: 1}));
+coll.dropIndex("dropMe");  // on buckets ns
+coll.drop();               // on buckets ns and view ns
+
+// document key, _id, uuid, cluster and wall times omitted
+let expectedChanges = [
+    {
+        "operationType": "create",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "operationDescription": {
+            "validator": {
+                "$jsonSchema": {
+                    "bsonType": "object",
+                    "required": ["_id", "control", "data"],
+                    "properties": {
+                        "_id": {"bsonType": "objectId"},
+                        "control": {
+                            "bsonType": "object",
+                            "required": ["version", "min", "max"],
+                            "properties": {
+                                "version": {"bsonType": "number"},
+                                "min": {
+                                    "bsonType": "object",
+                                    "required": ["ts"],
+                                    "properties": {"ts": {"bsonType": "date"}}
+                                },
+                                "max": {
+                                    "bsonType": "object",
+                                    "required": ["ts"],
+                                    "properties": {"ts": {"bsonType": "date"}}
+                                },
+                                "closed": {"bsonType": "bool"}
+                            }
+                        },
+                        "data": {"bsonType": "object"},
+                        "meta": {
+
+                        }
+                    },
+                    "additionalProperties": false
+                }
+            },
+            "clusteredIndex": true,
+            "timeseries": {
+                "timeField": "ts",
+                "metaField": "meta",
+                "granularity": "seconds",
+                "bucketMaxSpanSeconds": 3600
+            }
+        }
+    },
+    {
+        "operationType": "create",
+        "ns": {"db": dbName, "coll": collName},
+        "operationDescription": {
+            "viewOn": bucketsCollName,
+            "pipeline": [{
+                "$_internalUnpackBucket":
+                    {"timeField": "ts", "metaField": "meta", "bucketMaxSpanSeconds": 3600}
+            }]
+        }
+    },
+    {
+        "operationType": "createIndexes",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "operationDescription": {
+            "indexes": [{
+                "v": 2,
+                "key": {"control.min.ts": 1, "control.max.ts": 1, "meta.b": 1},
+                "name": "dropMe"
+            }]
+        }
+    },
+    {
+        "operationType": "insert",
+        "fullDocument": {
+            "control": {
+                "version": 1,
+                "min": {"_id": 1, "ts": ISODate("1970-01-01T00:00:00Z")},
+                "max": {"_id": 1, "ts": ISODate("1970-01-01T00:00:01Z")}
+            },
+            "meta": {"a": 1},
+            "data": {"_id": {"0": 1}, "ts": {"0": ISODate("1970-01-01T00:00:01Z")}}
+        },
+        "ns": {"db": dbName, "coll": bucketsCollName}
+    },
+    {
+        "operationType": "update",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "updateDescription": {
+            "updatedFields": {"data._id.1": 1, "data.ts.1": ISODate("1970-01-01T00:00:01Z")},
+            "removedFields": [],
+            "truncatedArrays": []
+        }
+    },
+    {"operationType": "delete", "ns": {"db": dbName, "coll": bucketsCollName}},
+    {
+        "operationType": "modify",
+        "ns": {"db": dbName, "coll": collName},
+        "operationDescription": {
+            "viewOn": "system.buckets.timeseries",
+            "pipeline": [{
+                "$_internalUnpackBucket":
+                    {"timeField": "ts", "metaField": "meta", "bucketMaxSpanSeconds": 2592000}
+            }]
+        }
+    },
+    {
+        "operationType": "modify",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "operationDescription": {"timeseries": {"granularity": "hours"}},
+        "stateBeforeChange": {
+            "collectionOptions": {
+                "validator": {
+                    "$jsonSchema": {
+                        "bsonType": "object",
+                        "required": ["_id", "control", "data"],
+                        "properties": {
+                            "_id": {"bsonType": "objectId"},
+                            "control": {
+                                "bsonType": "object",
+                                "required": ["version", "min", "max"],
+                                "properties": {
+                                    "version": {"bsonType": "number"},
+                                    "min": {
+                                        "bsonType": "object",
+                                        "required": ["ts"],
+                                        "properties": {"ts": {"bsonType": "date"}}
+                                    },
+                                    "max": {
+                                        "bsonType": "object",
+                                        "required": ["ts"],
+                                        "properties": {"ts": {"bsonType": "date"}}
+                                    },
+                                    "closed": {"bsonType": "bool"}
+                                }
+                            },
+                            "data": {"bsonType": "object"},
+                            "meta": {
+
+                            }
+                        },
+                        "additionalProperties": false
+                    }
+                },
+                "clusteredIndex": true,
+                "timeseries": {
+                    "timeField": "ts",
+                    "metaField": "meta",
+                    "granularity": "seconds",
+                    "bucketMaxSpanSeconds": 3600
+                }
+            }
+        }
+    },
+    {
+        "operationType": "modify",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "operationDescription": {"expireAfterSeconds": NumberLong(1)},
+        "stateBeforeChange": {
+            "collectionOptions": {
+                "validator": {
+                    "$jsonSchema": {
+                        "bsonType": "object",
+                        "required": ["_id", "control", "data"],
+                        "properties": {
+                            "_id": {"bsonType": "objectId"},
+                            "control": {
+                                "bsonType": "object",
+                                "required": ["version", "min", "max"],
+                                "properties": {
+                                    "version": {"bsonType": "number"},
+                                    "min": {
+                                        "bsonType": "object",
+                                        "required": ["ts"],
+                                        "properties": {"ts": {"bsonType": "date"}}
+                                    },
+                                    "max": {
+                                        "bsonType": "object",
+                                        "required": ["ts"],
+                                        "properties": {"ts": {"bsonType": "date"}}
+                                    },
+                                    "closed": {"bsonType": "bool"}
+                                }
+                            },
+                            "data": {"bsonType": "object"},
+                            "meta": {
+
+                            }
+                        },
+                        "additionalProperties": false
+                    }
+                },
+                "clusteredIndex": true,
+                "timeseries": {
+                    "timeField": "ts",
+                    "metaField": "meta",
+                    "granularity": "hours",
+                    "bucketMaxSpanSeconds": 2592000
+                }
+            }
+        }
+    },
+    {
+        "operationType": "dropIndexes",
+        "ns": {"db": dbName, "coll": bucketsCollName},
+        "operationDescription": {
+            "indexes": [{
+                "v": 2,
+                "key": {"control.min.ts": 1, "control.max.ts": 1, "meta.b": 1},
+                "name": "dropMe"
+            }]
+        }
+    },
+    {"operationType": "drop", "ns": {"db": dbName, "coll": collName}},
+    {"operationType": "drop", "ns": {"db": dbName, "coll": bucketsCollName}}
+];
+
+cst.assertNextChangesEqual({cursor: curWithEvents, expectedChanges});
+
+const assertNoMoreBucketsEvents = (cur) => {
+    assert.soon(() => {
+        if (!cur.hasNext())
+            return true;
+        let event = cur.next();
+        assert(event.ns.coll !== bucketsCollName,
+               "shouldn't have seen without showSystemEvents" + tojson(event));
+        return !cur.hasNext();
+    });
+};
+
+// After all the expected events we should have no more events on the system.buckets ns.
+let curWithEventsNormal = new DBCommandCursor(testDB, {ok: 1, cursor: curWithEvents});
+assertNoMoreBucketsEvents(curWithEventsNormal);
+
+// No events cursor should have no system.buckets events.
+assertNoMoreBucketsEvents(curNoEvents);
+}());
diff --git a/src/mongo/db/pipeline/document_source_change_stream.h b/src/mongo/db/pipeline/document_source_change_stream.h
index 60013e64444..cbc00f6a7ba 100644
--- a/src/mongo/db/pipeline/document_source_change_stream.h
+++ b/src/mongo/db/pipeline/document_source_change_stream.h
@@ -243,9 +243,13 @@ public:
 
     // Default regex for collections match which prohibits system collections.
     static constexpr StringData kRegexAllCollections = R"((?!(\$|system\.)))"_sd;
-    // Regex matching all regular collections plus certain system collections.
+
+    // Regex matching all user collections plus collections exposed when 'showSystemEvents' is set.
+    // Does not match a collection named $ or a collection with 'system.' in the name.
+    // However, it will still match collection names starting with system.buckets or a collection
+    // exactly named system.js.
     static constexpr StringData kRegexAllCollectionsShowSystemEvents =
-        R"((?!(\$|system\.(?!(js$)))))"_sd;
+        R"((?!(\$|system\.(?!(js$|buckets\.)))))"_sd;
 
     static constexpr StringData kRegexAllDBs = R"(^(?!(admin|config|local)\.)[^.]+)"_sd;
     static constexpr StringData kRegexCmdColl = R"(\$cmd$)"_sd;