diff options
| -rw-r--r-- | jstests/change_streams/timeseries.js | 291 | ||||
| -rw-r--r-- | src/mongo/db/pipeline/document_source_change_stream.h | 8 |
2 files changed, 297 insertions, 2 deletions
diff --git a/jstests/change_streams/timeseries.js b/jstests/change_streams/timeseries.js new file mode 100644 index 00000000000..558e2e85a7a --- /dev/null +++ b/jstests/change_streams/timeseries.js @@ -0,0 +1,291 @@ +/** + * Basic test to make sure events from timeseries buckets collections look normal and don't get + * filtered. + * @tags: [ + * change_stream_does_not_expect_txns, + * assumes_unsharded_collection, + * requires_fcv_60, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/change_stream_util.js"); // For ChangeStreamTest. + +let testDB = db.getSiblingDB(jsTestName()); +testDB.dropDatabase(); +let dbName = testDB.getName(); + +let coll = testDB[jsTestName()]; +let collName = coll.getName(); +let bucketsCollName = "system.buckets." + collName; + +let cst = new ChangeStreamTest(testDB); +let curWithEvents = cst.startWatchingChanges({ + pipeline: [ + {$changeStream: {showExpandedEvents: true, showSystemEvents: true}}, + { + $project: { + documentKey: 0, + "fullDocument._id": 0, + collectionUUID: 0, + "stateBeforeChange.collectionOptions.uuid": 0, + clusterTime: 0, + wallTime: 0 + } + }, + {$match: {operationType: {$regex: "(?!shard)", $options: "i"}}} + ], + collection: 1 +}); +let curNoEvents = testDB.watch([], {showExpandedEvents: true}); + +assert.commandWorked(testDB.createCollection( + jsTestName(), + {timeseries: {timeField: "ts", metaField: "meta"}})); // on buckets ns and view ns +coll.createIndex({ts: 1, "meta.b": 1}, {name: "dropMe"}); // on buckets ns +coll.insertOne({_id: 1, ts: new Date(1000), meta: {a: 1}}); // on buckets ns +coll.insertOne({_id: 1, ts: new Date(1000), meta: {a: 1}}); // on buckets ns +coll.update({"meta.a": 1}, {$set: {"meta.b": 2}}); // on buckets ns +coll.remove({"meta.a": 1}); // on buckets ns +// collMod granularity. on both buckets ns and view ns +assert.commandWorked(testDB.runCommand({collMod: collName, timeseries: {granularity: "hours"}})); +// collMod expiration. just on buckets ns +assert.commandWorked(testDB.runCommand({collMod: collName, expireAfterSeconds: 1})); +coll.dropIndex("dropMe"); // on buckets ns +coll.drop(); // on buckets ns and view ns + +// document key, _id, uuid, cluster and wall times omitted +let expectedChanges = [ + { + "operationType": "create", + "ns": {"db": dbName, "coll": bucketsCollName}, + "operationDescription": { + "validator": { + "$jsonSchema": { + "bsonType": "object", + "required": ["_id", "control", "data"], + "properties": { + "_id": {"bsonType": "objectId"}, + "control": { + "bsonType": "object", + "required": ["version", "min", "max"], + "properties": { + "version": {"bsonType": "number"}, + "min": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "max": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "closed": {"bsonType": "bool"} + } + }, + "data": {"bsonType": "object"}, + "meta": { + + } + }, + "additionalProperties": false + } + }, + "clusteredIndex": true, + "timeseries": { + "timeField": "ts", + "metaField": "meta", + "granularity": "seconds", + "bucketMaxSpanSeconds": 3600 + } + } + }, + { + "operationType": "create", + "ns": {"db": dbName, "coll": collName}, + "operationDescription": { + "viewOn": bucketsCollName, + "pipeline": [{ + "$_internalUnpackBucket": + {"timeField": "ts", "metaField": "meta", "bucketMaxSpanSeconds": 3600} + }] + } + }, + { + "operationType": "createIndexes", + "ns": {"db": dbName, "coll": bucketsCollName}, + "operationDescription": { + "indexes": [{ + "v": 2, + "key": {"control.min.ts": 1, "control.max.ts": 1, "meta.b": 1}, + "name": "dropMe" + }] + } + }, + { + "operationType": "insert", + "fullDocument": { + "control": { + "version": 1, + "min": {"_id": 1, "ts": ISODate("1970-01-01T00:00:00Z")}, + "max": {"_id": 1, "ts": ISODate("1970-01-01T00:00:01Z")} + }, + "meta": {"a": 1}, + "data": {"_id": {"0": 1}, "ts": {"0": ISODate("1970-01-01T00:00:01Z")}} + }, + "ns": {"db": dbName, "coll": bucketsCollName} + }, + { + "operationType": "update", + "ns": {"db": dbName, "coll": bucketsCollName}, + "updateDescription": { + "updatedFields": {"data._id.1": 1, "data.ts.1": ISODate("1970-01-01T00:00:01Z")}, + "removedFields": [], + "truncatedArrays": [] + } + }, + {"operationType": "delete", "ns": {"db": dbName, "coll": bucketsCollName}}, + { + "operationType": "modify", + "ns": {"db": dbName, "coll": collName}, + "operationDescription": { + "viewOn": "system.buckets.timeseries", + "pipeline": [{ + "$_internalUnpackBucket": + {"timeField": "ts", "metaField": "meta", "bucketMaxSpanSeconds": 2592000} + }] + } + }, + { + "operationType": "modify", + "ns": {"db": dbName, "coll": bucketsCollName}, + "operationDescription": {"timeseries": {"granularity": "hours"}}, + "stateBeforeChange": { + "collectionOptions": { + "validator": { + "$jsonSchema": { + "bsonType": "object", + "required": ["_id", "control", "data"], + "properties": { + "_id": {"bsonType": "objectId"}, + "control": { + "bsonType": "object", + "required": ["version", "min", "max"], + "properties": { + "version": {"bsonType": "number"}, + "min": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "max": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "closed": {"bsonType": "bool"} + } + }, + "data": {"bsonType": "object"}, + "meta": { + + } + }, + "additionalProperties": false + } + }, + "clusteredIndex": true, + "timeseries": { + "timeField": "ts", + "metaField": "meta", + "granularity": "seconds", + "bucketMaxSpanSeconds": 3600 + } + } + } + }, + { + "operationType": "modify", + "ns": {"db": dbName, "coll": bucketsCollName}, + "operationDescription": {"expireAfterSeconds": NumberLong(1)}, + "stateBeforeChange": { + "collectionOptions": { + "validator": { + "$jsonSchema": { + "bsonType": "object", + "required": ["_id", "control", "data"], + "properties": { + "_id": {"bsonType": "objectId"}, + "control": { + "bsonType": "object", + "required": ["version", "min", "max"], + "properties": { + "version": {"bsonType": "number"}, + "min": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "max": { + "bsonType": "object", + "required": ["ts"], + "properties": {"ts": {"bsonType": "date"}} + }, + "closed": {"bsonType": "bool"} + } + }, + "data": {"bsonType": "object"}, + "meta": { + + } + }, + "additionalProperties": false + } + }, + "clusteredIndex": true, + "timeseries": { + "timeField": "ts", + "metaField": "meta", + "granularity": "hours", + "bucketMaxSpanSeconds": 2592000 + } + } + } + }, + { + "operationType": "dropIndexes", + "ns": {"db": dbName, "coll": bucketsCollName}, + "operationDescription": { + "indexes": [{ + "v": 2, + "key": {"control.min.ts": 1, "control.max.ts": 1, "meta.b": 1}, + "name": "dropMe" + }] + } + }, + {"operationType": "drop", "ns": {"db": dbName, "coll": collName}}, + {"operationType": "drop", "ns": {"db": dbName, "coll": bucketsCollName}} +]; + +cst.assertNextChangesEqual({cursor: curWithEvents, expectedChanges}); + +const assertNoMoreBucketsEvents = (cur) => { + assert.soon(() => { + if (!cur.hasNext()) + return true; + let event = cur.next(); + assert(event.ns.coll !== bucketsCollName, + "shouldn't have seen without showSystemEvents" + tojson(event)); + return !cur.hasNext(); + }); +}; + +// After all the expected events we should have no more events on the system.buckets ns. +let curWithEventsNormal = new DBCommandCursor(testDB, {ok: 1, cursor: curWithEvents}); +assertNoMoreBucketsEvents(curWithEventsNormal); + +// No events cursor should have no system.buckets events. +assertNoMoreBucketsEvents(curNoEvents); +}()); diff --git a/src/mongo/db/pipeline/document_source_change_stream.h b/src/mongo/db/pipeline/document_source_change_stream.h index 60013e64444..cbc00f6a7ba 100644 --- a/src/mongo/db/pipeline/document_source_change_stream.h +++ b/src/mongo/db/pipeline/document_source_change_stream.h @@ -243,9 +243,13 @@ public: // Default regex for collections match which prohibits system collections. static constexpr StringData kRegexAllCollections = R"((?!(\$|system\.)))"_sd; - // Regex matching all regular collections plus certain system collections. + + // Regex matching all user collections plus collections exposed when 'showSystemEvents' is set. + // Does not match a collection named $ or a collection with 'system.' in the name. + // However, it will still match collection names starting with system.buckets or a collection + // exactly named system.js. static constexpr StringData kRegexAllCollectionsShowSystemEvents = - R"((?!(\$|system\.(?!(js$)))))"_sd; + R"((?!(\$|system\.(?!(js$|buckets\.)))))"_sd; static constexpr StringData kRegexAllDBs = R"(^(?!(admin|config|local)\.)[^.]+)"_sd; static constexpr StringData kRegexCmdColl = R"(\$cmd$)"_sd; |