diff options
| author | Nick Zolnierz <nicholas.zolnierz@mongodb.com> | 2019-05-22 13:45:47 -0400 |
|---|---|---|
| committer | Nick Zolnierz <nicholas.zolnierz@mongodb.com> | 2019-05-23 12:08:20 -0400 |
| commit | f2b7b6ecde3f3e6d164609163129ccbebb1426d7 (patch) | |
| tree | de70802c583c35a00b15d0d95f7847b1d6d584f1 | |
| parent | fedc4754c04d71434fb7901d83e2947aeaeabe77 (diff) | |
| download | mongo-f2b7b6ecde3f3e6d164609163129ccbebb1426d7.tar.gz | |
SERVER-41198 Translate $out tests in jstests/aggregation/sources/out to use $merge
22 files changed, 1219 insertions, 886 deletions
diff --git a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml index 7e4a6a56436..e552fb10f0b 100644 --- a/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/aggregation_read_concern_majority_passthrough.yml @@ -7,7 +7,7 @@ selector: - jstests/aggregation/extras/*.js - jstests/aggregation/data/*.js # Blocked by SERVER-37191 - - jstests/aggregation/sources/out/exchange_explain.js + - jstests/aggregation/sources/merge/exchange_explain.js # This test specifies a $out stage not as the last stage in the aggregation pipeline, causing a # non-local readConcern to erroneously be sent with the command. - jstests/aggregation/sources/out/required_last_position.js diff --git a/jstests/aggregation/explain_writing_aggs.js b/jstests/aggregation/explain_writing_aggs.js index 1ef34c53d7d..8eab952b7cd 100644 --- a/jstests/aggregation/explain_writing_aggs.js +++ b/jstests/aggregation/explain_writing_aggs.js @@ -10,7 +10,7 @@ load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos(). load("jstests/libs/analyze_plan.js"); // For getAggPlanStage(). - load("jstests/aggregation/extras/out_helpers.js"); // For withEachOutMode(). + load("jstests/aggregation/extras/out_helpers.js"); // For withEachMergeMode(). let sourceColl = db.explain_writing_aggs_source; let targetColl = db.explain_writing_aggs_target; diff --git a/jstests/aggregation/extras/out_helpers.js b/jstests/aggregation/extras/out_helpers.js index 5974da0fbd5..6c6a0e13f8a 100644 --- a/jstests/aggregation/extras/out_helpers.js +++ b/jstests/aggregation/extras/out_helpers.js @@ -4,13 +4,6 @@ load("jstests/libs/fixture_helpers.js"); // For isSharded. -// TODO SERVER-40432: Remove this helper as it shouldn't be needed anymore. -function withEachOutMode(callback) { - callback("replaceCollection"); - callback("insertDocuments"); - callback("replaceDocuments"); -} - /** * Executes the callback function with each valid combination of 'whenMatched' and 'whenNotMatched' * modes (as named arguments). Note that one mode is a pipeline. @@ -33,51 +26,6 @@ function withEachMergeMode(callback) { callback({whenMatchedMode: [], whenNotMatchedMode: "discard"}); } -// TODO SERVER-40432: Remove this helper as it shouldn't be needed anymore. -function assertUniqueKeyIsInvalid({source, target, uniqueKey, options, prevStages}) { - withEachOutMode((mode) => { - if (mode === "replaceCollection" && FixtureHelpers.isSharded(target)) - return; - - prevStages = (prevStages || []); - const pipeline = prevStages.concat([{ - $out: { - db: target.getDB().getName(), - to: target.getName(), - mode: mode, - uniqueKey: uniqueKey - } - }]); - - const cmd = {aggregate: source.getName(), pipeline: pipeline, cursor: {}}; - assert.commandFailedWithCode(source.getDB().runCommand(Object.merge(cmd, options)), 50938); - }); -} - -// TODO SERVER-40432: Remove this helper as it shouldn't be needed anymore. -function assertUniqueKeyIsValid({source, target, uniqueKey, options, prevStages}) { - withEachOutMode((mode) => { - if (mode === "replaceCollection" && FixtureHelpers.isSharded(target)) - return; - - prevStages = (prevStages || []); - let outStage = { - db: target.getDB().getName(), - to: target.getName(), - mode: mode, - }; - - // Do not include the uniqueKey in the command if the caller did not specify it. - if (uniqueKey !== undefined) { - outStage = Object.extend(outStage, {uniqueKey: uniqueKey}); - } - const pipeline = prevStages.concat([{$out: outStage}]); - - assert.commandWorked(target.remove({})); - assert.doesNotThrow(() => source.aggregate(pipeline, options)); - }); -} - function assertFailsWithoutUniqueIndex({source, target, onFields, options, prevStages}) { withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { prevStages = (prevStages || []); @@ -90,8 +38,11 @@ function assertFailsWithoutUniqueIndex({source, target, onFields, options, prevS } }]); + // In sharded passthrough suites, the error code may be different depending on where we + // extract the "on" fields. const cmd = {aggregate: source.getName(), pipeline: pipeline, cursor: {}}; - assert.commandFailedWithCode(source.getDB().runCommand(Object.merge(cmd, options)), 51190); + assert.commandFailedWithCode(source.getDB().runCommand(Object.merge(cmd, options)), + [51183, 51190]); }); } diff --git a/jstests/aggregation/sources/merge/batch_writes.js b/jstests/aggregation/sources/merge/batch_writes.js new file mode 100644 index 00000000000..bc76f2d5e7c --- /dev/null +++ b/jstests/aggregation/sources/merge/batch_writes.js @@ -0,0 +1,74 @@ +// Tests the behavior of an $merge stage which encounters an error in the middle of processing. We +// don't guarantee any particular behavior in this scenario, but this test exists to make sure +// nothing horrendous happens and to characterize the current behavior. +// @tags: [assumes_unsharded_collection] +(function() { + "use strict"; + + load("jstests/aggregation/extras/out_helpers.js"); // For withEachMergeMode. + load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. + + const coll = db.batch_writes; + const outColl = db.batch_writes_out; + coll.drop(); + outColl.drop(); + + // Test with 2 very large documents that do not fit into a single batch. + const kSize15MB = 15 * 1024 * 1024; + const largeArray = new Array(kSize15MB).join("a"); + assert.commandWorked(coll.insert({_id: 0, a: largeArray})); + assert.commandWorked(coll.insert({_id: 1, a: largeArray})); + + // Make sure the $merge succeeds without any duplicate keys. + withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { + // Skip the combination of merge modes which will fail depending on the contents of the + // source and target collection, as this will cause the aggregation to fail. + if (whenMatchedMode == "fail" || whenNotMatchedMode == "fail") + return; + + coll.aggregate([{ + $merge: { + into: outColl.getName(), + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }]); + assert.eq(whenNotMatchedMode == "discard" ? 0 : 2, outColl.find().itcount()); + outColl.drop(); + }); + + coll.drop(); + for (let i = 0; i < 10; i++) { + assert.commandWorked(coll.insert({_id: i, a: i})); + } + + // Create a unique index on 'a' in the output collection to create a unique key violation when + // running the $merge. The second document to be written ({_id: 1, a: 1}) will conflict with the + // existing document in the output collection. We use a unique index on a field other than _id + // because whenMatched: "replaceWithNew" will not change _id when one already exists. + outColl.drop(); + assert.commandWorked(outColl.insert({_id: 2, a: 1})); + assert.commandWorked(outColl.createIndex({a: 1}, {unique: true})); + + // Test that the writes for $merge are unordered, meaning the operation continues even if it + // encounters a duplicate key error. We don't guarantee any particular behavior in this case, + // but this test is meant to characterize the current behavior. + assertErrorCode( + coll, + [{$merge: {into: outColl.getName(), whenMatched: "fail", whenNotMatched: "insert"}}], + ErrorCodes.DuplicateKey); + assert.soon(() => { + return outColl.find().itcount() == 9; + }); + + assertErrorCode( + coll, + [{ + $merge: + {into: outColl.getName(), whenMatched: "replaceWithNew", whenNotMatched: "insert"} + }], + ErrorCodes.DuplicateKey); + assert.soon(() => { + return outColl.find().itcount() == 9; + }); +}()); diff --git a/jstests/aggregation/sources/out/bypass_doc_validation.js b/jstests/aggregation/sources/merge/bypass_doc_validation.js index 41d5ac4d41d..62e1d7a75bd 100644 --- a/jstests/aggregation/sources/out/bypass_doc_validation.js +++ b/jstests/aggregation/sources/merge/bypass_doc_validation.js @@ -1,5 +1,5 @@ /** - * Tests for $out with bypassDocumentValidation. + * Tests for $merge with bypassDocumentValidation. * * @tags: [assumes_unsharded_collection] */ @@ -21,22 +21,30 @@ // Test that the bypassDocumentValidation flag is passed through to the writes on the output // collection. (function testBypassDocValidationTrue() { - sourceColl.aggregate([{$out: targetColl.getName()}], {bypassDocumentValidation: true}); - assert.eq([{_id: 0, a: 1}], targetColl.find().toArray()); - - sourceColl.aggregate([{$out: {to: targetColl.getName(), mode: "replaceCollection"}}], - {bypassDocumentValidation: true}); + sourceColl.aggregate([{$merge: targetColl.getName()}], {bypassDocumentValidation: true}); assert.eq([{_id: 0, a: 1}], targetColl.find().toArray()); sourceColl.aggregate( - [{$addFields: {a: 3}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}], + [ + {$addFields: {a: 3}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ], {bypassDocumentValidation: true}); assert.eq([{_id: 0, a: 3}], targetColl.find().toArray()); sourceColl.aggregate( [ {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, - {$out: {to: targetColl.getName(), mode: "insertDocuments"}} + { + $merge: + {into: targetColl.getName(), whenMatched: "fail", whenNotMatched: "insert"} + } ], {bypassDocumentValidation: true}); assert.eq([{_id: 0, a: 3}, {_id: 1, a: 4}], targetColl.find().sort({_id: 1}).toArray()); @@ -45,11 +53,28 @@ // Test that mode "replaceDocuments" passes without the bypassDocumentValidation flag if the // updated doc is valid. (function testReplacementStyleUpdateWithoutBypass() { - sourceColl.aggregate( - [{$addFields: {a: 2}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}]); + sourceColl.aggregate([ + {$addFields: {a: 2}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ]); assert.eq([{_id: 0, a: 2}], targetColl.find({_id: 0}).toArray()); sourceColl.aggregate( - [{$addFields: {a: 2}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}], + [ + {$addFields: {a: 2}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ], {bypassDocumentValidation: false}); assert.eq([{_id: 0, a: 2}], targetColl.find({_id: 0}).toArray()); }()); @@ -57,40 +82,46 @@ function assertDocValidationFailure(cmdOptions) { assert.commandWorked(targetColl.remove({})); assertErrorCode(sourceColl, - [{$out: targetColl.getName()}], + [{$merge: targetColl.getName()}], ErrorCodes.DocumentValidationFailure, "Expected failure without bypass set", cmdOptions); assertErrorCode(sourceColl, - [{$out: {to: targetColl.getName(), mode: "replaceCollection"}}], + [ + {$addFields: {a: 3}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ], ErrorCodes.DocumentValidationFailure, "Expected failure without bypass set", cmdOptions); assertErrorCode( sourceColl, - [{$addFields: {a: 3}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}], + [ + {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, + { + $merge: + {into: targetColl.getName(), whenMatched: "fail", whenNotMatched: "insert"} + } + ], ErrorCodes.DocumentValidationFailure, "Expected failure without bypass set", cmdOptions); - - assertErrorCode(sourceColl, - [ - {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, - {$out: {to: targetColl.getName(), mode: "insertDocuments"}} - ], - ErrorCodes.DocumentValidationFailure, - "Expected failure without bypass set", - cmdOptions); assert.eq(0, targetColl.find().itcount()); } - // Test that $out fails if the output document is not valid, and the bypassDocumentValidation + // Test that $merge fails if the output document is not valid, and the bypassDocumentValidation // flag is not set. assertDocValidationFailure({}); - // Test that $out fails if the output document is not valid, and the bypassDocumentValidation + // Test that $merge fails if the output document is not valid, and the bypassDocumentValidation // flag is explicitly set to false. assertDocValidationFailure({bypassDocumentValidation: false}); @@ -100,19 +131,24 @@ targetColl.drop(); assert.commandWorked(testDB.runCommand({collMod: sourceColl.getName(), validator: {a: 1}})); - sourceColl.aggregate([{$out: targetColl.getName()}]); + sourceColl.aggregate([{$merge: targetColl.getName()}]); assert.eq([{_id: 0, a: 1}], targetColl.find().toArray()); - sourceColl.aggregate([{$out: {to: targetColl.getName(), mode: "replaceCollection"}}]); - assert.eq([{_id: 0, a: 1}], targetColl.find().toArray()); - - sourceColl.aggregate( - [{$addFields: {a: 3}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}]); + sourceColl.aggregate([ + {$addFields: {a: 3}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ]); assert.eq([{_id: 0, a: 3}], targetColl.find().toArray()); sourceColl.aggregate([ {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, - {$out: {to: targetColl.getName(), mode: "insertDocuments"}} + {$merge: {into: targetColl.getName(), whenMatched: "fail", whenNotMatched: "insert"}} ]); assert.eq([{_id: 0, a: 3}, {_id: 1, a: 4}], targetColl.find().sort({_id: 1}).toArray()); }()); @@ -124,16 +160,25 @@ sourceColl.drop(); assert.commandWorked(sourceColl.insert({_id: 0, a: 1})); - sourceColl.aggregate([{$out: targetColl.getName()}], {bypassDocumentValidation: 5}); + sourceColl.aggregate([{$merge: targetColl.getName()}], {bypassDocumentValidation: 5}); assert.eq([{_id: 0, a: 1}], targetColl.find().toArray()); sourceColl.aggregate( - [{$addFields: {a: 3}}, {$out: {to: targetColl.getName(), mode: "replaceDocuments"}}], + [ + {$addFields: {a: 3}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ], {bypassDocumentValidation: "false"}); assert.eq([{_id: 0, a: 3}], targetColl.find().toArray()); }()); - // Test bypassDocumentValidation with $out to a collection in a foreign database. + // Test bypassDocumentValidation with $merge to a collection in a foreign database. (function testForeignDb() { const foreignDB = db.getSiblingDB("foreign_db"); const foreignColl = foreignDB.foreign_coll; @@ -145,10 +190,13 @@ [ {$addFields: {a: 3}}, { - $out: { - db: foreignDB.getName(), - to: foreignColl.getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: foreignDB.getName(), + coll: foreignColl.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } } ], @@ -159,8 +207,14 @@ [ {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, { - $out: - {db: foreignDB.getName(), to: foreignColl.getName(), mode: "insertDocuments"} + $merge: { + into: { + db: foreignDB.getName(), + coll: foreignColl.getName(), + }, + whenMatched: "fail", + whenNotMatched: "insert" + } } ], {bypassDocumentValidation: true}); @@ -171,25 +225,33 @@ [ {$addFields: {a: 3}}, { - $out: { - db: foreignDB.getName(), - to: foreignColl.getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: foreignDB.getName(), + coll: foreignColl.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } } ], ErrorCodes.DocumentValidationFailure); - assertErrorCode( - sourceColl, - [ - {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, - { - $out: - {db: foreignDB.getName(), to: foreignColl.getName(), mode: "insertDocuments"} - } - ], - ErrorCodes.DocumentValidationFailure); + assertErrorCode(sourceColl, + [ + {$replaceRoot: {newRoot: {_id: 1, a: 4}}}, + { + $merge: { + into: { + db: foreignDB.getName(), + coll: foreignColl.getName(), + }, + whenMatched: "fail", + whenNotMatched: "insert" + } + } + ], + ErrorCodes.DocumentValidationFailure); assert.eq(0, foreignColl.find().itcount()); }()); }()); diff --git a/jstests/aggregation/sources/merge/disallowed_in_lookup.js b/jstests/aggregation/sources/merge/disallowed_in_lookup.js new file mode 100644 index 00000000000..f5f1685effd --- /dev/null +++ b/jstests/aggregation/sources/merge/disallowed_in_lookup.js @@ -0,0 +1,94 @@ +// Tests that $merge cannot be used within a $lookup pipeline. +(function() { + "use strict"; + + load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. + load("jstests/libs/collection_drop_recreate.js"); // For assertDropCollection. + load("jstests/noPassthrough/libs/server_parameter_helpers.js"); // For setParameterOnAllHosts. + load("jstests/libs/discover_topology.js"); // For findNonConfigNodes. + load("jstests/libs/fixture_helpers.js"); // For isSharded. + + const kErrorCodeMergeBannedInLookup = 51047; + const kErrorCodeMergeLastStageOnly = 40601; + const coll = db.merge_in_lookup_not_allowed; + coll.drop(); + + const from = db.merge_in_lookup_not_allowed_from; + from.drop(); + + if (FixtureHelpers.isSharded(from)) { + setParameterOnAllHosts(DiscoverTopology.findNonConfigNodes(db.getMongo()), + "internalQueryAllowShardedLookup", + true); + } + + let pipeline = [ + { + $lookup: { + pipeline: [{$merge: {into: "out_collection", on: "_id"}}], + from: from.getName(), + as: "c", + } + }, + ]; + assertErrorCode(coll, pipeline, kErrorCodeMergeBannedInLookup); + + pipeline = [ + { + $lookup: { + pipeline: [{$project: {x: 0}}, {$merge: {into: "out_collection", on: "_id"}}], + from: from.getName(), + as: "c", + } + }, + ]; + assertErrorCode(coll, pipeline, kErrorCodeMergeBannedInLookup); + + pipeline = [ + { + $lookup: { + pipeline: [{$merge: {into: "out_collection", on: "_id"}}, {$match: {x: true}}], + from: from.getName(), + as: "c", + } + }, + ]; + // Pipeline will fail because $merge is not last in the subpipeline. + // Validation for $merge in a $lookup's subpipeline occurs at a later point. + assertErrorCode(coll, pipeline, kErrorCodeMergeLastStageOnly); + + // Create view which contains $merge within $lookup. + assertDropCollection(coll.getDB(), "view1"); + + pipeline = [ + { + $lookup: { + pipeline: [{$merge: {into: "out_collection", on: "_id"}}], + from: from.getName(), + as: "c", + } + }, + ]; + // Pipeline will fail because $merge is not allowed to exist within a $lookup. + // Validation for $merge in a view occurs at a later point. + const cmdRes = + coll.getDB().runCommand({create: "view1", viewOn: coll.getName(), pipeline: pipeline}); + assert.commandFailedWithCode(cmdRes, kErrorCodeMergeBannedInLookup); + + // Test that a $merge without an explicit "on" field still fails within a $lookup. Note that we + // may get a different error code since the pipeline will passthrough to the shards in the + // sharded passthrough suites, and fail because mongod expects mongos to populate "on" if the + // user does not specify it. + pipeline = [ + { + $lookup: { + pipeline: [{$merge: {into: "out_collection"}}], + from: from.getName(), + as: "c", + } + }, + ]; + assert.commandFailedWithCode( + db.runCommand({aggregate: coll.getName(), pipeline: pipeline, cursor: {}}), + [kErrorCodeMergeBannedInLookup, 51124]); +}()); diff --git a/jstests/aggregation/sources/out/exchange_explain.js b/jstests/aggregation/sources/merge/exchange_explain.js index ebf85336677..17d350557d1 100644 --- a/jstests/aggregation/sources/out/exchange_explain.js +++ b/jstests/aggregation/sources/merge/exchange_explain.js @@ -1,5 +1,5 @@ /** - * Test $out and exchange with explain. + * Test $merge and exchange with explain. * * @tags: [requires_sharding] */ @@ -13,33 +13,39 @@ load('jstests/aggregation/extras/utils.js'); const mongosDB = st.s.getDB("test_db"); const inColl = mongosDB["inColl"]; - const outCollRange = mongosDB["outCollRange"]; - const outCollRangeOtherField = mongosDB["outCollRangeOtherField"]; - const outCollHash = mongosDB["outCollHash"]; + const targetCollRange = mongosDB["targetCollRange"]; + const targetCollRangeOtherField = mongosDB["targetCollRangeOtherField"]; + const targetCollHash = mongosDB["targetCollHash"]; const numDocs = 1000; - function runExplainQuery(outColl) { + function runExplainQuery(targetColl) { return inColl.explain("allPlansExecution").aggregate([ {$group: {_id: "$a", a: {$avg: "$a"}}}, { - $out: { - to: outColl.getName(), - db: outColl.getDB().getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: targetColl.getDB().getName(), + coll: targetColl.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } } ]); } - function runRealQuery(outColl) { + function runRealQuery(targetColl) { return inColl.aggregate([ {$group: {_id: "$a", a: {$avg: "$a"}}}, { - $out: { - to: outColl.getName(), - db: outColl.getDB().getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: targetColl.getDB().getName(), + coll: targetColl.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } } ]); @@ -63,12 +69,12 @@ load('jstests/aggregation/extras/utils.js'); assert.commandWorked(bulk.execute()); // Shard the output collections. - st.shardColl(outCollRange, {_id: 1}, {_id: 500}, {_id: 500}, mongosDB.getName()); - st.shardColl(outCollRangeOtherField, {b: 1}, {b: 500}, {b: 500}, mongosDB.getName()); - st.shardColl(outCollHash, {_id: "hashed"}, false, false, mongosDB.getName()); + st.shardColl(targetCollRange, {_id: 1}, {_id: 500}, {_id: 500}, mongosDB.getName()); + st.shardColl(targetCollRangeOtherField, {b: 1}, {b: 500}, {b: 500}, mongosDB.getName()); + st.shardColl(targetCollHash, {_id: "hashed"}, false, false, mongosDB.getName()); // Run the explain. We expect to see the range based exchange here. - let explain = runExplainQuery(outCollRange); + let explain = runExplainQuery(targetCollRange); // Make sure we see the exchange in the explain output. assert.eq(explain.mergeType, "exchange", tojson(explain)); @@ -77,12 +83,12 @@ load('jstests/aggregation/extras/utils.js'); assert.eq(exchangeSpec.key, {_id: 1}); // Run the real query. - runRealQuery(outCollRange); - let results = outCollRange.aggregate([{'$count': "count"}]).next().count; + runRealQuery(targetCollRange); + let results = targetCollRange.aggregate([{'$count': "count"}]).next().count; assert.eq(results, numDocs); // Rerun the same query with the hash based exchange. - explain = runExplainQuery(outCollHash); + explain = runExplainQuery(targetCollHash); // Make sure we see the exchange in the explain output. assert.eq(explain.mergeType, "exchange", tojson(explain)); @@ -91,39 +97,44 @@ load('jstests/aggregation/extras/utils.js'); assert.eq(exchangeSpec.key, {_id: "hashed"}); // Run the real query. - runRealQuery(outCollHash); - results = outCollHash.aggregate([{'$count': "count"}]).next().count; + runRealQuery(targetCollHash); + results = targetCollHash.aggregate([{'$count': "count"}]).next().count; assert.eq(results, numDocs); - // This should fail with the error '$out write error: uniqueKey field 'b' cannot be missing, - // null, undefined or an array.' as we are trying to insert an array value. + // This should fail because the "on" field ('b' in this case, the shard key of the target + // collection) cannot be an array. assertErrorCode(inColl, [{ - $out: { - to: outCollRangeOtherField.getName(), - db: outCollRangeOtherField.getDB().getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: targetCollRangeOtherField.getDB().getName(), + coll: targetCollRangeOtherField.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } }], 51132); // Turn off the exchange and rerun the query. assert.commandWorked(mongosDB.adminCommand({setParameter: 1, internalQueryDisableExchange: 1})); - explain = runExplainQuery(outCollRange); + explain = runExplainQuery(targetCollRange); // Make sure there is no exchange. assert.eq(explain.mergeType, "anyShard", tojson(explain)); assert(explain.hasOwnProperty("splitPipeline"), tojson(explain)); assert(!explain.splitPipeline.hasOwnProperty("exchange"), tojson(explain)); - // This should fail with the same error '$out write error: uniqueKey field 'b' cannot be - // missing, null, undefined or an array.' as before even if we are not running the exchange. + // This should fail similar to before even if we are not running the exchange. assertErrorCode(inColl, [{ - $out: { - to: outCollRangeOtherField.getName(), - db: outCollRangeOtherField.getDB().getName(), - mode: "replaceDocuments" + $merge: { + into: { + db: targetCollRangeOtherField.getDB().getName(), + coll: targetCollRangeOtherField.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert" } }], 51132); @@ -145,7 +156,13 @@ load('jstests/aggregation/extras/utils.js'); assert.commandFailedWithCode(mongosDB.runCommand({ aggregate: inColl.getName(), - pipeline: [{$out: {to: outCollRange.getName(), mode: "replaceDocuments"}}], + pipeline: [{ + $merge: { + into: targetCollRange.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + }], cursor: {}, exchange: { policy: "keyRange", diff --git a/jstests/aggregation/sources/merge/merge_to_referenced_collection.js b/jstests/aggregation/sources/merge/merge_to_referenced_collection.js new file mode 100644 index 00000000000..c04922024e8 --- /dev/null +++ b/jstests/aggregation/sources/merge/merge_to_referenced_collection.js @@ -0,0 +1,192 @@ +/** + * Tests that the server behaves as expected when an $merge stage is targeting a collection which is + * involved in the aggregate in some other way, e.g. as the source namespace or via a $lookup. We + * disallow this combination in an effort to prevent the "halloween problem" of a never-ending + * query. + * + * This test issues queries over views, so cannot be run in passthroughs which implicitly shard + * collections. + * @tags: [assumes_unsharded_collection] + */ +(function() { + 'use strict'; + + load('jstests/aggregation/extras/out_helpers.js'); // For 'withEachMergeMode'. + load('jstests/libs/fixture_helpers.js'); // For 'FixtureHelpers'. + + const testDB = db.getSiblingDB("merge_to_referenced_coll"); + const coll = testDB.test; + + withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { + coll.drop(); + + // Seed the collection to ensure each pipeline will actually do something. + assert.commandWorked(coll.insert({_id: 0})); + + // Each of the following assertions will somehow use $merge to write to a namespace that is + // being read from elsewhere in the pipeline. + const assertFailsWithCode = ((fn) => { + const error = assert.throws(fn); + assert.contains(error.code, [51188, 51079]); + }); + + // Test $merge to the aggregate command's source collection. + assertFailsWithCode(() => coll.aggregate([{ + $merge: { + into: coll.getName(), + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }])); + + // Test $merge to the same namespace as a $lookup which is the same as the aggregate + // command's source collection. + assertFailsWithCode(() => coll.aggregate([ + {$lookup: {from: coll.getName(), as: "x", localField: "f_id", foreignField: "_id"}}, + { + $merge: { + into: coll.getName(), + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + + // Test $merge to the same namespace as a $lookup which is *not* the same as the aggregate + // command's source collection. + assertFailsWithCode(() => coll.aggregate([ + {$lookup: {from: "bar", as: "x", localField: "f_id", foreignField: "_id"}}, + { + $merge: { + into: "bar", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + + // Test $merge to the same namespace as a $graphLookup. + assertFailsWithCode(() => coll.aggregate([ + { + $graphLookup: { + from: "bar", + startWith: "$_id", + connectFromField: "_id", + connectToField: "parent_id", + as: "graph", + } + }, + { + $merge: { + into: "bar", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + + // Test $merge to the same namespace as a $lookup which is nested within another $lookup. + assertFailsWithCode(() => coll.aggregate([ + { + $lookup: { + from: "bar", + as: "x", + let : {}, + pipeline: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}] + } + }, + { + $merge: { + into: "TARGET", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + // Test $merge to the same namespace as a $lookup which is nested within a $facet. + assertFailsWithCode(() => coll.aggregate([ + { + $facet: { + y: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}], + } + }, + { + $merge: { + into: "TARGET", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + assertFailsWithCode(() => coll.aggregate([ + { + $facet: { + x: [{$lookup: {from: "other", as: "y", pipeline: []}}], + y: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}], + } + }, + { + $merge: { + into: "TARGET", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + + // Test that we use the resolved namespace of a view to detect this sort of halloween + // problem. + assert.commandWorked( + testDB.runCommand({create: "view_on_TARGET", viewOn: "TARGET", pipeline: []})); + assertFailsWithCode(() => testDB.view_on_TARGET.aggregate([{ + $merge: { + into: "TARGET", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }])); + assertFailsWithCode(() => coll.aggregate([ + { + $facet: { + x: [{$lookup: {from: "other", as: "y", pipeline: []}}], + y: [{ + $lookup: { + from: "yet_another", + as: "y", + pipeline: [{$lookup: {from: "view_on_TARGET", as: "z", pipeline: []}}] + } + }], + } + }, + { + $merge: { + into: "TARGET", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } + ])); + + function generateNestedPipeline(foreignCollName, numLevels) { + let pipeline = [{"$lookup": {pipeline: [], from: foreignCollName, as: "same"}}]; + + for (let level = 1; level < numLevels; level++) { + pipeline = [{"$lookup": {pipeline: pipeline, from: foreignCollName, as: "same"}}]; + } + + return pipeline; + } + + const nestedPipeline = generateNestedPipeline("lookup", 20).concat([{ + $merge: { + into: "lookup", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }]); + assertFailsWithCode(() => coll.aggregate(nestedPipeline)); + + testDB.dropDatabase(); // Clear any of the collections which would be created by the + // successful "replaceCollection" mode of this test. + }); +}()); diff --git a/jstests/aggregation/sources/merge/merge_to_same_collection.js b/jstests/aggregation/sources/merge/merge_to_same_collection.js new file mode 100644 index 00000000000..bf384072a88 --- /dev/null +++ b/jstests/aggregation/sources/merge/merge_to_same_collection.js @@ -0,0 +1,28 @@ +/** + * Tests that $merge fails when the target collection is the aggregation collection. + * + * @tags: [assumes_unsharded_collection] +*/ +(function() { + "use strict"; + + load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. + + const coll = db.name; + coll.drop(); + + const nDocs = 10; + for (let i = 0; i < nDocs; i++) { + assert.commandWorked(coll.insert({_id: i, a: i})); + } + + assertErrorCode( + coll, + [{$merge: {into: coll.getName(), whenMatched: "replaceWithNew", whenNotMatched: "insert"}}], + 51188); + + assertErrorCode( + coll, + [{$merge: {into: coll.getName(), whenMatched: "fail", whenNotMatched: "insert"}}], + 51188); +}()); diff --git a/jstests/aggregation/sources/out/mode_insert_documents.js b/jstests/aggregation/sources/merge/mode_fail_insert.js index 9cbe3dd8c93..7cfd6aee02e 100644 --- a/jstests/aggregation/sources/out/mode_insert_documents.js +++ b/jstests/aggregation/sources/merge/mode_fail_insert.js @@ -1,4 +1,4 @@ -// Tests the behavior of $out with mode "insertDocuments". +// Tests the behavior of $merge with whenMatched: "fail" and whenNotMatched: "insert". // @tags: [assumes_unsharded_collection, assumes_no_implicit_collection_creation_after_drop] (function() { "use strict"; @@ -6,16 +6,17 @@ load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos. - const coll = db.mode_insert_documents; + const coll = db.merge_insert_only; coll.drop(); - const targetColl = db.mode_insert_documents_out; + const targetColl = db.merge_insert_only_out; targetColl.drop(); - const pipeline = [{$out: {to: targetColl.getName(), mode: "insertDocuments"}}]; + const pipeline = + [{$merge: {into: targetColl.getName(), whenMatched: "fail", whenNotMatched: "insert"}}]; // - // Test $out with a non-existent output collection. + // Test $merge with a non-existent output collection. // assert.commandWorked(coll.insert({_id: 0})); @@ -23,7 +24,7 @@ assert.eq(1, targetColl.find().itcount()); // - // Test $out with an existing output collection. + // Test $merge with an existing output collection. // assert.commandWorked(coll.remove({_id: 0})); assert.commandWorked(coll.insert({_id: 1})); @@ -31,12 +32,12 @@ assert.eq(2, targetColl.find().itcount()); // - // Test that $out fails if there's a duplicate key error. + // Test that $merge fails if there's a duplicate key error. // assertErrorCode(coll, pipeline, ErrorCodes.DuplicateKey); // - // Test that $out will preserve the indexes and options of the output collection. + // Test that $merge will preserve the indexes and options of the output collection. // const validator = {a: {$gt: 0}}; targetColl.drop(); @@ -55,7 +56,7 @@ assert.eq(validator, listColl.cursor.firstBatch[0].options["validator"]); // - // Test that $out fails if it violates a unique index constraint. + // Test that $merge fails if it violates a unique index constraint. // coll.drop(); assert.commandWorked(coll.insert([{_id: 0, a: 0}, {_id: 1, a: 0}])); @@ -65,7 +66,7 @@ assertErrorCode(coll, pipeline, ErrorCodes.DuplicateKey); // - // Test that an $out aggregation succeeds even if the _id is stripped out and the "uniqueKey" + // Test that a $merge aggregation succeeds even if the _id is stripped out and the "unique key" // is the document key, which will be _id for a new collection. // coll.drop(); @@ -73,13 +74,13 @@ targetColl.drop(); assert.doesNotThrow(() => coll.aggregate([ {$project: {_id: 0}}, - {$out: {to: targetColl.getName(), mode: "insertDocuments"}}, + {$merge: {into: targetColl.getName(), whenMatched: "fail", whenNotMatched: "insert"}}, ])); assert.eq(1, targetColl.find().itcount()); // - // Test that an $out aggregation succeeds even if the _id is stripped out and _id is part of a - // multi-field "uniqueKey". + // Test that a $merge aggregation succeeds even if the _id is stripped out and _id is included + // in the "on" fields. // coll.drop(); assert.commandWorked(coll.insert([{_id: "should be projected away", name: "kyle"}])); @@ -87,22 +88,32 @@ assert.commandWorked(targetColl.createIndex({_id: 1, name: -1}, {unique: true})); assert.doesNotThrow(() => coll.aggregate([ {$project: {_id: 0}}, - {$out: {to: targetColl.getName(), mode: "insertDocuments", uniqueKey: {_id: 1, name: 1}}}, + { + $merge: { + into: targetColl.getName(), + whenMatched: "fail", + whenNotMatched: "insert", + on: ["_id", "name"] + } + }, ])); assert.eq(1, targetColl.find().itcount()); // - // Tests for $out to a database that differs from the aggregation database. + // Tests for $merge to a database that differs from the aggregation database. // - const foreignDb = db.getSiblingDB("mode_insert_documents_foreign"); - const foreignTargetColl = foreignDb.mode_insert_documents_out; + const foreignDb = db.getSiblingDB("merge_insert_only_foreign"); + const foreignTargetColl = foreignDb.merge_insert_only_out; const pipelineDifferentOutputDb = [ {$project: {_id: 0}}, { - $out: { - to: foreignTargetColl.getName(), - db: foreignDb.getName(), - mode: "insertDocuments", + $merge: { + into: { + db: foreignDb.getName(), + coll: foreignTargetColl.getName(), + }, + whenMatched: "fail", + whenNotMatched: "insert", } } ]; @@ -113,7 +124,7 @@ if (!FixtureHelpers.isMongos(db)) { // - // Test that $out implicitly creates a new database when the output collection's database + // Test that $merge implicitly creates a new database when the output collection's database // doesn't exist. // coll.aggregate(pipelineDifferentOutputDb); @@ -128,7 +139,7 @@ } // - // Re-run the $out aggregation, which should merge with the existing contents of the + // Re-run the $merge aggregation, which should merge with the existing contents of the // collection. We rely on implicit _id generation to give us unique _id values. // assert.doesNotThrow(() => coll.aggregate(pipelineDifferentOutputDb)); diff --git a/jstests/aggregation/sources/out/mode_replace_documents.js b/jstests/aggregation/sources/merge/mode_replace_insert.js index 269d3ca2109..588500702e2 100644 --- a/jstests/aggregation/sources/out/mode_replace_documents.js +++ b/jstests/aggregation/sources/merge/mode_replace_insert.js @@ -1,4 +1,4 @@ -// Tests for the $out stage with mode set to "replaceDocuments". +// Tests for the $merge stage with whenMatched: "replaceWithNew" and whenNotMatched: "insert". // @tags: [assumes_unsharded_collection] (function() { "use strict"; @@ -6,8 +6,8 @@ load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos. - const coll = db.replace_docs; - const outColl = db.replace_docs_out; + const coll = db.merge_replace_insert; + const outColl = db.merge_replace_insert_out; coll.drop(); outColl.drop(); @@ -16,62 +16,82 @@ assert.commandWorked(coll.insert({_id: i, a: i})); } - // Test that a $out with 'replaceDocuments' mode will default the unique key to "_id". - coll.aggregate([{$out: {to: outColl.getName(), mode: "replaceDocuments"}}]); + // Test that a $merge with whenMatched: "replaceWithNew" and whenNotMatched: "insert" mode will + // default the "on" fields to "_id". + coll.aggregate([{ + $merge: + {into: outColl.getName(), whenMatched: "replaceWithNew", whenNotMatched: "insert"} + }]); assert.eq(nDocs, outColl.find().itcount()); - // Test that 'replaceDocuments' mode will update existing documents that match the unique key. + // Test that $merge will update existing documents that match the "on" fields. const nDocsReplaced = 5; coll.aggregate([ {$project: {_id: {$mod: ["$_id", nDocsReplaced]}}}, - {$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {_id: 1}}} + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "_id" + } + } ]); assert.eq(nDocsReplaced, outColl.find({a: {$exists: false}}).itcount()); - // Test 'replaceDocuments' mode with a dotted path unique key. + // Test $merge with a dotted path "on" fields. coll.drop(); outColl.drop(); assert.commandWorked(coll.insert([{_id: 0, a: {b: 1}}, {_id: 1, a: {b: 1}, c: 1}])); assert.commandWorked(outColl.createIndex({"a.b": 1, _id: 1}, {unique: true})); coll.aggregate([ {$addFields: {_id: 0}}, - {$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {_id: 1, "a.b": 1}}} + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["_id", "a.b"] + } + } ]); assert.eq([{_id: 0, a: {b: 1}, c: 1}], outColl.find().toArray()); - // Test that 'replaceDocuments' mode will automatically generate a missing "_id" uniqueKey. + // Test that $merge will automatically generate a missing "_id" for the "on" field. coll.drop(); outColl.drop(); assert.commandWorked(coll.insert({field: "will be removed"})); assert.doesNotThrow(() => coll.aggregate([ {$replaceRoot: {newRoot: {}}}, { - $out: { - to: outColl.getName(), - mode: "replaceDocuments", + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", } } ])); assert.eq(1, outColl.find({field: {$exists: false}}).itcount()); - // Test that 'replaceDocuments' mode will automatically generate a missing "_id", and the - // aggregation succeeds with a multi-field uniqueKey. + // Test that $merge will automatically generate a missing "_id", and the aggregation succeeds + // with multiple "on" fields. outColl.drop(); assert.commandWorked(outColl.createIndex({name: -1, _id: 1}, {unique: true, sparse: true})); assert.doesNotThrow(() => coll.aggregate([ {$replaceRoot: {newRoot: {name: "jungsoo"}}}, { - $out: { - to: outColl.getName(), - mode: "replaceDocuments", - uniqueKey: {_id: 1, name: 1}, + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["_id", "name"] } } ])); assert.eq(1, outColl.find().itcount()); // Test that we will not attempt to modify the _id of an existing document if the _id is - // projected away but the uniqueKey does not involve _id. + // projected away but the "on" field does not involve _id. coll.drop(); assert.commandWorked(coll.insert({name: "kyle"})); assert.commandWorked(coll.insert({name: "nick"})); @@ -81,7 +101,14 @@ assert.doesNotThrow(() => coll.aggregate([ {$project: {_id: 0}}, {$addFields: {newField: 1}}, - {$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {name: 1}}} + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "name" + } + } ])); const outResult = outColl.find().sort({name: 1}).toArray(); const errmsgFn = () => tojson(outResult); @@ -91,58 +118,88 @@ assert.eq(1, outResult[1].newField, errmsgFn); assert.neq(null, outResult[1]._id, errmsgFn); - // Test that 'replaceDocuments' mode with a missing non-id unique key fails. + // Test that $merge with a missing non-id "on" field fails. outColl.drop(); assert.commandWorked(outColl.createIndex({missing: 1}, {unique: true})); assertErrorCode( coll, - [{$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {missing: 1}}}], + [{ + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "missing" + } + }], 51132 // This attempt should fail because there's no field 'missing' in the document. ); // Test that a replace fails to insert a document if it violates a unique index constraint. In - // this example, $out will attempt to insert multiple documents with {a: 0} which is not allowed - // with the unique index on {a: 1}. + // this example, $merge will attempt to insert multiple documents with {a: 0} which is not + // allowed with the unique index on {a: 1}. coll.drop(); assert.commandWorked(coll.insert([{_id: 0}, {_id: 1}])); outColl.drop(); assert.commandWorked(outColl.createIndex({a: 1}, {unique: true})); - assertErrorCode( - coll, - [{$addFields: {a: 0}}, {$out: {to: outColl.getName(), mode: "replaceDocuments"}}], - ErrorCodes.DuplicateKey); - - // Test that $out fails if the unique key contains an array. + assertErrorCode(coll, + [ + {$addFields: {a: 0}}, + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert" + } + } + ], + ErrorCodes.DuplicateKey); + + // Test that $merge fails if the "on" fields contains an array. coll.drop(); assert.commandWorked(coll.insert({_id: 0, a: [1, 2]})); assert.commandWorked(outColl.createIndex({"a.b": 1, _id: 1}, {unique: true})); - assertErrorCode( - coll, - [ - {$addFields: {_id: 0}}, - {$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {_id: 1, "a.b": 1}}} - ], - 51132); + assertErrorCode(coll, + [ + {$addFields: {_id: 0}}, + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["_id", "a.b"] + } + } + ], + 51132); coll.drop(); assert.commandWorked(coll.insert({_id: 0, a: [{b: 1}]})); - assertErrorCode( - coll, - [ - {$addFields: {_id: 0}}, - {$out: {to: outColl.getName(), mode: "replaceDocuments", uniqueKey: {_id: 1, "a.b": 1}}} - ], - 51132); - - // Tests for $out to a database that differs from the aggregation database. - const foreignDb = db.getSiblingDB("mode_replace_documents_foreign"); - const foreignTargetColl = foreignDb.mode_replace_documents_out; + assertErrorCode(coll, + [ + {$addFields: {_id: 0}}, + { + $merge: { + into: outColl.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["_id", "a.b"] + } + } + ], + 51132); + + // Tests for $merge to a database that differs from the aggregation database. + const foreignDb = db.getSiblingDB("merge_replace_insert_foreign"); + const foreignTargetColl = foreignDb.out; const pipelineDifferentOutputDb = [{ - $out: { - to: foreignTargetColl.getName(), - db: foreignDb.getName(), - mode: "replaceDocuments", + $merge: { + into: { + db: foreignDb.getName(), + coll: foreignTargetColl.getName(), + }, + whenMatched: "replaceWithNew", + whenNotMatched: "insert", } }]; @@ -151,7 +208,7 @@ foreignDb.dropDatabase(); if (!FixtureHelpers.isMongos(db)) { - // Test that $out implicitly creates a new database when the output collection's database + // Test that $merge implicitly creates a new database when the output collection's database // doesn't exist. coll.aggregate(pipelineDifferentOutputDb); assert.eq(foreignTargetColl.find().itcount(), 1); diff --git a/jstests/aggregation/sources/out/unique_key_validation.js b/jstests/aggregation/sources/merge/on_fields_validation.js index 73d3a49c29b..15de5f2eb41 100644 --- a/jstests/aggregation/sources/out/unique_key_validation.js +++ b/jstests/aggregation/sources/merge/on_fields_validation.js @@ -1,7 +1,6 @@ /** - * Tests for the validation of the "uniqueKey" at parse-time of the "uniqueKey" specification - * itself, as well as during runtime extraction of the "uniqueKey" from documents in the aggregation - * pipeline. + * Tests for the validation of the "on" fields at parse-time of $merge stage itself, as well as + * during runtime extraction of the "on" fields from documents in the aggregation pipeline. * * This test creates unique indexes on various combinations of fields, so it cannot be run in suites * that implicitly shard the collection with a hashed shard key. @@ -19,51 +18,47 @@ assert.commandWorked(source.insert({_id: 0})); // - // Tests for invalid "uniqueKey" specifications. + // Tests for invalid "on" fields specifications. // - function assertUniqueKeyIsInvalid(uniqueKey, expectedErrorCode) { + function assertOnFieldsIsInvalid(onFields, expectedErrorCode) { const stage = { - $out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: uniqueKey} + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: onFields + } }; assertErrorCode(source, stage, expectedErrorCode); } - // A non-object "uniqueKey" is prohibited. - assertUniqueKeyIsInvalid(3.14, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid("_id", ErrorCodes.TypeMismatch); - - // Explicitly specifying an empty-object "uniqueKey" is invalid. - assertUniqueKeyIsInvalid({}, ErrorCodes.InvalidOptions); - - // The "uniqueKey" won't be accepted if any field is not a number. - assertUniqueKeyIsInvalid({name: "hashed"}, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid({x: 1, y: 1, z: [1]}, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid({nested: {field: 1}}, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid({uniqueKey: true}, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid({string: "true"}, ErrorCodes.TypeMismatch); - assertUniqueKeyIsInvalid({bool: false}, ErrorCodes.TypeMismatch); - - // A numerical "uniqueKey" won't be accepted if any field isn't exactly the value 1. - assertUniqueKeyIsInvalid({_id: -1}, ErrorCodes.BadValue); - assertUniqueKeyIsInvalid({x: 10}, ErrorCodes.BadValue); - - // Test that the value 1 represented as different numerical types will be accepred. - [1.0, NumberInt(1), NumberLong(1), NumberDecimal(1)].forEach(one => { - assert.commandWorked(target.remove({})); - assert.doesNotThrow( - () => source.aggregate( - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {_id: one}}})); - assert.eq(target.find().toArray(), [{_id: 0}]); - }); + // A non-array or string "on" fields is prohibited. + assertOnFieldsIsInvalid(3.14, 51186); + assertOnFieldsIsInvalid({_id: 1}, 51186); + + // Explicitly specifying an empty-array "on" fields is invalid. + assertOnFieldsIsInvalid([], 51187); + + // The "on" fields array won't be accepted if any element is not a string. + assertOnFieldsIsInvalid(["hashed", 1], 51134); + assertOnFieldsIsInvalid([["_id"]], 51134); + assertOnFieldsIsInvalid([null], 51134); + assertOnFieldsIsInvalid([true, "a"], 51134); // - // An error is raised if $out encounters a document that is missing one or more of the - // "uniqueKey" fields. + // An error is raised if $merge encounters a document that is missing one or more of the + // "on" fields. // assert.commandWorked(target.remove({})); assert.commandWorked(target.createIndex({name: 1, team: -1}, {unique: true})); - const pipelineNameTeam = - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {name: 1, team: 1}}}]; + const pipelineNameTeam = [{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["name", "team"] + } + }]; // Missing both "name" and "team". assertErrorCode(source, pipelineNameTeam, 51132); @@ -82,19 +77,25 @@ assert.eq(target.find().toArray(), [{_id: 0, name: "nicholas", team: "query"}]); // - // An error is raised if $out encounters a document where one of the "uniqueKey" fields is a - // nullish value. + // An error is raised if $merge encounters a document where one of the "on" fields is a nullish + // value. // assert.commandWorked(target.remove({})); assert.commandWorked(target.createIndex({"song.artist": 1}, {unique: 1})); - const pipelineSongDotArtist = - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {"song.artist": 1}}}]; - - // Explicit null "song" (a prefix of a "uniqueKey" field). + const pipelineSongDotArtist = [{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["song.artist"] + } + }]; + + // Explicit null "song" (a prefix of an "on" field). assert.commandWorked(source.update({_id: 0}, {_id: 0, song: null})); assertErrorCode(source, pipelineSongDotArtist, 51132); - // Explicit undefined "song" (a prefix of a "uniqueKey" field). + // Explicit undefined "song" (a prefix of an "on" field). assert.commandWorked(source.update({_id: 0}, {_id: 0, song: undefined})); assertErrorCode(source, pipelineSongDotArtist, 51132); @@ -112,21 +113,26 @@ assert.eq(target.find().toArray(), [{_id: 0, song: {artist: "Illenium"}}]); // - // An error is raised if $out encounters a document where one of the "uniqueKey" fields (or a - // prefix of a "uniqueKey" field) is an array. + // An error is raised if $merge encounters a document where one of the "on" fields (or a prefix + // of an "on" field) is an array. // assert.commandWorked(target.remove({})); assert.commandWorked(target.createIndex({"address.street": 1}, {unique: 1})); - const pipelineAddressDotStreet = [ - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {"address.street": 1}}} - ]; + const pipelineAddressDotStreet = [{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["address.street"] + } + }]; // "address.street" is an array. assert.commandWorked( source.update({_id: 0}, {_id: 0, address: {street: ["West 43rd St", "1633 Broadway"]}})); assertErrorCode(source, pipelineAddressDotStreet, 51185); - // "address" is an array (a prefix of a "uniqueKey" field). + // "address" is an array (a prefix of an "on" field). assert.commandWorked(source.update({_id: 0}, {_id: 0, address: [{street: "1633 Broadway"}]})); assertErrorCode(source, pipelineAddressDotStreet, 51132); diff --git a/jstests/aggregation/sources/merge/requires_unique_index.js b/jstests/aggregation/sources/merge/requires_unique_index.js new file mode 100644 index 00000000000..8e0361baa79 --- /dev/null +++ b/jstests/aggregation/sources/merge/requires_unique_index.js @@ -0,0 +1,403 @@ +// Tests that the $merge stage enforces that the "on" fields can be used to uniquely identify +// documents by checking that there is a supporting unique, non-partial, collator-compatible index +// in the index catalog. +// +// Note that this test does *not* use the drop shell helper but instead runs the drop command +// manually. This is to avoid implicit creation and sharding of the $merge target collections in the +// passthrough suites. +(function() { + "use strict"; + + load("jstests/aggregation/extras/out_helpers.js"); // For withEachMergeMode, + // assertFailsWithoutUniqueIndex. + + const testDB = db.getSiblingDB("merge_requires_unique_index"); + assert.commandWorked(testDB.dropDatabase()); + + const source = testDB.source; + assert.commandWorked(source.insert([{_id: 0, a: 0}, {_id: 1, a: 1}])); + + // Helper to drop a collection without using the shell helper, and thus avoiding the implicit + // recreation in the passthrough suites. + function dropWithoutImplicitRecreate(coll) { + testDB.runCommand({drop: coll.getName()}); + } + + // Test that using {_id: 1} or not providing a unique key does not require any special indexes. + (function simpleIdOnFieldsOrDefaultShouldNotRequireIndexes() { + function assertDefaultOnFieldsSucceeds({setupCallback, collName}) { + withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { + // Skip the combination of merge modes which will fail depending on the contents of + // the source and target collection, as this will cause the assertion below to trip. + if (whenMatchedMode == "fail" || whenNotMatchedMode == "fail") + return; + + setupCallback(); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: collName, + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }])); + setupCallback(); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: collName, + on: "_id", + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + }])); + }); + } + + // Test that using "_id" or not specifying "on" fields works for a collection which does + // not exist. + const non_existent = testDB.non_existent; + assertDefaultOnFieldsSucceeds({ + setupCallback: () => dropWithoutImplicitRecreate(non_existent), + collName: non_existent.getName() + }); + + const unindexed = testDB.unindexed; + assertDefaultOnFieldsSucceeds({ + setupCallback: () => { + dropWithoutImplicitRecreate(unindexed); + assert.commandWorked(testDB.runCommand({create: unindexed.getName()})); + }, + collName: unindexed.getName() + }); + }()); + + // Test that a unique index on the "on" fields can be used to satisfy the requirement. + (function basicUniqueIndexWorks() { + const target = testDB.regular_unique; + dropWithoutImplicitRecreate(target); + assertFailsWithoutUniqueIndex({source: source, onFields: ["_id", "a"], target: target}); + + assert.commandWorked(testDB.runCommand({create: target.getName()})); + assert.commandWorked(target.createIndex({a: 1, _id: 1}, {unique: true})); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["_id", "a"] + } + }])); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["a", "_id"] + } + }])); + + assertFailsWithoutUniqueIndex( + {source: source, onFields: ["_id", "a", "b"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a", "b"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["b"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a"], target: target}); + + assert.commandWorked(target.dropIndex({a: 1, _id: 1})); + assert.commandWorked(target.createIndex({a: 1}, {unique: true})); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }])); + + // Create a non-unique index and make sure that doesn't work. + assert.commandWorked(target.dropIndex({a: 1})); + assert.commandWorked(target.createIndex({a: 1})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["_id", "a"], target: target}); + }()); + + // Test that a unique index on the "on" fields cannot be used to satisfy the requirement if it + // is a partial index. + (function uniqueButPartialShouldNotWork() { + const target = testDB.unique_but_partial_indexes; + dropWithoutImplicitRecreate(target); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + + assert.commandWorked( + target.createIndex({a: 1}, {unique: true, partialFilterExpression: {a: {$gte: 2}}})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["_id", "a"], target: target}); + }()); + + // Test that a unique index on the "on" fields cannot be used to satisfy the requirement if it + // has a different collation. + (function indexMustMatchCollationOfOperation() { + const target = testDB.collation_indexes; + dropWithoutImplicitRecreate(target); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + + assert.commandWorked( + target.createIndex({a: 1}, {unique: true, collation: {locale: "en_US"}})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assertFailsWithoutUniqueIndex( + {source: source, onFields: "a", target: target, options: {collation: {locale: "en"}}}); + assertFailsWithoutUniqueIndex({ + source: source, + onFields: "a", + target: target, + options: {collation: {locale: "simple"}} + }); + assertFailsWithoutUniqueIndex({ + source: source, + onFields: "a", + target: target, + options: {collation: {locale: "en_US", strength: 1}} + }); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }], + {collation: {locale: "en_US"}})); + + // Test that a non-unique index with the same collation cannot be used. + assert.commandWorked(target.dropIndex({a: 1})); + assert.commandWorked(target.createIndex({a: 1}, {collation: {locale: "en_US"}})); + assertFailsWithoutUniqueIndex({ + source: source, + onFields: "a", + target: target, + options: {collation: {locale: "en_US"}} + }); + + // Test that a collection-default collation will be applied to the index, but not the + // $merge's update or insert into that collection. The pipeline will inherit a + // collection-default collation, but from the source collection, not the $merge's target + // collection. + dropWithoutImplicitRecreate(target); + assert.commandWorked( + testDB.runCommand({create: target.getName(), collation: {locale: "en_US"}})); + assert.commandWorked(target.createIndex({a: 1}, {unique: true})); + assertFailsWithoutUniqueIndex({ + source: source, + onFields: "a", + target: target, + }); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }], + {collation: {locale: "en_US"}})); + + // Test that when the source collection and foreign collection have the same default + // collation, a unique index on the foreign collection can be used. + const newSourceColl = testDB.new_source; + dropWithoutImplicitRecreate(newSourceColl); + assert.commandWorked( + testDB.runCommand({create: newSourceColl.getName(), collation: {locale: "en_US"}})); + assert.commandWorked(newSourceColl.insert([{_id: 1, a: 1}, {_id: 2, a: 2}])); + // This aggregate does not specify a collation, but it should inherit the default collation + // from 'newSourceColl', and therefore the index on 'target' should be eligible for use + // since it has the same collation. + assert.doesNotThrow(() => newSourceColl.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }])); + + // Test that an explicit "simple" collation can be used with an index without a collation. + dropWithoutImplicitRecreate(target); + assert.commandWorked(target.createIndex({a: 1}, {unique: true})); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }], + {collation: {locale: "simple"}})); + assertFailsWithoutUniqueIndex({ + source: source, + onFields: "a", + target: target, + options: {collation: {locale: "en_US"}} + }); + }()); + + // Test that a unique index which is not simply ascending/descending fields cannot be used for + // the "on" fields. + (function testSpecialIndexTypes() { + const target = testDB.special_index_types; + dropWithoutImplicitRecreate(target); + + assert.commandWorked(target.createIndex({a: 1, text: "text"}, {unique: true})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a", "text"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: "text", target: target}); + + dropWithoutImplicitRecreate(target); + assert.commandWorked(target.createIndex({a: 1, geo: "2dsphere"}, {unique: true})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a", "geo"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["geo", "a"], target: target}); + + dropWithoutImplicitRecreate(target); + assert.commandWorked(target.createIndex({geo: "2d"}, {unique: true})); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a", "geo"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: "geo", target: target}); + + dropWithoutImplicitRecreate(target); + assert.commandWorked( + target.createIndex({geo: "geoHaystack", a: 1}, {unique: true, bucketSize: 5})); + assertFailsWithoutUniqueIndex({source: source, onFields: ["a", "geo"], target: target}); + assertFailsWithoutUniqueIndex({source: source, onFields: ["geo", "a"], target: target}); + + dropWithoutImplicitRecreate(target); + // MongoDB does not support unique hashed indexes. + assert.commandFailedWithCode(target.createIndex({a: "hashed"}, {unique: true}), 16764); + assert.commandWorked(target.createIndex({a: "hashed"})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + }()); + + // Test that a unique index with dotted field names can be used. + (function testDottedFieldNames() { + const target = testDB.dotted_field_paths; + dropWithoutImplicitRecreate(target); + + assert.commandWorked(target.createIndex({a: 1, "b.c.d": -1}, {unique: true})); + assertFailsWithoutUniqueIndex({source: source, onFields: "a", target: target}); + assert.doesNotThrow(() => source.aggregate([ + {$project: {_id: 1, a: 1, b: {c: {d: "x"}}}}, + { + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["a", "b.c.d"] + } + } + ])); + + dropWithoutImplicitRecreate(target); + assert.commandWorked(target.createIndex({"id.x": 1, "id.y": -1}, {unique: true})); + assert.doesNotThrow(() => source.aggregate([ + {$group: {_id: {x: "$_id", y: "$a"}}}, + {$project: {id: "$_id"}}, + { + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["id.x", "id.y"] + } + } + ])); + assert.doesNotThrow(() => source.aggregate([ + {$group: {_id: {x: "$_id", y: "$a"}}}, + {$project: {id: "$_id"}}, + { + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: ["id.y", "id.x"] + } + } + ])); + + // Test that we cannot use arrays with a dotted path within a $merge. + dropWithoutImplicitRecreate(target); + assert.commandWorked(target.createIndex({"b.c": 1}, {unique: true})); + withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { + assert.commandFailedWithCode(testDB.runCommand({ + aggregate: source.getName(), + pipeline: [ + {$replaceRoot: {newRoot: {b: [{c: 1}, {c: 2}]}}}, + { + $merge: { + into: target.getName(), + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode, + on: "b.c" + } + } + ], + cursor: {} + }), + [50905, 51132]); + }); + }()); + + // Test that a unique index that is multikey can still be used. + (function testMultikeyIndex() { + const target = testDB.multikey_index; + dropWithoutImplicitRecreate(target); + + assert.commandWorked(target.createIndex({"a.b": 1}, {unique: true})); + assert.doesNotThrow(() => source.aggregate([ + {$project: {_id: 1, "a.b": "$a"}}, + { + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a.b" + } + } + ])); + assert.commandWorked(target.insert({_id: "TARGET", a: [{b: "hi"}, {b: "hello"}]})); + assert.commandWorked(source.insert({a: "hi", proofOfUpdate: "PROOF"})); + assert.doesNotThrow(() => source.aggregate([ + {$project: {_id: 0, proofOfUpdate: "PROOF", "a.b": "$a"}}, + { + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a.b" + } + } + ])); + assert.docEq(target.findOne({"a.b": "hi", proofOfUpdate: "PROOF"}), + {_id: "TARGET", a: {b: "hi"}, proofOfUpdate: "PROOF"}); + }()); + + // Test that a unique index that is sparse can still be used. + (function testSparseIndex() { + const target = testDB.multikey_index; + dropWithoutImplicitRecreate(target); + + assert.commandWorked(target.createIndex({a: 1}, {unique: true, sparse: true})); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }])); + assert.commandWorked(target.insert([{b: 1, c: 1}, {a: null}, {d: 4}])); + assert.doesNotThrow(() => source.aggregate([{ + $merge: { + into: target.getName(), + whenMatched: "replaceWithNew", + whenNotMatched: "insert", + on: "a" + } + }])); + }()); +}()); diff --git a/jstests/aggregation/sources/out/use_cases.js b/jstests/aggregation/sources/merge/use_cases.js index 243f1f78ff4..910767cb8eb 100644 --- a/jstests/aggregation/sources/out/use_cases.js +++ b/jstests/aggregation/sources/merge/use_cases.js @@ -1,5 +1,5 @@ /** - * Tests a practical use case for $out from a collection of samples to an hourly rollup output + * Tests a practical use case for $merge from a collection of samples to an hourly rollup output * collection. * * @tags: [requires_sharding] @@ -40,9 +40,9 @@ return [ticksSum, tempSum]; } - // Runs a $out aggregate on the metrics collection to the rollup collection, grouping by hour, + // Runs a $merge aggregate on the metrics collection to the rollup collection, grouping by hour, // summing the ticks, and averaging the temps. - function runAggregate(startDate, mode) { + function runAggregate({startDate, whenMatchedMode, whenNotMatchedMode}) { metricsColl.aggregate([ {$match: {_id: {$gte: startDate}}}, { @@ -52,7 +52,13 @@ avgTemp: {$avg: "$temp"}, } }, - {$out: {to: rollupColl.getName(), db: rollupColl.getDB().getName(), mode: mode}} + { + $merge: { + into: {db: rollupColl.getDB().getName(), coll: rollupColl.getName()}, + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode + } + } ]); } @@ -65,29 +71,30 @@ const samplesPerHour = 10; let [ticksSum, tempSum] = insertRandomData(metricsColl, hourZero, samplesPerHour); - runAggregate(hourZero, "insertDocuments"); + runAggregate({startDate: hourZero, whenMatchedMode: "fail", whenNotMatchedMode: "insert"}); - // Verify the results of the $out in the rollup collection. + // Verify the results of the $merge in the rollup collection. let res = rollupColl.find().sort({_id: 1}); assert.eq([{_id: "2018-08-15T00", ticks: ticksSum, avgTemp: tempSum / samplesPerHour}], res.toArray()); - // Insert another hour's worth of data, and verify that the $out will append the result to the + // Insert another hour's worth of data, and verify that the $merge will append the result to the // output collection. [ticksSum, tempSum] = insertRandomData(metricsColl, hourOne, samplesPerHour); - runAggregate(hourOne, "insertDocuments"); + runAggregate({startDate: hourOne, whenMatchedMode: "fail", whenNotMatchedMode: "insert"}); res = rollupColl.find().sort({_id: 1}).toArray(); assert.eq(2, res.length); assert.eq(res[1], {_id: "2018-08-15T01", ticks: ticksSum, avgTemp: tempSum / samplesPerHour}); // Whoops, there was a mistake in the last hour of data. Let's re-run the aggregation and update - // the rollup collection using the "replaceDocuments" mode. + // the rollup collection using the "replaceWithNew". assert.commandWorked(metricsColl.update({_id: hourOne}, {$inc: {ticks: 10}})); ticksSum += 10; - runAggregate(hourOne, "replaceDocuments"); + runAggregate( + {startDate: hourOne, whenMatchedMode: "replaceWithNew", whenNotMatchedMode: "insert"}); res = rollupColl.find().sort({_id: 1}).toArray(); assert.eq(2, res.length); @@ -100,7 +107,7 @@ // Insert hour 7 data into the metrics collection and re-run the aggregation. [ticksSum, tempSum] = insertRandomData(metricsColl, hourSix, samplesPerHour); - runAggregate(hourSix, "insertDocuments"); + runAggregate({startDate: hourSix, whenMatchedMode: "fail", whenNotMatchedMode: "insert"}); res = rollupColl.find().sort({_id: 1}).toArray(); assert.eq(3, res.length, tojson(res)); diff --git a/jstests/aggregation/sources/out/batch_writes.js b/jstests/aggregation/sources/out/batch_writes.js deleted file mode 100644 index de6320ac086..00000000000 --- a/jstests/aggregation/sources/out/batch_writes.js +++ /dev/null @@ -1,59 +0,0 @@ -// Tests the behavior of an $out stage which encounters an error in the middle of processing. We -// don't guarantee any particular behavior in this scenario, but this test exists to make sure -// nothing horrendous happens and to characterize the current behavior. -// @tags: [assumes_unsharded_collection] -(function() { - "use strict"; - - load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. - - const coll = db.batch_writes; - const outColl = db.batch_writes_out; - coll.drop(); - outColl.drop(); - - // Test with 2 very large documents that do not fit into a single batch. - const kSize15MB = 15 * 1024 * 1024; - const largeArray = new Array(kSize15MB).join("a"); - assert.commandWorked(coll.insert({_id: 0, a: largeArray})); - assert.commandWorked(coll.insert({_id: 1, a: largeArray})); - - // Make sure the $out succeeds without any duplicate keys. - ["replaceCollection", "insertDocuments", "replaceDocuments"].forEach(mode => { - coll.aggregate([{$out: {to: outColl.getName(), mode: mode}}]); - assert.eq(2, outColl.find().itcount()); - outColl.drop(); - }); - - coll.drop(); - for (let i = 0; i < 10; i++) { - assert.commandWorked(coll.insert({_id: i, a: i})); - } - - // Create a unique index on 'a' in the output collection to create a unique key violation when - // running the $out. The second document to be written ({_id: 1, a: 1}) will conflict with the - // existing document in the output collection. We use a unique index on a field other than _id - // because "replaceDocuments" mode will not change _id when one already exists. - outColl.drop(); - assert.commandWorked(outColl.insert({_id: 2, a: 1})); - assert.commandWorked(outColl.createIndex({a: 1}, {unique: true})); - - // Test that the writes for $out are unordered, meaning the operation continues even if it - // encounters a duplicate key error. We don't guarantee any particular behavior in this case, - // but this test is meant to characterize the current behavior. - ["insertDocuments", "replaceDocuments"].forEach(mode => { - assertErrorCode( - coll, [{$out: {to: outColl.getName(), mode: mode}}], ErrorCodes.DuplicateKey); - assert.soon(() => { - return outColl.find().itcount() == 9; - }); - }); - - // Mode "replaceCollection" will drop the contents of the output collection, so there is no - // duplicate key error. - outColl.drop(); - assert.commandWorked(outColl.insert({_id: 2, a: 1})); - assert.commandWorked(outColl.createIndex({a: 1}, {unique: true})); - coll.aggregate([{$out: {to: outColl.getName(), mode: "replaceCollection"}}]); - assert.eq(10, outColl.find().itcount()); -}()); diff --git a/jstests/aggregation/sources/out/out_in_lookup_not_allowed.js b/jstests/aggregation/sources/out/out_in_lookup_not_allowed.js index c1a2d286c77..d81eaaaab83 100644 --- a/jstests/aggregation/sources/out/out_in_lookup_not_allowed.js +++ b/jstests/aggregation/sources/out/out_in_lookup_not_allowed.js @@ -5,7 +5,7 @@ load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. load("jstests/libs/collection_drop_recreate.js"); // For assertDropCollection. load("jstests/noPassthrough/libs/server_parameter_helpers.js"); // For setParameterOnAllHosts. - load("jstests/libs/discover_topology.js"); // For findDataBearingNodes. + load("jstests/libs/discover_topology.js"); // For findNonConfigNodes. load("jstests/libs/fixture_helpers.js"); // For isSharded. const ERROR_CODE_OUT_BANNED_IN_LOOKUP = 51047; diff --git a/jstests/aggregation/sources/out/out_to_referenced_collection.js b/jstests/aggregation/sources/out/out_to_referenced_collection.js deleted file mode 100644 index a85f90a9816..00000000000 --- a/jstests/aggregation/sources/out/out_to_referenced_collection.js +++ /dev/null @@ -1,132 +0,0 @@ -// Tests that the server behaves as expected when an $out stage is targeting a collection which is -// involved in the aggregate in some other way, e.g. as the source namespace or via a $lookup. We -// disallow this combination in an effort to prevent the "halloween problem" of a never-ending -// query. If the $out is using mode "replaceCollection" then this is legal because we use a -// temporary collection. If the out is using any other mode which would be "in place", we expect the -// server to error in an effort to prevent server-side infinite loops. -// This test issues queries over views, so cannot be run in passthroughs which implicitly shard -// collections. -// @tags: [assumes_unsharded_collection] -(function() { - 'use strict'; - - load('jstests/aggregation/extras/out_helpers.js'); // For 'withEachOutMode'. - load('jstests/libs/fixture_helpers.js'); // For 'FixtureHelpers'. - - const testDB = db.getSiblingDB("out_to_referenced_coll"); - const coll = testDB.test; - - withEachOutMode(mode => { - coll.drop(); - if (FixtureHelpers.isSharded(coll) && mode === "replaceCollection") { - return; // Not a supported combination. - } - - // Seed the collection to ensure each pipeline will actually do something. - assert.commandWorked(coll.insert({_id: 0})); - - // Each of the following assertions will somehow use $out to write to a namespace that is - // being read from elsewhere in the pipeline. This is legal with mode "replaceCollection". - const assertFailsWithCode = ((fn) => { - const error = assert.throws(fn); - assert.contains(error.code, [50992, 51079]); - }); - const asserter = (mode === "replaceCollection") ? assert.doesNotThrow : assertFailsWithCode; - - // Test $out to the aggregate command's source collection. - asserter(() => coll.aggregate([{$out: {to: coll.getName(), mode: mode}}])); - // Test $out to the same namespace as a $lookup which is the same as the aggregate command's - // source collection. - asserter(() => coll.aggregate([ - {$lookup: {from: coll.getName(), as: "x", localField: "f_id", foreignField: "_id"}}, - {$out: {to: coll.getName(), mode: mode}} - ])); - // Test $out to the same namespace as a $lookup which is *not* the same as the aggregate - // command's source collection. - asserter(() => coll.aggregate([ - {$lookup: {from: "bar", as: "x", localField: "f_id", foreignField: "_id"}}, - {$out: {to: "bar", mode: mode}} - ])); - // Test $out to the same namespace as a $graphLookup. - asserter(() => coll.aggregate([ - { - $graphLookup: { - from: "bar", - startWith: "$_id", - connectFromField: "_id", - connectToField: "parent_id", - as: "graph", - } - }, - {$out: {to: "bar", mode: mode}} - ])); - // Test $out to the same namespace as a $lookup which is nested within another $lookup. - asserter(() => coll.aggregate([ - { - $lookup: { - from: "bar", - as: "x", - let : {}, - pipeline: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}] - } - }, - {$out: {to: "TARGET", mode: mode}} - ])); - // Test $out to the same namespace as a $lookup which is nested within a $facet. - asserter(() => coll.aggregate([ - { - $facet: { - y: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}], - } - }, - {$out: {to: "TARGET", mode: mode}} - ])); - asserter(() => coll.aggregate([ - { - $facet: { - x: [{$lookup: {from: "other", as: "y", pipeline: []}}], - y: [{$lookup: {from: "TARGET", as: "y", pipeline: []}}], - } - }, - {$out: {to: "TARGET", mode: mode}} - ])); - - // Test that we use the resolved namespace of a view to detect this sort of halloween - // problem. - assert.commandWorked( - testDB.runCommand({create: "view_on_TARGET", viewOn: "TARGET", pipeline: []})); - asserter(() => testDB.view_on_TARGET.aggregate([{$out: {to: "TARGET", mode: mode}}])); - asserter(() => coll.aggregate([ - { - $facet: { - x: [{$lookup: {from: "other", as: "y", pipeline: []}}], - y: [{ - $lookup: { - from: "yet_another", - as: "y", - pipeline: [{$lookup: {from: "view_on_TARGET", as: "z", pipeline: []}}] - } - }], - } - }, - {$out: {to: "TARGET", mode: mode}} - ])); - - function generateNestedPipeline(foreignCollName, numLevels) { - let pipeline = [{"$lookup": {pipeline: [], from: foreignCollName, as: "same"}}]; - - for (let level = 1; level < numLevels; level++) { - pipeline = [{"$lookup": {pipeline: pipeline, from: foreignCollName, as: "same"}}]; - } - - return pipeline; - } - - const nestedPipeline = - generateNestedPipeline("lookup", 20).concat([{$out: {to: "lookup", mode: mode}}]); - asserter(() => coll.aggregate(nestedPipeline)); - - testDB.dropDatabase(); // Clear any of the collections which would be created by the - // successful "replaceCollection" mode of this test. - }); -}()); diff --git a/jstests/aggregation/sources/out/mode_replace_collection.js b/jstests/aggregation/sources/out/replace_collection.js index 68cd7a9c2b3..63204485a7c 100644 --- a/jstests/aggregation/sources/out/mode_replace_collection.js +++ b/jstests/aggregation/sources/out/replace_collection.js @@ -1,8 +1,8 @@ /** - * Tests the behavior of $out with mode "replaceCollection". + * Tests the behavior of legacy $out. * - * This test assumes that collections are not implicitly sharded, since mode "replaceCollection" is - * prohibited if the output collection is sharded. + * This test assumes that collections are not implicitly sharded, since $out is prohibited if the + * output collection is sharded. * @tags: [assumes_unsharded_collection] */ (function() { @@ -11,13 +11,13 @@ load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. load("jstests/libs/fixture_helpers.js"); // For FixtureHelpers.isMongos. - const coll = db.mode_replace_collection; + const coll = db.source; coll.drop(); - const targetColl = db.mode_replace_collection_out; + const targetColl = db.target; targetColl.drop(); - const pipeline = [{$out: {to: targetColl.getName(), mode: "replaceCollection"}}]; + const pipeline = [{$out: targetColl.getName()}]; // // Test $out with a non-existent output collection. @@ -71,52 +71,4 @@ coll.aggregate(pipeline); assert.eq(1, targetColl.find().itcount()); assert.eq(2, targetColl.getIndexes().length); - - // - // Test that an $out aggregation succeeds even if the _id is stripped out and the "uniqueKey" - // is the document key. - // - coll.drop(); - targetColl.drop(); - assert.commandWorked(coll.insert({val: "will be removed"})); - assert.doesNotThrow(() => coll.aggregate([ - {$replaceRoot: {newRoot: {name: "kyle"}}}, - {$out: {to: targetColl.getName(), mode: "replaceCollection"}} - ])); - assert.eq(1, targetColl.find({name: "kyle", val: {$exists: false}}).itcount()); - - // - // Test that an $out aggregation succeeds even if the _id is stripped out and _id is part of a - // multi-field "uniqueKey". - // - targetColl.drop(); - assert.commandWorked(targetColl.createIndex({name: -1, _id: -1}, {unique: true})); - assert.doesNotThrow(() => coll.aggregate([ - {$replaceRoot: {newRoot: {name: "jungsoo"}}}, - { - $out: { - to: targetColl.getName(), - mode: "replaceCollection", - uniqueKey: {_id: 1, name: 1} - } - } - ])); - assert.eq(1, targetColl.find({val: {$exists: false}}).itcount()); - - // - // Tests for $out to a database that differs from the aggregation database. - // - const foreignDb = db.getSiblingDB("mode_replace_collection_foreign"); - const foreignTargetColl = foreignDb.mode_replace_collection_out; - const pipelineDifferentOutputDb = [{ - $out: { - to: foreignTargetColl.getName(), - db: foreignDb.getName(), - mode: "replaceCollection", - } - }]; - - // TODO (SERVER-36832): Allow "replaceCollection" mode with a foreign output database. - assert.commandWorked(foreignTargetColl.insert({val: "forcing database creation"})); - assertErrorCode(coll, pipelineDifferentOutputDb, 50939); }()); diff --git a/jstests/aggregation/sources/out/unique_key_requires_index.js b/jstests/aggregation/sources/out/unique_key_requires_index.js deleted file mode 100644 index 6448fcd5074..00000000000 --- a/jstests/aggregation/sources/out/unique_key_requires_index.js +++ /dev/null @@ -1,323 +0,0 @@ -// Tests that the $out stage enforces that the uniqueKey argument can be used to uniquely identify -// documents by checking that there is a supporting unique, non-partial, collator-compatible index -// in the index catalog. -// -// Note that this test does *not* use the drop shell helper but instead runs the drop command -// manually. This is to avoid implicit creation and sharding of the $out target collections in the -// passthrough suites. -(function() { - "use strict"; - - load("jstests/aggregation/extras/out_helpers.js"); // For withEachOutMode, - // assertUniqueKeyIsInvalid. - - const testDB = db.getSiblingDB("unique_key_requires_index"); - assert.commandWorked(testDB.dropDatabase()); - - const source = testDB.source; - assert.commandWorked(source.insert([{_id: 0, a: 0}, {_id: 1, a: 1}])); - - // Helper to drop a collection without using the shell helper, and thus avoiding the implicit - // recreation in the passthrough suites. - function dropWithoutImplicitRecreate(coll) { - testDB.runCommand({drop: coll.getName()}); - } - - // Test that using {_id: 1} or not providing a unique key does not require any special indexes. - (function simpleIdUniqueKeyOrDefaultShouldNotRequireIndexes() { - function assertDefaultUniqueKeySuceeds({setupCallback, collName}) { - // Legacy style $out - "replaceCollection". - setupCallback(); - assert.doesNotThrow(() => source.aggregate([{$out: collName}])); - - withEachOutMode((mode) => { - setupCallback(); - assert.doesNotThrow(() => source.aggregate([{$out: {to: collName, mode: mode}}])); - setupCallback(); - assert.doesNotThrow(() => source.aggregate( - [{$out: {to: collName, uniqueKey: {_id: 1}, mode: mode}}])); - }); - } - - // Test that using {_id: 1} or not specifying a uniqueKey works for a collection which does - // not exist. - const non_existent = testDB.non_existent; - assertDefaultUniqueKeySuceeds({ - setupCallback: () => dropWithoutImplicitRecreate(non_existent), - collName: non_existent.getName() - }); - - const unindexed = testDB.unindexed; - assertDefaultUniqueKeySuceeds({ - setupCallback: () => { - dropWithoutImplicitRecreate(unindexed); - assert.commandWorked(testDB.runCommand({create: unindexed.getName()})); - }, - collName: unindexed.getName() - }); - }()); - - // Test that a unique index on the unique key can be used to satisfy the requirement. - (function basicUniqueIndexWorks() { - const target = testDB.regular_unique; - dropWithoutImplicitRecreate(target); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {_id: 1, a: 1}, target: target}); - - assert.commandWorked(testDB.runCommand({create: target.getName()})); - assert.commandWorked(target.createIndex({a: 1, _id: 1}, {unique: true})); - assert.doesNotThrow(() => source.aggregate([ - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {_id: 1, a: 1}}} - ])); - assert.doesNotThrow(() => source.aggregate([ - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1, _id: 1}}} - ])); - - assertUniqueKeyIsInvalid({source: source, uniqueKey: {_id: 1, a: 1, b: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, b: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {b: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - - assert.commandWorked(target.dropIndex({a: 1, _id: 1})); - assert.commandWorked(target.createIndex({a: 1}, {unique: true})); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}])); - - // Create a non-unique index and make sure that doesn't work. - assert.commandWorked(target.dropIndex({a: 1})); - assert.commandWorked(target.createIndex({a: 1})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, _id: 1}, target: target}); - }()); - - // Test that a unique index on the unique key cannot be used to satisfy the requirement if it is - // a partial index. - (function uniqueButPartialShouldNotWork() { - const target = testDB.unique_but_partial_indexes; - dropWithoutImplicitRecreate(target); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - - assert.commandWorked( - target.createIndex({a: 1}, {unique: true, partialFilterExpression: {a: {$gte: 2}}})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, _id: 1}, target: target}); - }()); - - // Test that a unique index on the unique key cannot be used to satisfy the requirement if it - // has a different collation. - (function indexMustMatchCollationOfOperation() { - const target = testDB.collation_indexes; - dropWithoutImplicitRecreate(target); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - - assert.commandWorked( - target.createIndex({a: 1}, {unique: true, collation: {locale: "en_US"}})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - options: {collation: {locale: "en"}} - }); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - options: {collation: {locale: "simple"}} - }); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - options: {collation: {locale: "en_US", strength: 1}} - }); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}], - {collation: {locale: "en_US"}})); - - // Test that a non-unique index with the same collation cannot be used. - assert.commandWorked(target.dropIndex({a: 1})); - assert.commandWorked(target.createIndex({a: 1}, {collation: {locale: "en_US"}})); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - options: {collation: {locale: "en_US"}} - }); - - // Test that a collection-default collation will be applied to the index, but not the $out's - // update or insert into that collection. The pipeline will inherit a collection-default - // collation, but from the source collection, not the $out's target collection. - dropWithoutImplicitRecreate(target); - assert.commandWorked( - testDB.runCommand({create: target.getName(), collation: {locale: "en_US"}})); - assert.commandWorked(target.createIndex({a: 1}, {unique: true})); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - }); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}], - {collation: {locale: "en_US"}})); - - // Test that when the source collection and foreign collection have the same default - // collation, a unique index on the foreign collection can be used. - const newSourceColl = testDB.new_source; - dropWithoutImplicitRecreate(newSourceColl); - assert.commandWorked( - testDB.runCommand({create: newSourceColl.getName(), collation: {locale: "en_US"}})); - assert.commandWorked(newSourceColl.insert([{_id: 1, a: 1}, {_id: 2, a: 2}])); - // This aggregate does not specify a collation, but it should inherit the default collation - // from 'newSourceColl', and therefore the index on 'target' should be eligible for use - // since it has the same collation. - assert.doesNotThrow( - () => newSourceColl.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}])); - - // Test that an explicit "simple" collation can be used with an index without a collation. - dropWithoutImplicitRecreate(target); - assert.commandWorked(target.createIndex({a: 1}, {unique: true})); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}], - {collation: {locale: "simple"}})); - assertUniqueKeyIsInvalid({ - source: source, - uniqueKey: {a: 1}, - target: target, - options: {collation: {locale: "en_US"}} - }); - }()); - - // Test that a unique index which is not simply ascending/descending fields cannot be used for - // the uniqueKey. - (function testSpecialIndexTypes() { - const target = testDB.special_index_types; - dropWithoutImplicitRecreate(target); - - assert.commandWorked(target.createIndex({a: 1, text: "text"}, {unique: true})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, text: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {text: 1}, target: target}); - - dropWithoutImplicitRecreate(target); - assert.commandWorked(target.createIndex({a: 1, geo: "2dsphere"}, {unique: true})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, geo: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {geo: 1, a: 1}, target: target}); - - dropWithoutImplicitRecreate(target); - assert.commandWorked(target.createIndex({geo: "2d"}, {unique: true})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, geo: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {geo: 1}, target: target}); - - dropWithoutImplicitRecreate(target); - assert.commandWorked( - target.createIndex({geo: "geoHaystack", a: 1}, {unique: true, bucketSize: 5})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1, geo: 1}, target: target}); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {geo: 1, a: 1}, target: target}); - - dropWithoutImplicitRecreate(target); - // MongoDB does not support unique hashed indexes. - assert.commandFailedWithCode(target.createIndex({a: "hashed"}, {unique: true}), 16764); - assert.commandWorked(target.createIndex({a: "hashed"})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - }()); - - // Test that a unique index with dotted field names can be used. - (function testDottedFieldNames() { - const target = testDB.dotted_field_paths; - dropWithoutImplicitRecreate(target); - - assert.commandWorked(target.createIndex({a: 1, "b.c.d": -1}, {unique: true})); - assertUniqueKeyIsInvalid({source: source, uniqueKey: {a: 1}, target: target}); - assert.doesNotThrow(() => source.aggregate([ - {$project: {_id: 1, a: 1, b: {c: {d: "x"}}}}, - { - $out: { - to: target.getName(), - mode: "replaceDocuments", - uniqueKey: {a: 1, "b.c.d": 1} - } - } - ])); - - dropWithoutImplicitRecreate(target); - assert.commandWorked(target.createIndex({"id.x": 1, "id.y": -1}, {unique: true})); - assert.doesNotThrow(() => source.aggregate([ - {$group: {_id: {x: "$_id", y: "$a"}}}, - {$project: {id: "$_id"}}, - { - $out: { - to: target.getName(), - mode: "replaceDocuments", - uniqueKey: {"id.x": 1, "id.y": 1} - } - } - ])); - assert.doesNotThrow(() => source.aggregate([ - {$group: {_id: {x: "$_id", y: "$a"}}}, - {$project: {id: "$_id"}}, - { - $out: { - to: target.getName(), - mode: "replaceDocuments", - uniqueKey: {"id.y": 1, "id.x": 1} - } - } - ])); - - // Test that we cannot use arrays with a dotted path within an $out. - dropWithoutImplicitRecreate(target); - assert.commandWorked(target.createIndex({"b.c": 1}, {unique: true})); - withEachOutMode((mode) => { - assert.commandFailedWithCode(testDB.runCommand({ - aggregate: source.getName(), - pipeline: [ - {$replaceRoot: {newRoot: {b: [{c: 1}, {c: 2}]}}}, - {$out: {to: target.getName(), mode: mode, uniqueKey: {"b.c": 1}}} - ], - cursor: {} - }), - [50905, 51132]); - }); - }()); - - // Test that a unique index that is multikey can still be used. - (function testMultikeyIndex() { - const target = testDB.multikey_index; - dropWithoutImplicitRecreate(target); - - assert.commandWorked(target.createIndex({"a.b": 1}, {unique: true})); - assert.doesNotThrow(() => source.aggregate([ - {$project: {_id: 1, "a.b": "$a"}}, - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {"a.b": 1}}} - ])); - assert.commandWorked(target.insert({_id: "TARGET", a: [{b: "hi"}, {b: "hello"}]})); - assert.commandWorked(source.insert({a: "hi", proofOfUpdate: "PROOF"})); - assert.doesNotThrow(() => source.aggregate([ - {$project: {_id: 0, proofOfUpdate: "PROOF", "a.b": "$a"}}, - {$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {"a.b": 1}}} - ])); - assert.docEq(target.findOne({"a.b": "hi", proofOfUpdate: "PROOF"}), - {_id: "TARGET", a: {b: "hi"}, proofOfUpdate: "PROOF"}); - }()); - - // Test that a unique index that is sparse can still be used. - (function testSparseIndex() { - const target = testDB.multikey_index; - dropWithoutImplicitRecreate(target); - - assert.commandWorked(target.createIndex({a: 1}, {unique: true, sparse: true})); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}])); - assert.commandWorked(target.insert([{b: 1, c: 1}, {a: null}, {d: 4}])); - assert.doesNotThrow( - () => source.aggregate( - [{$out: {to: target.getName(), mode: "replaceDocuments", uniqueKey: {a: 1}}}])); - }()); -}()); diff --git a/jstests/aggregation/sources/out/write_same_collection.js b/jstests/aggregation/sources/out/write_same_collection.js deleted file mode 100644 index ec3f7ba4d00..00000000000 --- a/jstests/aggregation/sources/out/write_same_collection.js +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Tests the behavior of $out when the target collection is the aggregation collection. Parsing - * should fail if the aggregation has $out mode set to "replaceDocuments" or "insertDocuments", - * but pass with the legacy syntax or when mode = "replaceCollection". - * - * @tags: [assumes_unsharded_collection] -*/ -(function() { - "use strict"; - - load("jstests/aggregation/extras/utils.js"); // For assertErrorCode. - - const coll = db.name; - coll.drop(); - - const nDocs = 10; - for (let i = 0; i < nDocs; i++) { - assert.commandWorked(coll.insert({_id: i, a: i})); - } - - assertErrorCode(coll, [{$out: {to: coll.getName(), mode: "replaceDocuments"}}], 50992); - - assertErrorCode(coll, [{$out: {to: coll.getName(), mode: "insertDocuments"}}], 50992); - - assert.commandWorked(db.runCommand({ - aggregate: coll.getName(), - pipeline: [{$out: {to: coll.getName(), mode: "replaceCollection"}}], - cursor: {}, - })); -}()); diff --git a/jstests/sharding/merge_to_non_existing.js b/jstests/sharding/merge_to_non_existing.js index 585428fcde6..5aad5e003ab 100644 --- a/jstests/sharding/merge_to_non_existing.js +++ b/jstests/sharding/merge_to_non_existing.js @@ -35,12 +35,35 @@ $merge: { into: {db: targetColl.getDB().getName(), coll: targetColl.getName()}, whenMatched: whenMatchedMode, - whenNotMatched: whenNotMatchedMode + whenNotMatched: whenNotMatchedMode, + on: "_id" } }]); assert.eq(whenNotMatchedMode == "discard" ? 0 : 10, targetColl.find().itcount()); }); + // Test that $merge fails if the "on" field is anything but "_id" when the target collection + // does not exist. + withEachMergeMode(({whenMatchedMode, whenNotMatchedMode}) => { + // Skip the combination of merge modes which will fail depending on the contents of the + // source and target collection, as this will cause the assertion below to trip. + if (whenMatchedMode == "fail" || whenNotMatchedMode == "fail") + return; + + targetColl.drop(); + assertErrorCode( + sourceColl, + [{ + $merge: { + into: {db: targetColl.getDB().getName(), coll: targetColl.getName()}, + whenMatched: whenMatchedMode, + whenNotMatched: whenNotMatchedMode, + on: "not_allowed" + } + }], + 51190); + }); + // If 'targetColl' is in the same database as 'sourceColl', test that the legacy $out works // correctly. if (targetColl.getDB() == sourceColl.getDB()) { diff --git a/src/mongo/db/pipeline/document_source_merge_spec.cpp b/src/mongo/db/pipeline/document_source_merge_spec.cpp index 988d322c4f4..fe6123261b4 100644 --- a/src/mongo/db/pipeline/document_source_merge_spec.cpp +++ b/src/mongo/db/pipeline/document_source_merge_spec.cpp @@ -78,7 +78,7 @@ std::vector<std::string> mergeOnFieldsParseFromBSON(const BSONElement& elem) { while (iter.more()) { const BSONElement matchByElem = iter.next(); uassert(51134, - "{} 'on' array elements must be strings, but found "_format( + "{} 'on' array elements must be strings, but found {}"_format( DocumentSourceMerge::kStageName, typeName(matchByElem.type())), matchByElem.type() == BSONType::String); fields.push_back(matchByElem.str()); |