diff options
3 files changed, 665 insertions, 470 deletions
diff --git a/jstests/aggregation/sources/lookup/lookup_array_semantics.js b/jstests/aggregation/sources/lookup/lookup_array_semantics.js deleted file mode 100644 index 5980f995f03..00000000000 --- a/jstests/aggregation/sources/lookup/lookup_array_semantics.js +++ /dev/null @@ -1,354 +0,0 @@ -/** - * Test that $lookup behaves correctly for array key values. - */ -(function() { -"use strict"; - -load("jstests/aggregation/extras/utils.js"); -load("jstests/libs/fixture_helpers.js"); // For isSharded. - -const localColl = db.lookup_arrays_semantics_local; -const foreignColl = db.lookup_arrays_semantics_foreign; - -// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to -// allow $lookup/$graphLookup into a sharded collection is disabled. -const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1}); -const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") && - getShardedLookupParam.featureFlagShardedLookup.value; -if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) { - return; -} - -// To make the tests easier to read, we run each with exactly one record in the foreign collection, -// so we don't need to check the content of the "matched" field but only that it's not empty. -function runTest( - {testDescription, localData, localField, foreignRecord, foreignField, idsExpectedToMatch}) { - localColl.drop(); - assert.commandWorked(localColl.insert(localData)); - - foreignColl.drop(); - assert.commandWorked(foreignColl.insert(foreignRecord)); - - const results = localColl.aggregate([{ - $lookup: { - from: foreignColl.getName(), - localField: localField, - foreignField: foreignField, - as: "matched" - } - }]).toArray(); - - // Build the array of ids for the results that have non-empty array in the "matched" field. - const matchedIds = results - .filter(function(x) { - return tojson(x.matched) != tojson([]); - }) - .map(x => (x._id)); - - // Order of the elements within the arrays is not significant for 'assertArrayEq'. - assertArrayEq( - {actual: matchedIds, expected: idsExpectedToMatch, extraErrorMsg: testDescription}); -} - -runTest({ - testDescription: "Scalar in local should match elements of arrays in foreign.", - localData: [ - // Expect to match "a" to 1. - {_id: 0, a: 1}, - {_id: 1, a: [1]}, - {_id: 2, a: [1, 2, 3]}, - {_id: 3, a: [1, [2, 3]]}, - {_id: 4, a: [1, [1, 2]]}, - {_id: 5, a: [1, 2, 1]}, - - // Don't expect to match. - {_id: 10, a: [[1]]}, - {_id: 11, a: [[1, 2], 3]}, - {_id: 12, a: 3}, - {_id: 13, a: [2, 3]}, - {_id: 14, a: {b: 1}} - ], - localField: "a", - foreignRecord: {_id: 0, b: 1}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3, 4, 5] -}); - -runTest({ - testDescription: "Elements in an array in foreign should match to scalars in local.", - localData: [ - // Expect to match "a" to [1, 2]. - {_id: 0, a: 1}, - {_id: 1, a: 2}, - {_id: 2, a: [1]}, - {_id: 3, a: [1, 3]}, - {_id: 4, a: [3, 2]}, - - // Don't expect to match. - {_id: 10, a: 3}, - {_id: 11, a: [[1]]}, - {_id: 12, a: [3, 4]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: [1, 2]}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3, 4] -}); - -runTest({ - testDescription: "An array in foreign should match as a whole value.", - localData: [ - // Expect to match "a" to [[1, 2], 3]. - {_id: 0, a: [[1, 2], 4]}, - {_id: 1, a: [[1, 2]]}, - - // Don't expect to match. - {_id: 10, a: [1, 2]}, // top-level arrays in local are always traversed - {_id: 11, a: [[[1, 2]]]}, - {_id: 12, a: [[1, 2, 3]]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: [[1, 2], 3]}, - foreignField: "b", - idsExpectedToMatch: [0, 1] -}); - -runTest({ - testDescription: "Scalar in foreign should match dotted paths in local.", - localData: [ - // Expect to match "a.x" to 1. - {_id: 0, a: [{x: 1}, {x: 2}]}, - {_id: 1, a: [{x: [1, 2]}]}, - {_id: 2, a: {x: 1}}, - {_id: 3, a: {x: [1, 2]}}, - - // Don't expect to match. - {_id: 10, a: [{x: 2}, {x: 3}]}, - {_id: 11, a: 1}, - {_id: 12, a: [1]}, - {_id: 13, a: [[1]]}, - {_id: 14, a: [{y: 1}]}, - ], - localField: "a.x", - foreignRecord: {_id: 0, b: 1}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3] -}); - -runTest({ - testDescription: "Elements of array in foreign should match dotted paths in local.", - localData: [ - // Expect to match "a.x" to [1, 2]. - {_id: 0, a: [{x: 1}, {x: 3}]}, - {_id: 1, a: [{x: [1, 3]}]}, - {_id: 2, a: {x: 1}}, - {_id: 3, a: {x: [1, 2]}}, - - // Don't expect to match. - {_id: 10, a: 1}, - {_id: 11, a: [1]}, - {_id: 12, a: [{x: 3}]}, - {_id: 13, a: [{y: 1}]}, - ], - localField: "a.x", - foreignRecord: {_id: 0, b: [1, 2]}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3] -}); - -runTest({ - testDescription: "An array in foreign should not match as a whole value to dotted paths.", - localData: [ - // Expect to match "a.x" to [[1, 2], 3]. - - // Don't expect to match. - {_id: 10, a: [{x: 1}, {x: 2}]}, - {_id: 11, a: [{x: [1, 2]}]}, - {_id: 12, a: {x: [1, 2]}}, - {_id: 13, a: [1, 2]}, - ], - localField: "a.x", - foreignRecord: {_id: 0, b: [[1, 2], 3]}, - foreignField: "b", - idsExpectedToMatch: [] -}); - -runTest({ - testDescription: "Array of objects in foreign should match on subfield.", - localData: [ - // Expect to match the same as if the foreign value was [1, 2]. - {_id: 0, a: 1}, - {_id: 1, a: 2}, - {_id: 2, a: [1]}, - {_id: 3, a: [1, 3]}, - {_id: 4, a: [3, 2]}, - {_id: 5, a: [[1, 2], 3]}, - - // Don't expect to match. - {_id: 10, a: 4}, - {_id: 11, a: [[1]]}, - {_id: 12, a: [3, 4]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: [{x: 1}, {x: 2}]}, - foreignField: "b.x", - idsExpectedToMatch: [0, 1, 2, 3, 4] -}); - -runTest({ - testDescription: "Empty array in foreign should only match to empty nested arrays.", - localData: [ - // Expect to match - {_id: 0, a: [[]]}, - {_id: 1, a: [[], 1]}, - - // Don't expect to match. - {_id: 10}, - {_id: 11, a: null}, - {_id: 12, a: []}, - {_id: 13, a: [null]}, - {_id: 14, a: [null, 1]}, - {_id: 15, a: 1}, - {_id: 16, a: [[[]]]}, - {_id: 17, a: [1, 2]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: []}, - foreignField: "b", - idsExpectedToMatch: [0, 1] -}); - -// This tests documents current behavior of the classic engine, which matches empty array in local -// to null in foreign. Update the test if SERVER-63368 is fixed. -runTest({ - testDescription: "Empty array in local and null in foreign.", - localData: [ - // Expect to match "a" to null. - {_id: 0, a: []}, - - // Don't expect to match. - {_id: 10, a: [[]]}, - {_id: 11, a: [[], 1]}, - {_id: 12, a: [1]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: null}, - foreignField: "b", - idsExpectedToMatch: [0] -}); - -// This tests documents current behavior of the classic engine, which matches empty array in local -// to missing in foreign. Update the test if SERVER-63368 is fixed. -runTest({ - testDescription: "Empty array in local and missing in foreign.", - localData: [ - // Expect to match - {_id: 0, a: []}, - - // Don't expect to match. - {_id: 10, a: [[]]}, - {_id: 11, a: [[], 1]}, - {_id: 12, a: [1]}, - ], - localField: "a", - foreignRecord: {_id: 0, no_b: true}, - foreignField: "b", - idsExpectedToMatch: [0] -}); - -// This tests documents current behavior of the classic engine, which matches empty array in local -// to undefined in foreign. Update the test if SERVER-63368 is fixed. -runTest({ - testDescription: "Empty array in local and undefined in foreign.", - localData: [ - // Expect to match - {_id: 0, a: []}, - - // Don't expect to match. - {_id: 10, a: [[]]}, - {_id: 11, a: [[], 1]}, - {_id: 12, a: [1]}, - ], - localField: "a", - foreignRecord: {_id: 0, b: undefined}, - foreignField: "b", - idsExpectedToMatch: [0] -}); - -runTest({ - testDescription: "Path with arrays in local can match to empty array", - localData: [ - {_id: 0, a: [{x: [[]]}]}, - - // Don't expect to match. - {_id: 10, a: [{y: 1}, {y: 2}]}, - {_id: 11, a: [[]]}, - {_id: 12, a: [{x: []}]}, - {_id: 13, a: []}, - ], - localField: "a.x", - foreignRecord: {_id: 0, b: []}, - foreignField: "b", - idsExpectedToMatch: [0] -}); - -runTest({ - testDescription: "Path with arrays in local can match to null", - localData: [ - {_id: 0, a: [{y: 1}, {y: 2}]}, - {_id: 1, a: [{x: []}]}, - {_id: 2, a: []}, - {_id: 3, a: [[]]}, - {_id: 4, a: {x: []}}, - {_id: 5, x: [1, 2]}, - - // Don't expect to match. - {_id: 10, a: [{x: 1}, {y: 2}]}, - {_id: 11, a: [{x: []}, {x: 1}]}, - {_id: 12, a: [{x: [[]]}]}, - ], - localField: "a.x", - foreignRecord: {_id: 0, b: null}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3, 4, 5] -}); - -runTest({ - testDescription: "Paths with numbers in local match to scalar", - localData: [ - {_id: 0, a: [{x: 1}, {x: 2}]}, - {_id: 1, a: [{x: [2, 3, 1]}]}, - - // Don't expect to match. - {_id: 10, a: [{x: 2}, {x: 1}]}, - {_id: 11, a: [{x: [2, 3]}]}, - {_id: 12, a: {x: 1}}, - {_id: 13, a: {x: [1, 2]}}, - {_id: 14, a: [{y: 1}, {x: 1}]}, - ], - localField: "a.0.x", - foreignRecord: {_id: 0, b: 1}, - foreignField: "b", - idsExpectedToMatch: [0, 1] -}); - -runTest({ - testDescription: "Paths with numbers in local can match to null", - localData: [ - {_id: 0, a: {x: 1}}, - {_id: 1, a: {x: [1, 2]}}, - {_id: 2, a: [{y: 1}, {x: 1}]}, - {_id: 3, a: []}, - {_id: 4, a: [[]]}, - {_id: 5, x: [1, 2]}, - - // Don't expect to match. - {_id: 10, a: [{x: 1}, {x: 2}]}, - {_id: 11, a: [{x: [1, 2]}]}, - ], - localField: "a.0.x", - foreignRecord: {_id: 0, b: null}, - foreignField: "b", - idsExpectedToMatch: [0, 1, 2, 3, 4, 5] -}); -}()); diff --git a/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js b/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js new file mode 100644 index 00000000000..0ced12819c6 --- /dev/null +++ b/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js @@ -0,0 +1,665 @@ +/** + * Tests for $lookup with localField/foreignField syntax. + */ +(function() { +"use strict"; + +load("jstests/aggregation/extras/utils.js"); +load("jstests/libs/fixture_helpers.js"); // For isSharded. + +const localColl = db.lookup_arrays_semantics_local; +const foreignColl = db.lookup_arrays_semantics_foreign; + +// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to +// allow $lookup/$graphLookup into a sharded collection is disabled. +const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1}); +const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") && + getShardedLookupParam.featureFlagShardedLookup.value; +if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) { + return; +} + +/** + * Executes $lookup with exactly one record in the foreign collection, so we don't need to check the + * content of the "as" field but only that it's not empty for local records with ids in + * 'idsExpectToMatch'. + */ +function runTest_SingleForeignRecord( + {testDescription, localRecords, localField, foreignRecord, foreignField, idsExpectedToMatch}) { + assert('object' === typeof (foreignRecord) && !Array.isArray(foreignRecord), + "foreignRecord should be a single document"); + + localColl.drop(); + assert.commandWorked(localColl.insert(localRecords)); + + foreignColl.drop(); + assert.commandWorked(foreignColl.insert(foreignRecord)); + + const results = localColl.aggregate([{ + $lookup: { + from: foreignColl.getName(), + localField: localField, + foreignField: foreignField, + as: "matched" + } + }]).toArray(); + + // Build the array of ids for the results that have non-empty array in the "matched" field. + const matchedIds = results + .filter(function(x) { + return tojson(x.matched) != tojson([]); + }) + .map(x => (x._id)); + + // Order of the elements within the arrays is not significant for 'assertArrayEq'. + assertArrayEq({ + actual: matchedIds, + expected: idsExpectedToMatch, + extraErrorMsg: " **TEST** " + testDescription + }); +} + +/** + * Executes $lookup with exactly one record in the local collection and checks that the "as" field + * for it contains documents with ids from `idsExpectedToMatch`. + */ +function runTest_SingleLocalRecord( + {testDescription, localRecord, localField, foreignRecords, foreignField, idsExpectedToMatch}) { + assert('object' === typeof (localRecord) && !Array.isArray(localRecord), + "localRecord should be a single document"); + + localColl.drop(); + assert.commandWorked(localColl.insert(localRecord)); + + foreignColl.drop(); + assert.commandWorked(foreignColl.insert(foreignRecords)); + + const results = localColl.aggregate([{ + $lookup: { + from: foreignColl.getName(), + localField: localField, + foreignField: foreignField, + as: "matched" + } + }]).toArray(); + assert.eq(1, results.length); + + // Extract matched foreign ids from the "matched" field. + const matchedIds = results[0].matched.map(x => (x._id)); + + // Order of the elements within the arrays is not significant for 'assertArrayEq'. + assertArrayEq({ + actual: matchedIds, + expected: idsExpectedToMatch, + extraErrorMsg: " **TEST** " + testDescription + }); +} + +/** + * Executes $lookup and expects it to fail with the specified 'expectedErrorCode`. + */ +function runTest_ExpectFailure( + {testDescription, localRecords, localField, foreignRecords, foreignField, expectedErrorCode}) { + localColl.drop(); + assert.commandWorked(localColl.insert(localRecords)); + + foreignColl.drop(); + assert.commandWorked(foreignColl.insert(foreignRecords)); + + assert.commandFailedWithCode( + localColl.runCommand("aggregate", { + pipeline: [{ + $lookup: { + from: foreignColl.getName(), + localField: localField, + foreignField: foreignField, + as: "matched" + } + }], + cursor: {} + }), + expectedErrorCode, + "**TEST** " + testDescription); +} + +/** + * Tests. + */ +(function testMatchingNullAndMissing() { + const docs = [ + {_id: 0, no_a: 1}, + {_id: 1, a: null}, + {_id: 2, a: [null, 1]}, + + {_id: 10, a: []}, + + {_id: 20, a: 1}, + {_id: 21, a: {x: null}}, + {_id: 22, a: [[null, 1], 2]}, + {_id: 23, a: {x: undefined}}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Null in foreign, top-level field in local", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: null}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 10] + }); + runTest_SingleLocalRecord({ + testDescription: "Null in local, top-level field in foreign", + localRecord: {_id: 0, b: null}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [0, 1, 2] + }); + + runTest_SingleForeignRecord({ + testDescription: "Missing in foreign, top-level field in local", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, no_b: 1}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 10] + }); + runTest_SingleLocalRecord({ + testDescription: "Missing in local, top-level field in foreign", + localRecord: {_id: 0, no_b: 1}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [0, 1, 2] + }); +})(); + +(function testMatchingUndefined() { + const docs = [ + {_id: 0, no_a: 1}, + {_id: 1, a: null}, + {_id: 2, a: []}, + {_id: 3, a: [null, 1]}, + + {_id: 10, a: 1}, + {_id: 11, a: {x: null}}, + {_id: 12, a: [[null, 1], 2]}, + {_id: 13, a: {x: undefined}}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Undefined in foreign, top-level field in local", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: undefined}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3] + }); + + // If the left-hand side collection has a value of undefined for "localField", then the query + // will fail. This is a consequence of the fact that queries which explicitly compare to + // undefined, such as {$eq:undefined}, are banned. Arguably this behavior could be improved, but + // we are unlikely to change it given that the undefined BSON type has been deprecated for many + // years. + runTest_ExpectFailure({ + testDescription: "Undefined in local, top-level field in foreign", + localRecords: {_id: 0, b: undefined}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + expectedErrorCode: ErrorCodes.BadValue + }); +})(); + +(function testMatchingTopLevelFieldToScalar() { + const docs = [ + // For these docs "a" resolves to a (logical) set that contains value "1". + {_id: 0, a: 1, y: 2}, + {_id: 1, a: [1]}, + {_id: 2, a: [1, 2, 3]}, + {_id: 3, a: [1, [2, 3]]}, + {_id: 4, a: [1, [1, 2]]}, + {_id: 5, a: [1, 2, 1]}, + {_id: 6, a: [1, null]}, + {_id: 7, a: [1, []]}, + + // For these docs "a" resolves to a (logical) set that does _not_ contain value "1". + {_id: 10, a: 2}, + {_id: 11, a: [[1], 2]}, + {_id: 12, a: [2, 3]}, + {_id: 13, a: {y: 1}}, + {_id: 14, no_a: 1}, + ]; + + // When matching a scalar, local and foreign collections are fully symmetric. + runTest_SingleForeignRecord({ + testDescription: "Top-level field in local and top-level scalar in foreign", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: 1}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level scalar in local and top-level field in foreign", + localRecord: {_id: 0, b: 1}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7] + }); +})(); + +(function testMatchingPathToScalar() { + const docs = [ + // For these docs "a.x" resolves to a (logical) set that contains value "1". + {_id: 0, a: {x: 1, y: 2}}, + {_id: 1, a: [{x: 1}, {x: 2}]}, + {_id: 2, a: [{x: 1}, {x: null}]}, + {_id: 3, a: [{x: 1}, {x: []}]}, + {_id: 4, a: [{x: 1}, {no_x: 2}]}, + {_id: 5, a: {x: [1, 2]}}, + {_id: 6, a: [{x: [1, 2]}]}, + {_id: 7, a: [{x: [1, 2]}, {no_x: 2}]}, + + // For these docs "a.x" should resolve to a (logical) set that does _not_ contain value "1". + {_id: 10, a: {x: 2, y: 1}}, + {_id: 11, a: {x: [2, 3], y: 1}}, + {_id: 12, a: [{no_x: 1}, {x: 2}, {x: 3}]}, + {_id: 13, a: {x: [[1], 2]}}, + {_id: 14, a: [{x: [[1], 2]}]}, + {_id: 15, a: {no_x: 1}}, + ]; + + // When matching a scalar, local and foreign collections are fully symmetric. + runTest_SingleForeignRecord({ + testDescription: "Path in local and top-level scalar in foreign", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: 1}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level scalar in local and path in foreign", + localRecord: {_id: 0, b: 1}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7] + }); +})(); + +(function testMatchingTopLevelFieldToArray() { + const docs = [ + // For these docs "a" resolves to a (logical) set that contains [1,2] array as a value. + {_id: 0, a: [[1, 2], 3], y: 4}, + {_id: 1, a: [[1, 2]]}, + + // For these docs "a.x" contains [1,2], 1 and 2 values when in foreign, but in local + // the contained values are 1 and 2 (no array). + {_id: 2, a: [1, 2], y: 3}, + + // For these docs "a" resolves to a (logical) set that does _not_ contain [1,2] as a value + // in neither local nor foreign but might contain "1" and/or "2". + {_id: 10, a: [[[1, 2], 3], 4]}, + {_id: 11, a: [2, 1]}, + {_id: 12, a: [[2, 1], 3], y: [1, 2]}, + {_id: 13, a: [[2, 1], 3], y: [[1, 2], 3]}, + {_id: 14, a: null}, + {_id: 15, no_a: [1, 2]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Top-level field in local and top-level array in foreign", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: [1, 2]}, + foreignField: "b", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, /*match on 1: */ 2, 11] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level array in local and top-level field in foreign", + localRecord: {_id: 0, b: [1, 2]}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [/*match on 1: */ 2, 11] + }); + + runTest_SingleForeignRecord({ + testDescription: "Top-level field in local and nested array in foreign", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: [[1, 2], 42]}, + foreignField: "b", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1] + }); + runTest_SingleLocalRecord({ + testDescription: "Nested array in local and top-level field in foreign", + localRecord: {_id: 0, b: [[1, 2], 42]}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2] + }); +})(); + +(function testMatchingPathToArray() { + const docs = [ + // For these docs "a.x" resolves to a (logical) set that contains [1,2] array as a value. + {_id: 0, a: {x: [[1, 2], 3], y: 4}}, + {_id: 1, a: [{x: [[1, 2], 3]}, {x: 4}]}, + {_id: 2, a: [{x: [[1, 2], 3]}, {x: null}]}, + {_id: 3, a: [{x: [[1, 2], 3]}, {no_x: 4}]}, + + // For these docs "a.x" contains [1,2], 1 and 2 values when in foreign, but in local + // the contained values are 1 and 2 (no array). + {_id: 4, a: {x: [1, 2], y: 4}}, + {_id: 5, a: [{x: [1, 2]}, {x: 4}]}, + {_id: 6, a: [{x: [1, 2]}, {x: null}]}, + {_id: 7, a: [{x: [1, 2]}, {no_x: 4}]}, + + // For these docs "a.x" resolves to a (logical) set that doesn't contain [1,2] as a value + // in neither local nor foreign but might contain "1" and/or "2". + {_id: 10, a: {x: [2, 1], y: [1, 2]}}, + {_id: 11, a: {x: [[2, 1], 3], y: [[1, 2], 3]}}, + {_id: 12, a: [{x: 1}, {x: 2}]}, + {_id: 13, a: {x: [[[1, 2], 3]]}}, + {_id: 14, a: [{x: [[[1, 2], 3]]}]}, + {_id: 15, a: {no_x: [[1, 2], 3]}}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Path in local and top-level array in foreign", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: [1, 2]}, + foreignField: "b", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3, /*match on 1: */ 4, 5, 6, 7, 10, 12] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level array in local and path in foreign", + localRecord: {_id: 0, b: [1, 2]}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [/*match on 1: */ 4, 5, 6, 7, 10, 12] + }); + + runTest_SingleForeignRecord({ + testDescription: "Path in local and nested array in foreign", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: [[1, 2], 42]}, + foreignField: "b", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3] + }); + runTest_SingleLocalRecord({ + testDescription: "Nested array in local and path in foreign", + localRecord: {_id: 0, b: [[1, 2], 42]}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3, 4, 5, 6, 7] + }); +})(); + +(function testMatchingMissingOnPath() { + const docs = [ + // "a.x" does not exist. + {_id: 0, a: {no_x: 1}}, + {_id: 1, a: {no_x: [1, 2]}}, + {_id: 2, a: [{no_x: 1}, {no_x: 2}]}, + {_id: 3, a: [{no_x: 2}, {x: 1}]}, + {_id: 4, a: [{no_x: [1, 2]}, {x: 1}]}, + {_id: 5, a: {x: null}}, + {_id: 6, a: [{x: null}, {x: 1}]}, + {_id: 7, a: [{x: null}, {x: [1]}]}, + {_id: 8, no_a: 1}, + {_id: 9, a: [1]}, + {_id: 10, a: []}, + {_id: 11, a: [[1]]}, + {_id: 12, a: [[]]}, + + // "a.x" exists. + {_id: 20, a: {x: 2, y: 1}}, + {_id: 21, a: [{x: 2}, {x: 3}]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Missing in local path and top-level null in foreign", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: null}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, /*SERVER-64060: 3, 4,*/ 5, 6, 7, 8, 9, 10, 11, 12] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level null in local and missing in foreign path", + localRecord: {_id: 0, b: null}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7, 8, /*SERVER-64006: 9, 10, 11, 12, 13*/] + }); + + runTest_SingleForeignRecord({ + testDescription: "Missing in local path and top-level missing in foreign", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, no_b: 1}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, /*SERVER-64060: 3, 4,*/ 5, 6, 7, 8, 9, 10, 11, 12] + }); + runTest_SingleLocalRecord({ + testDescription: "Top-level missing in local and missing in foreign path", + localRecord: {_id: 0, no_b: 1}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7, 8, /*SERVER-64006: 9, 10, 11, 12, 13*/] + }); +})(); + +(function testMatchingEmptyArrayInTopLevelField() { + const docs = [ + // For these docs "a" resolves to a (logical) set that contains empty array as a value. + {_id: 0, a: [[]]}, + {_id: 1, a: [[], 1]}, + {_id: 2, a: [[], null]}, + + // For these docs "a" resolves to a (logical) set that contains empty array as a value in + // foreign collection only. + {_id: 3, a: []}, + + // For these docs "a" key is either missing or contains null. + {_id: 10, no_a: 1}, + {_id: 11, a: null}, + {_id: 12, a: [null]}, + {_id: 13, a: [null, 1]}, + + // "a" doesn't contain neither empty array nor null. + {_id: 20, a: 1}, + {_id: 21, a: [[[], 1], 2]}, + {_id: 22, a: [1, 2]}, + {_id: 23, a: [[null, 1], 2]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Empty top-level array in foreign, top field in local", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: []}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2] + }); + runTest_SingleLocalRecord({ + testDescription: "Empty top-level array in local, top field in foreign", + localRecord: {_id: 0, b: []}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [/*SERVER-63368: */ 2, 10, 11, 12, 13] + }); + + runTest_SingleForeignRecord({ + testDescription: "Empty nested array in foreign, top field in local", + localRecords: docs, + localField: "a", + foreignRecord: {_id: 0, b: [[], 42]}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2] + }); + runTest_SingleLocalRecord({ + testDescription: "Empty nested array in local, top field in foreign", + localRecord: {_id: 0, b: [[], 42]}, + localField: "b", + foreignRecords: docs, + foreignField: "a", + idsExpectedToMatch: [0, 1, 2, 3] + }); +})(); + +(function testMatchingEmptyArrayOnPath() { + const docs = [ + // For these docs "a.x" resolves to a (logical) set that contains empty array as a value. + {_id: 0, a: {x: [[]], y: 1}}, + {_id: 1, a: [{x: [[]]}]}, + {_id: 2, a: [{x: [[]]}, {x: 1}]}, + {_id: 3, a: [{x: [[]]}, {x: null}]}, + {_id: 4, a: [{x: [[]]}, {no_x: 1}]}, + {_id: 5, a: {x: [[], 1]}}, + + // For these docs "a.x" resolves to a (logical) set that contains empty array as a value in + // foreign collection only. + {_id: 10, a: {x: [], y: 1}}, + {_id: 11, a: [{x: []}, {x: 1}]}, + {_id: 12, a: [{x: []}, {x: null}]}, + {_id: 13, a: [{x: []}, {no_x: 1}]}, + + // For these docs "a.x" key is either missing or contains null. + {_id: 20, no_a: 1}, + {_id: 21, a: {no_x: 1}}, + {_id: 22, a: [{no_x: 1}, {no_x: 2}]}, + {_id: 23, a: [{x: null}, {x: 1}]}, + + // SERVER-64006 + {_id: 30, a: []}, + {_id: 31, a: [1]}, + {_id: 32, a: [null, 1]}, + {_id: 33, a: [[]]}, + {_id: 34, a: [[1], 2]}, + + // "a.x" doesn't contain neither empty array nor null. + {_id: 40, a: {x: 1}}, + {_id: 41, a: {x: [[[], 1], 2]}}, + {_id: 42, a: {x: [1, 2]}}, + {_id: 43, a: {x: [[null, 1], 2]}}, + {_id: 44, a: [{x: 1}]}, + {_id: 45, a: [{x: [[[], 1], 2]}]}, + {_id: 46, a: [{x: [1, 2]}]}, + {_id: 47, a: [{x: [[null, 1], 2]}]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Empty top-level array in foreign, path in local", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: []}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5] + }); + runTest_SingleLocalRecord({ + testDescription: "Empty top-level array in local, path in foreign", + localRecord: {_id: 0, b: []}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [3, 4, 12, 13, 20, 21, 22, 23] + }); + + runTest_SingleForeignRecord({ + testDescription: "Empty nested array in foreign, path in local", + localRecords: docs, + localField: "a.x", + foreignRecord: {_id: 0, b: [[], 42]}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5] + }); + runTest_SingleLocalRecord({ + testDescription: "Empty nested array in local, path in foreign", + localRecord: {_id: 0, b: [[], 42]}, + localField: "b", + foreignRecords: docs, + foreignField: "a.x", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 10, 11, 12, 13] + }); +})(); + +(function testMatchingPathWithNumericComponentToScalar() { + const docs = [ + // For these docs "a.0.x" resolves to a (logical) set that contains value "1". + {_id: 0, a: [{x: 1}, {x: 2}]}, + {_id: 1, a: [{x: [2, 3, 1]}]}, + {_id: 2, a: [{x: 1}, {y: 1}]}, + + // For these docs "a.0.x" resolves to a (logical) set that does _not_ contain value "1". + {_id: 10, a: [{x: 2}, {x: 1}]}, + {_id: 11, a: [{x: [2, 3]}]}, + {_id: 12, a: {x: 1}}, + {_id: 13, a: {x: [1, 2]}}, + {_id: 14, a: [{y: 1}, {x: 1}]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Scalar in foreign, path with numeral in local", + localRecords: docs, + localField: "a.0.x", + foreignRecord: {_id: 0, b: 1}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2] + }); + runTest_SingleLocalRecord({ + testDescription: "Scalar in local, path with numeral in foreign", + localRecord: {_id: 0, b: 1}, + localField: "b", + foreignRecords: docs, + foreignField: "a.0.x", + idsExpectedToMatch: [0, 1, 2] + }); +})(); + +(function testMatchingPathWithNumericComponentToNull() { + const docs = [ + // For these docs "a.0.x" resolves to a (logical) set that contains value "null". + {_id: 0, a: {x: 1}}, + {_id: 1, a: {x: [1, 2]}}, + {_id: 2, a: [{y: 1}, {x: 1}]}, + {_id: 3, a: [{x: null}, {x: 1}]}, + {_id: 4, a: [{x: [1, null]}, {x: 1}]}, + {_id: 5, a: [1, 2]}, + + // For these docs "a.0.x" resolves to a (logical) set that does _not_ contain value "null". + {_id: 10, a: [{x: 1}, {y: 1}]}, + {_id: 11, a: [{x: [1, 2]}]}, + ]; + + runTest_SingleForeignRecord({ + testDescription: "Null in foreign, path with numeral in local", + localRecords: docs, + localField: "a.0.x", + foreignRecord: {_id: 0, b: null}, + foreignField: "b", + idsExpectedToMatch: [0, 1, 2, 3, 4, 5] + }); + runTest_SingleLocalRecord({ + testDescription: "Null in local, path with numeral in foreign", + localRecord: {_id: 0, b: null}, + localField: "b", + foreignRecords: docs, + foreignField: "a.0.x", + idsExpectedToMatch: [0, 1, 2, 3, 4, /*SERVER-64006: 5,*/ /*SERVER-64221: */ 10, 11] + }); +})(); +}()); diff --git a/jstests/aggregation/sources/lookup/lookup_null_semantics.js b/jstests/aggregation/sources/lookup/lookup_null_semantics.js deleted file mode 100644 index 38067839db2..00000000000 --- a/jstests/aggregation/sources/lookup/lookup_null_semantics.js +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Test that $lookup behaves correctly for null values, as well as "missing" and undefined. - */ -(function() { -"use strict"; - -load("jstests/aggregation/extras/utils.js"); -load("jstests/libs/fixture_helpers.js"); // For isSharded. - -const localColl = db.lookup_null_semantics_local; -const foreignColl = db.lookup_null_semantics_foreign; - -localColl.drop(); -foreignColl.drop(); - -// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to -// allow $lookup/$graphLookup into a sharded collection is disabled. -const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1}); -const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") && - getShardedLookupParam.featureFlagShardedLookup.value; -if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) { - return; -} - -assert.commandWorked(localColl.insert([ - {_id: 0}, - {_id: 1, a: null}, - {_id: 2, a: 9}, - {_id: 3, a: [null, 9]}, -])); -assert.commandWorked(foreignColl.insert([ - {_id: 0}, - {_id: 1, b: null}, - {_id: 2, b: undefined}, - {_id: 3, b: 9}, - {_id: 4, b: [null, 9]}, - {_id: 5, b: 42}, - {_id: 6, b: [undefined, 9]}, - {_id: 7, b: [9, 10]}, -])); - -const actualResults = localColl.aggregate([{ - $lookup: {from: foreignColl.getName(), localField: "a", foreignField: "b", as: "lookupResults"} -}]).toArray(); - -const expectedResults = [ - // Missing on the left-hand side results in a lookup with $eq:null semantics on the left-hand - // side. Namely, we expect this document to join with null, missing, and undefined, or arrays - // thereof. - { - _id: 0, - lookupResults: [ - {_id: 0}, - {_id: 1, b: null}, - {_id: 2, b: undefined}, - {_id: 4, b: [null, 9]}, - {_id: 6, b: [undefined, 9]} - ] - }, - // Null on the left-hand side is the same as the missing case above. - { - _id: 1, - a: null, - lookupResults: [ - {_id: 0}, - {_id: 1, b: null}, - {_id: 2, b: undefined}, - {_id: 4, b: [null, 9]}, - {_id: 6, b: [undefined, 9]} - ] - }, - // A "negative" test-case where the value being looked up is not nullish. - { - _id: 2, - a: 9, - lookupResults: [ - {_id: 3, b: 9}, - {_id: 4, b: [null, 9]}, - {_id: 6, b: [undefined, 9]}, - {_id: 7, b: [9, 10]}, - ] - }, - // Here we are looking up both null and a scalar. We expected missing, null, and undefined to - // match in addition to the matches due to the scalar. - { - _id: 3, - a: [null, 9], - lookupResults: [ - {_id: 0}, - {_id: 1, b: null}, - {_id: 2, b: undefined}, - {_id: 3, b: 9}, - {_id: 4, b: [null, 9]}, - {_id: 6, b: [undefined, 9]}, - {_id: 7, b: [9, 10]}, - ] - }, -]; - -assertArrayEq({actual: actualResults, expected: expectedResults}); - -// The results should not change there is an index available on the right-hand side. -assert.commandWorked(foreignColl.createIndex({b: 1})); -assertArrayEq({actual: actualResults, expected: expectedResults}); - -// If the left-hand side collection has a value of undefined for "localField", then the query will -// fail. This is a consequence of the fact that queries which explicitly compare to undefined, such -// as {$eq:undefined}, are banned. Arguably this behavior could be improved, but we are unlikely to -// change it given that the undefined BSON type has been deprecated for many years. -assert.commandWorked(localColl.insert({a: undefined})); -assert.throws(() => { - localColl.aggregate([{ - $lookup: {from: foreignColl.getName(), localField: "a", foreignField: "b", as: "lookupResults"} -}]).toArray(); -}); -}()); |