SERVER-64144 Improve JS tests for equijoin lookup

3 files changed, 665 insertions, 470 deletions

diff --git a/jstests/aggregation/sources/lookup/lookup_array_semantics.js b/jstests/aggregation/sources/lookup/lookup_array_semantics.js
deleted file mode 100644
index 5980f995f03..00000000000
--- a/jstests/aggregation/sources/lookup/lookup_array_semantics.js
+++ /dev/null
@@ -1,354 +0,0 @@
-/**
- * Test that $lookup behaves correctly for array key values.
- */
-(function() {
-"use strict";
-
-load("jstests/aggregation/extras/utils.js");
-load("jstests/libs/fixture_helpers.js");  // For isSharded.
-
-const localColl = db.lookup_arrays_semantics_local;
-const foreignColl = db.lookup_arrays_semantics_foreign;
-
-// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to
-// allow $lookup/$graphLookup into a sharded collection is disabled.
-const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1});
-const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") &&
-    getShardedLookupParam.featureFlagShardedLookup.value;
-if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) {
-    return;
-}
-
-// To make the tests easier to read, we run each with exactly one record in the foreign collection,
-// so we don't need to check the content of the "matched" field but only that it's not empty.
-function runTest(
-    {testDescription, localData, localField, foreignRecord, foreignField, idsExpectedToMatch}) {
-    localColl.drop();
-    assert.commandWorked(localColl.insert(localData));
-
-    foreignColl.drop();
-    assert.commandWorked(foreignColl.insert(foreignRecord));
-
-    const results = localColl.aggregate([{
-        $lookup: {
-            from: foreignColl.getName(),
-            localField: localField,
-            foreignField: foreignField,
-            as: "matched"
-        }
-    }]).toArray();
-
-    // Build the array of ids for the results that have non-empty array in the "matched" field.
-    const matchedIds = results
-                           .filter(function(x) {
-                               return tojson(x.matched) != tojson([]);
-                           })
-                           .map(x => (x._id));
-
-    // Order of the elements within the arrays is not significant for 'assertArrayEq'.
-    assertArrayEq(
-        {actual: matchedIds, expected: idsExpectedToMatch, extraErrorMsg: testDescription});
-}
-
-runTest({
-    testDescription: "Scalar in local should match elements of arrays in foreign.",
-    localData: [
-        // Expect to match "a" to 1.
-        {_id: 0, a: 1},
-        {_id: 1, a: [1]},
-        {_id: 2, a: [1, 2, 3]},
-        {_id: 3, a: [1, [2, 3]]},
-        {_id: 4, a: [1, [1, 2]]},
-        {_id: 5, a: [1, 2, 1]},
-
-        // Don't expect to match.
-        {_id: 10, a: [[1]]},
-        {_id: 11, a: [[1, 2], 3]},
-        {_id: 12, a: 3},
-        {_id: 13, a: [2, 3]},
-        {_id: 14, a: {b: 1}}
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: 1},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
-});
-
-runTest({
-    testDescription: "Elements in an array in foreign should match to scalars in local.",
-    localData: [
-        // Expect to match "a" to [1, 2].
-        {_id: 0, a: 1},
-        {_id: 1, a: 2},
-        {_id: 2, a: [1]},
-        {_id: 3, a: [1, 3]},
-        {_id: 4, a: [3, 2]},
-
-        // Don't expect to match.
-        {_id: 10, a: 3},
-        {_id: 11, a: [[1]]},
-        {_id: 12, a: [3, 4]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: [1, 2]},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3, 4]
-});
-
-runTest({
-    testDescription: "An array in foreign should match as a whole value.",
-    localData: [
-        // Expect to match "a" to [[1, 2], 3].
-        {_id: 0, a: [[1, 2], 4]},
-        {_id: 1, a: [[1, 2]]},
-
-        // Don't expect to match.
-        {_id: 10, a: [1, 2]},  // top-level arrays in local are always traversed
-        {_id: 11, a: [[[1, 2]]]},
-        {_id: 12, a: [[1, 2, 3]]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: [[1, 2], 3]},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1]
-});
-
-runTest({
-    testDescription: "Scalar in foreign should match dotted paths in local.",
-    localData: [
-        // Expect to match "a.x" to 1.
-        {_id: 0, a: [{x: 1}, {x: 2}]},
-        {_id: 1, a: [{x: [1, 2]}]},
-        {_id: 2, a: {x: 1}},
-        {_id: 3, a: {x: [1, 2]}},
-
-        // Don't expect to match.
-        {_id: 10, a: [{x: 2}, {x: 3}]},
-        {_id: 11, a: 1},
-        {_id: 12, a: [1]},
-        {_id: 13, a: [[1]]},
-        {_id: 14, a: [{y: 1}]},
-    ],
-    localField: "a.x",
-    foreignRecord: {_id: 0, b: 1},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3]
-});
-
-runTest({
-    testDescription: "Elements of array in foreign should match dotted paths in local.",
-    localData: [
-        // Expect to match "a.x" to [1, 2].
-        {_id: 0, a: [{x: 1}, {x: 3}]},
-        {_id: 1, a: [{x: [1, 3]}]},
-        {_id: 2, a: {x: 1}},
-        {_id: 3, a: {x: [1, 2]}},
-
-        // Don't expect to match.
-        {_id: 10, a: 1},
-        {_id: 11, a: [1]},
-        {_id: 12, a: [{x: 3}]},
-        {_id: 13, a: [{y: 1}]},
-    ],
-    localField: "a.x",
-    foreignRecord: {_id: 0, b: [1, 2]},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3]
-});
-
-runTest({
-    testDescription: "An array in foreign should not match as a whole value to dotted paths.",
-    localData: [
-        // Expect to match "a.x" to [[1, 2], 3].
-
-        // Don't expect to match.
-        {_id: 10, a: [{x: 1}, {x: 2}]},
-        {_id: 11, a: [{x: [1, 2]}]},
-        {_id: 12, a: {x: [1, 2]}},
-        {_id: 13, a: [1, 2]},
-    ],
-    localField: "a.x",
-    foreignRecord: {_id: 0, b: [[1, 2], 3]},
-    foreignField: "b",
-    idsExpectedToMatch: []
-});
-
-runTest({
-    testDescription: "Array of objects in foreign should match on subfield.",
-    localData: [
-        // Expect to match the same as if the foreign value was [1, 2].
-        {_id: 0, a: 1},
-        {_id: 1, a: 2},
-        {_id: 2, a: [1]},
-        {_id: 3, a: [1, 3]},
-        {_id: 4, a: [3, 2]},
-        {_id: 5, a: [[1, 2], 3]},
-
-        // Don't expect to match.
-        {_id: 10, a: 4},
-        {_id: 11, a: [[1]]},
-        {_id: 12, a: [3, 4]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: [{x: 1}, {x: 2}]},
-    foreignField: "b.x",
-    idsExpectedToMatch: [0, 1, 2, 3, 4]
-});
-
-runTest({
-    testDescription: "Empty array in foreign should only match to empty nested arrays.",
-    localData: [
-        // Expect to match
-        {_id: 0, a: [[]]},
-        {_id: 1, a: [[], 1]},
-
-        // Don't expect to match.
-        {_id: 10},
-        {_id: 11, a: null},
-        {_id: 12, a: []},
-        {_id: 13, a: [null]},
-        {_id: 14, a: [null, 1]},
-        {_id: 15, a: 1},
-        {_id: 16, a: [[[]]]},
-        {_id: 17, a: [1, 2]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: []},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1]
-});
-
-// This tests documents current behavior of the classic engine, which matches empty array in local
-// to null in foreign. Update the test if SERVER-63368 is fixed.
-runTest({
-    testDescription: "Empty array in local and null in foreign.",
-    localData: [
-        // Expect to match "a" to null.
-        {_id: 0, a: []},
-
-        // Don't expect to match.
-        {_id: 10, a: [[]]},
-        {_id: 11, a: [[], 1]},
-        {_id: 12, a: [1]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: null},
-    foreignField: "b",
-    idsExpectedToMatch: [0]
-});
-
-// This tests documents current behavior of the classic engine, which matches empty array in local
-// to missing in foreign. Update the test if SERVER-63368 is fixed.
-runTest({
-    testDescription: "Empty array in local and missing in foreign.",
-    localData: [
-        // Expect to match
-        {_id: 0, a: []},
-
-        // Don't expect to match.
-        {_id: 10, a: [[]]},
-        {_id: 11, a: [[], 1]},
-        {_id: 12, a: [1]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, no_b: true},
-    foreignField: "b",
-    idsExpectedToMatch: [0]
-});
-
-// This tests documents current behavior of the classic engine, which matches empty array in local
-// to undefined in foreign. Update the test if SERVER-63368 is fixed.
-runTest({
-    testDescription: "Empty array in local and undefined in foreign.",
-    localData: [
-        // Expect to match
-        {_id: 0, a: []},
-
-        // Don't expect to match.
-        {_id: 10, a: [[]]},
-        {_id: 11, a: [[], 1]},
-        {_id: 12, a: [1]},
-    ],
-    localField: "a",
-    foreignRecord: {_id: 0, b: undefined},
-    foreignField: "b",
-    idsExpectedToMatch: [0]
-});
-
-runTest({
-    testDescription: "Path with arrays in local can match to empty array",
-    localData: [
-        {_id: 0, a: [{x: [[]]}]},
-
-        // Don't expect to match.
-        {_id: 10, a: [{y: 1}, {y: 2}]},
-        {_id: 11, a: [[]]},
-        {_id: 12, a: [{x: []}]},
-        {_id: 13, a: []},
-    ],
-    localField: "a.x",
-    foreignRecord: {_id: 0, b: []},
-    foreignField: "b",
-    idsExpectedToMatch: [0]
-});
-
-runTest({
-    testDescription: "Path with arrays in local can match to null",
-    localData: [
-        {_id: 0, a: [{y: 1}, {y: 2}]},
-        {_id: 1, a: [{x: []}]},
-        {_id: 2, a: []},
-        {_id: 3, a: [[]]},
-        {_id: 4, a: {x: []}},
-        {_id: 5, x: [1, 2]},
-
-        // Don't expect to match.
-        {_id: 10, a: [{x: 1}, {y: 2}]},
-        {_id: 11, a: [{x: []}, {x: 1}]},
-        {_id: 12, a: [{x: [[]]}]},
-    ],
-    localField: "a.x",
-    foreignRecord: {_id: 0, b: null},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
-});
-
-runTest({
-    testDescription: "Paths with numbers in local match to scalar",
-    localData: [
-        {_id: 0, a: [{x: 1}, {x: 2}]},
-        {_id: 1, a: [{x: [2, 3, 1]}]},
-
-        // Don't expect to match.
-        {_id: 10, a: [{x: 2}, {x: 1}]},
-        {_id: 11, a: [{x: [2, 3]}]},
-        {_id: 12, a: {x: 1}},
-        {_id: 13, a: {x: [1, 2]}},
-        {_id: 14, a: [{y: 1}, {x: 1}]},
-    ],
-    localField: "a.0.x",
-    foreignRecord: {_id: 0, b: 1},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1]
-});
-
-runTest({
-    testDescription: "Paths with numbers in local can match to null",
-    localData: [
-        {_id: 0, a: {x: 1}},
-        {_id: 1, a: {x: [1, 2]}},
-        {_id: 2, a: [{y: 1}, {x: 1}]},
-        {_id: 3, a: []},
-        {_id: 4, a: [[]]},
-        {_id: 5, x: [1, 2]},
-
-        // Don't expect to match.
-        {_id: 10, a: [{x: 1}, {x: 2}]},
-        {_id: 11, a: [{x: [1, 2]}]},
-    ],
-    localField: "a.0.x",
-    foreignRecord: {_id: 0, b: null},
-    foreignField: "b",
-    idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
-});
-}());
diff --git a/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js b/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js
new file mode 100644
index 00000000000..0ced12819c6
--- /dev/null
+++ b/jstests/aggregation/sources/lookup/lookup_equijoin_semantics.js
@@ -0,0 +1,665 @@
+/**
+ * Tests for $lookup with localField/foreignField syntax.
+ */
+(function() {
+"use strict";
+
+load("jstests/aggregation/extras/utils.js");
+load("jstests/libs/fixture_helpers.js");  // For isSharded.
+
+const localColl = db.lookup_arrays_semantics_local;
+const foreignColl = db.lookup_arrays_semantics_foreign;
+
+// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to
+// allow $lookup/$graphLookup into a sharded collection is disabled.
+const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1});
+const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") &&
+    getShardedLookupParam.featureFlagShardedLookup.value;
+if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) {
+    return;
+}
+
+/**
+ * Executes $lookup with exactly one record in the foreign collection, so we don't need to check the
+ * content of the "as" field but only that it's not empty for local records with ids in
+ * 'idsExpectToMatch'.
+ */
+function runTest_SingleForeignRecord(
+    {testDescription, localRecords, localField, foreignRecord, foreignField, idsExpectedToMatch}) {
+    assert('object' === typeof (foreignRecord) && !Array.isArray(foreignRecord),
+           "foreignRecord should be a single document");
+
+    localColl.drop();
+    assert.commandWorked(localColl.insert(localRecords));
+
+    foreignColl.drop();
+    assert.commandWorked(foreignColl.insert(foreignRecord));
+
+    const results = localColl.aggregate([{
+            $lookup: {
+                from: foreignColl.getName(),
+                localField: localField,
+                foreignField: foreignField,
+                as: "matched"
+            }
+        }]).toArray();
+
+    // Build the array of ids for the results that have non-empty array in the "matched" field.
+    const matchedIds = results
+                           .filter(function(x) {
+                               return tojson(x.matched) != tojson([]);
+                           })
+                           .map(x => (x._id));
+
+    // Order of the elements within the arrays is not significant for 'assertArrayEq'.
+    assertArrayEq({
+        actual: matchedIds,
+        expected: idsExpectedToMatch,
+        extraErrorMsg: " **TEST** " + testDescription
+    });
+}
+
+/**
+ * Executes $lookup with exactly one record in the local collection and checks that the "as" field
+ * for it contains documents with ids from `idsExpectedToMatch`.
+ */
+function runTest_SingleLocalRecord(
+    {testDescription, localRecord, localField, foreignRecords, foreignField, idsExpectedToMatch}) {
+    assert('object' === typeof (localRecord) && !Array.isArray(localRecord),
+           "localRecord should be a single document");
+
+    localColl.drop();
+    assert.commandWorked(localColl.insert(localRecord));
+
+    foreignColl.drop();
+    assert.commandWorked(foreignColl.insert(foreignRecords));
+
+    const results = localColl.aggregate([{
+            $lookup: {
+                from: foreignColl.getName(),
+                localField: localField,
+                foreignField: foreignField,
+                as: "matched"
+            }
+        }]).toArray();
+    assert.eq(1, results.length);
+
+    // Extract matched foreign ids from the "matched" field.
+    const matchedIds = results[0].matched.map(x => (x._id));
+
+    // Order of the elements within the arrays is not significant for 'assertArrayEq'.
+    assertArrayEq({
+        actual: matchedIds,
+        expected: idsExpectedToMatch,
+        extraErrorMsg: " **TEST** " + testDescription
+    });
+}
+
+/**
+ * Executes $lookup and expects it to fail with the specified 'expectedErrorCode`.
+ */
+function runTest_ExpectFailure(
+    {testDescription, localRecords, localField, foreignRecords, foreignField, expectedErrorCode}) {
+    localColl.drop();
+    assert.commandWorked(localColl.insert(localRecords));
+
+    foreignColl.drop();
+    assert.commandWorked(foreignColl.insert(foreignRecords));
+
+    assert.commandFailedWithCode(
+        localColl.runCommand("aggregate", {
+            pipeline: [{
+                $lookup: {
+                    from: foreignColl.getName(),
+                    localField: localField,
+                    foreignField: foreignField,
+                    as: "matched"
+                }
+            }],
+            cursor: {}
+        }),
+        expectedErrorCode,
+        "**TEST** " + testDescription);
+}
+
+/**
+ * Tests.
+ */
+(function testMatchingNullAndMissing() {
+    const docs = [
+        {_id: 0, no_a: 1},
+        {_id: 1, a: null},
+        {_id: 2, a: [null, 1]},
+
+        {_id: 10, a: []},
+
+        {_id: 20, a: 1},
+        {_id: 21, a: {x: null}},
+        {_id: 22, a: [[null, 1], 2]},
+        {_id: 23, a: {x: undefined}},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Null in foreign, top-level field in local",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: null},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 10]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Null in local, top-level field in foreign",
+        localRecord: {_id: 0, b: null},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Missing in foreign, top-level field in local",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, no_b: 1},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 10]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Missing in local, top-level field in foreign",
+        localRecord: {_id: 0, no_b: 1},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+})();
+
+(function testMatchingUndefined() {
+    const docs = [
+        {_id: 0, no_a: 1},
+        {_id: 1, a: null},
+        {_id: 2, a: []},
+        {_id: 3, a: [null, 1]},
+
+        {_id: 10, a: 1},
+        {_id: 11, a: {x: null}},
+        {_id: 12, a: [[null, 1], 2]},
+        {_id: 13, a: {x: undefined}},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Undefined in foreign, top-level field in local",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: undefined},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3]
+    });
+
+    // If the left-hand side collection has a value of undefined for "localField", then the query
+    // will fail. This is a consequence of the fact that queries which explicitly compare to
+    // undefined, such as {$eq:undefined}, are banned. Arguably this behavior could be improved, but
+    // we are unlikely to change it given that the undefined BSON type has been deprecated for many
+    // years.
+    runTest_ExpectFailure({
+        testDescription: "Undefined in local, top-level field in foreign",
+        localRecords: {_id: 0, b: undefined},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        expectedErrorCode: ErrorCodes.BadValue
+    });
+})();
+
+(function testMatchingTopLevelFieldToScalar() {
+    const docs = [
+        // For these docs "a" resolves to a (logical) set that contains value "1".
+        {_id: 0, a: 1, y: 2},
+        {_id: 1, a: [1]},
+        {_id: 2, a: [1, 2, 3]},
+        {_id: 3, a: [1, [2, 3]]},
+        {_id: 4, a: [1, [1, 2]]},
+        {_id: 5, a: [1, 2, 1]},
+        {_id: 6, a: [1, null]},
+        {_id: 7, a: [1, []]},
+
+        // For these docs "a" resolves to a (logical) set that does _not_ contain value "1".
+        {_id: 10, a: 2},
+        {_id: 11, a: [[1], 2]},
+        {_id: 12, a: [2, 3]},
+        {_id: 13, a: {y: 1}},
+        {_id: 14, no_a: 1},
+    ];
+
+    // When matching a scalar, local and foreign collections are fully symmetric.
+    runTest_SingleForeignRecord({
+        testDescription: "Top-level field in local and top-level scalar in foreign",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: 1},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level scalar in local and top-level field in foreign",
+        localRecord: {_id: 0, b: 1},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7]
+    });
+})();
+
+(function testMatchingPathToScalar() {
+    const docs = [
+        // For these docs "a.x" resolves to a (logical) set that contains value "1".
+        {_id: 0, a: {x: 1, y: 2}},
+        {_id: 1, a: [{x: 1}, {x: 2}]},
+        {_id: 2, a: [{x: 1}, {x: null}]},
+        {_id: 3, a: [{x: 1}, {x: []}]},
+        {_id: 4, a: [{x: 1}, {no_x: 2}]},
+        {_id: 5, a: {x: [1, 2]}},
+        {_id: 6, a: [{x: [1, 2]}]},
+        {_id: 7, a: [{x: [1, 2]}, {no_x: 2}]},
+
+        // For these docs "a.x" should resolve to a (logical) set that does _not_ contain value "1".
+        {_id: 10, a: {x: 2, y: 1}},
+        {_id: 11, a: {x: [2, 3], y: 1}},
+        {_id: 12, a: [{no_x: 1}, {x: 2}, {x: 3}]},
+        {_id: 13, a: {x: [[1], 2]}},
+        {_id: 14, a: [{x: [[1], 2]}]},
+        {_id: 15, a: {no_x: 1}},
+    ];
+
+    // When matching a scalar, local and foreign collections are fully symmetric.
+    runTest_SingleForeignRecord({
+        testDescription: "Path in local and top-level scalar in foreign",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: 1},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level scalar in local and path in foreign",
+        localRecord: {_id: 0, b: 1},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7]
+    });
+})();
+
+(function testMatchingTopLevelFieldToArray() {
+    const docs = [
+        // For these docs "a" resolves to a (logical) set that contains [1,2] array as a value.
+        {_id: 0, a: [[1, 2], 3], y: 4},
+        {_id: 1, a: [[1, 2]]},
+
+        // For these docs "a.x" contains [1,2], 1 and 2 values when in foreign, but in local
+        // the contained values are 1 and 2 (no array).
+        {_id: 2, a: [1, 2], y: 3},
+
+        // For these docs "a" resolves to a (logical) set that does _not_ contain [1,2] as a value
+        // in neither local nor foreign but might contain "1" and/or "2".
+        {_id: 10, a: [[[1, 2], 3], 4]},
+        {_id: 11, a: [2, 1]},
+        {_id: 12, a: [[2, 1], 3], y: [1, 2]},
+        {_id: 13, a: [[2, 1], 3], y: [[1, 2], 3]},
+        {_id: 14, a: null},
+        {_id: 15, no_a: [1, 2]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Top-level field in local and top-level array in foreign",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: [1, 2]},
+        foreignField: "b",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, /*match on 1: */ 2, 11]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level array in local and top-level field in foreign",
+        localRecord: {_id: 0, b: [1, 2]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [/*match on 1: */ 2, 11]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Top-level field in local and nested array in foreign",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: [[1, 2], 42]},
+        foreignField: "b",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Nested array in local and top-level field in foreign",
+        localRecord: {_id: 0, b: [[1, 2], 42]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2]
+    });
+})();
+
+(function testMatchingPathToArray() {
+    const docs = [
+        // For these docs "a.x" resolves to a (logical) set that contains [1,2] array as a value.
+        {_id: 0, a: {x: [[1, 2], 3], y: 4}},
+        {_id: 1, a: [{x: [[1, 2], 3]}, {x: 4}]},
+        {_id: 2, a: [{x: [[1, 2], 3]}, {x: null}]},
+        {_id: 3, a: [{x: [[1, 2], 3]}, {no_x: 4}]},
+
+        // For these docs "a.x" contains [1,2], 1 and 2 values when in foreign, but in local
+        // the contained values are 1 and 2 (no array).
+        {_id: 4, a: {x: [1, 2], y: 4}},
+        {_id: 5, a: [{x: [1, 2]}, {x: 4}]},
+        {_id: 6, a: [{x: [1, 2]}, {x: null}]},
+        {_id: 7, a: [{x: [1, 2]}, {no_x: 4}]},
+
+        // For these docs "a.x" resolves to a (logical) set that doesn't contain [1,2] as a value
+        // in neither local nor foreign but might contain "1" and/or "2".
+        {_id: 10, a: {x: [2, 1], y: [1, 2]}},
+        {_id: 11, a: {x: [[2, 1], 3], y: [[1, 2], 3]}},
+        {_id: 12, a: [{x: 1}, {x: 2}]},
+        {_id: 13, a: {x: [[[1, 2], 3]]}},
+        {_id: 14, a: [{x: [[[1, 2], 3]]}]},
+        {_id: 15, a: {no_x: [[1, 2], 3]}},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Path in local and top-level array in foreign",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: [1, 2]},
+        foreignField: "b",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3, /*match on 1: */ 4, 5, 6, 7, 10, 12]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level array in local and path in foreign",
+        localRecord: {_id: 0, b: [1, 2]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [/*match on 1: */ 4, 5, 6, 7, 10, 12]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Path in local and nested array in foreign",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: [[1, 2], 42]},
+        foreignField: "b",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Nested array in local and path in foreign",
+        localRecord: {_id: 0, b: [[1, 2], 42]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [/*match on [1, 2]: */ 0, 1, 2, 3, 4, 5, 6, 7]
+    });
+})();
+
+(function testMatchingMissingOnPath() {
+    const docs = [
+        // "a.x" does not exist.
+        {_id: 0, a: {no_x: 1}},
+        {_id: 1, a: {no_x: [1, 2]}},
+        {_id: 2, a: [{no_x: 1}, {no_x: 2}]},
+        {_id: 3, a: [{no_x: 2}, {x: 1}]},
+        {_id: 4, a: [{no_x: [1, 2]}, {x: 1}]},
+        {_id: 5, a: {x: null}},
+        {_id: 6, a: [{x: null}, {x: 1}]},
+        {_id: 7, a: [{x: null}, {x: [1]}]},
+        {_id: 8, no_a: 1},
+        {_id: 9, a: [1]},
+        {_id: 10, a: []},
+        {_id: 11, a: [[1]]},
+        {_id: 12, a: [[]]},
+
+        // "a.x" exists.
+        {_id: 20, a: {x: 2, y: 1}},
+        {_id: 21, a: [{x: 2}, {x: 3}]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Missing in local path and top-level null in foreign",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: null},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, /*SERVER-64060: 3, 4,*/ 5, 6, 7, 8, 9, 10, 11, 12]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level null in local and missing in foreign path",
+        localRecord: {_id: 0, b: null},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7, 8, /*SERVER-64006: 9, 10, 11, 12, 13*/]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Missing in local path and top-level missing in foreign",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, no_b: 1},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, /*SERVER-64060: 3, 4,*/ 5, 6, 7, 8, 9, 10, 11, 12]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Top-level missing in local and missing in foreign path",
+        localRecord: {_id: 0, no_b: 1},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 6, 7, 8, /*SERVER-64006: 9, 10, 11, 12, 13*/]
+    });
+})();
+
+(function testMatchingEmptyArrayInTopLevelField() {
+    const docs = [
+        // For these docs "a" resolves to a (logical) set that contains empty array as a value.
+        {_id: 0, a: [[]]},
+        {_id: 1, a: [[], 1]},
+        {_id: 2, a: [[], null]},
+
+        // For these docs "a" resolves to a (logical) set that contains empty array as a value in
+        // foreign collection only.
+        {_id: 3, a: []},
+
+        // For these docs "a" key is either missing or contains null.
+        {_id: 10, no_a: 1},
+        {_id: 11, a: null},
+        {_id: 12, a: [null]},
+        {_id: 13, a: [null, 1]},
+
+        // "a" doesn't contain neither empty array nor null.
+        {_id: 20, a: 1},
+        {_id: 21, a: [[[], 1], 2]},
+        {_id: 22, a: [1, 2]},
+        {_id: 23, a: [[null, 1], 2]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Empty top-level array in foreign, top field in local",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: []},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Empty top-level array in local, top field in foreign",
+        localRecord: {_id: 0, b: []},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [/*SERVER-63368: */ 2, 10, 11, 12, 13]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Empty nested array in foreign, top field in local",
+        localRecords: docs,
+        localField: "a",
+        foreignRecord: {_id: 0, b: [[], 42]},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Empty nested array in local, top field in foreign",
+        localRecord: {_id: 0, b: [[], 42]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a",
+        idsExpectedToMatch: [0, 1, 2, 3]
+    });
+})();
+
+(function testMatchingEmptyArrayOnPath() {
+    const docs = [
+        // For these docs "a.x" resolves to a (logical) set that contains empty array as a value.
+        {_id: 0, a: {x: [[]], y: 1}},
+        {_id: 1, a: [{x: [[]]}]},
+        {_id: 2, a: [{x: [[]]}, {x: 1}]},
+        {_id: 3, a: [{x: [[]]}, {x: null}]},
+        {_id: 4, a: [{x: [[]]}, {no_x: 1}]},
+        {_id: 5, a: {x: [[], 1]}},
+
+        // For these docs "a.x" resolves to a (logical) set that contains empty array as a value in
+        // foreign collection only.
+        {_id: 10, a: {x: [], y: 1}},
+        {_id: 11, a: [{x: []}, {x: 1}]},
+        {_id: 12, a: [{x: []}, {x: null}]},
+        {_id: 13, a: [{x: []}, {no_x: 1}]},
+
+        // For these docs "a.x" key is either missing or contains null.
+        {_id: 20, no_a: 1},
+        {_id: 21, a: {no_x: 1}},
+        {_id: 22, a: [{no_x: 1}, {no_x: 2}]},
+        {_id: 23, a: [{x: null}, {x: 1}]},
+
+        // SERVER-64006
+        {_id: 30, a: []},
+        {_id: 31, a: [1]},
+        {_id: 32, a: [null, 1]},
+        {_id: 33, a: [[]]},
+        {_id: 34, a: [[1], 2]},
+
+        // "a.x" doesn't contain neither empty array nor null.
+        {_id: 40, a: {x: 1}},
+        {_id: 41, a: {x: [[[], 1], 2]}},
+        {_id: 42, a: {x: [1, 2]}},
+        {_id: 43, a: {x: [[null, 1], 2]}},
+        {_id: 44, a: [{x: 1}]},
+        {_id: 45, a: [{x: [[[], 1], 2]}]},
+        {_id: 46, a: [{x: [1, 2]}]},
+        {_id: 47, a: [{x: [[null, 1], 2]}]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Empty top-level array in foreign, path in local",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: []},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Empty top-level array in local, path in foreign",
+        localRecord: {_id: 0, b: []},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [3, 4, 12, 13, 20, 21, 22, 23]
+    });
+
+    runTest_SingleForeignRecord({
+        testDescription: "Empty nested array in foreign, path in local",
+        localRecords: docs,
+        localField: "a.x",
+        foreignRecord: {_id: 0, b: [[], 42]},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Empty nested array in local, path in foreign",
+        localRecord: {_id: 0, b: [[], 42]},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.x",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5, 10, 11, 12, 13]
+    });
+})();
+
+(function testMatchingPathWithNumericComponentToScalar() {
+    const docs = [
+        // For these docs "a.0.x" resolves to a (logical) set that contains value "1".
+        {_id: 0, a: [{x: 1}, {x: 2}]},
+        {_id: 1, a: [{x: [2, 3, 1]}]},
+        {_id: 2, a: [{x: 1}, {y: 1}]},
+
+        // For these docs "a.0.x" resolves to a (logical) set that does _not_ contain value "1".
+        {_id: 10, a: [{x: 2}, {x: 1}]},
+        {_id: 11, a: [{x: [2, 3]}]},
+        {_id: 12, a: {x: 1}},
+        {_id: 13, a: {x: [1, 2]}},
+        {_id: 14, a: [{y: 1}, {x: 1}]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Scalar in foreign, path with numeral in local",
+        localRecords: docs,
+        localField: "a.0.x",
+        foreignRecord: {_id: 0, b: 1},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Scalar in local, path with numeral in foreign",
+        localRecord: {_id: 0, b: 1},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.0.x",
+        idsExpectedToMatch: [0, 1, 2]
+    });
+})();
+
+(function testMatchingPathWithNumericComponentToNull() {
+    const docs = [
+        // For these docs "a.0.x" resolves to a (logical) set that contains value "null".
+        {_id: 0, a: {x: 1}},
+        {_id: 1, a: {x: [1, 2]}},
+        {_id: 2, a: [{y: 1}, {x: 1}]},
+        {_id: 3, a: [{x: null}, {x: 1}]},
+        {_id: 4, a: [{x: [1, null]}, {x: 1}]},
+        {_id: 5, a: [1, 2]},
+
+        // For these docs "a.0.x" resolves to a (logical) set that does _not_ contain value "null".
+        {_id: 10, a: [{x: 1}, {y: 1}]},
+        {_id: 11, a: [{x: [1, 2]}]},
+    ];
+
+    runTest_SingleForeignRecord({
+        testDescription: "Null in foreign, path with numeral in local",
+        localRecords: docs,
+        localField: "a.0.x",
+        foreignRecord: {_id: 0, b: null},
+        foreignField: "b",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, 5]
+    });
+    runTest_SingleLocalRecord({
+        testDescription: "Null in local, path with numeral in foreign",
+        localRecord: {_id: 0, b: null},
+        localField: "b",
+        foreignRecords: docs,
+        foreignField: "a.0.x",
+        idsExpectedToMatch: [0, 1, 2, 3, 4, /*SERVER-64006: 5,*/ /*SERVER-64221: */ 10, 11]
+    });
+})();
+}());
diff --git a/jstests/aggregation/sources/lookup/lookup_null_semantics.js b/jstests/aggregation/sources/lookup/lookup_null_semantics.js
deleted file mode 100644
index 38067839db2..00000000000
--- a/jstests/aggregation/sources/lookup/lookup_null_semantics.js
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Test that $lookup behaves correctly for null values, as well as "missing" and undefined.
- */
-(function() {
-"use strict";
-
-load("jstests/aggregation/extras/utils.js");
-load("jstests/libs/fixture_helpers.js");  // For isSharded.
-
-const localColl = db.lookup_null_semantics_local;
-const foreignColl = db.lookup_null_semantics_foreign;
-
-localColl.drop();
-foreignColl.drop();
-
-// Do not run the rest of the tests if the foreign collection is implicitly sharded but the flag to
-// allow $lookup/$graphLookup into a sharded collection is disabled.
-const getShardedLookupParam = db.adminCommand({getParameter: 1, featureFlagShardedLookup: 1});
-const isShardedLookupEnabled = getShardedLookupParam.hasOwnProperty("featureFlagShardedLookup") &&
-    getShardedLookupParam.featureFlagShardedLookup.value;
-if (FixtureHelpers.isSharded(foreignColl) && !isShardedLookupEnabled) {
-    return;
-}
-
-assert.commandWorked(localColl.insert([
-    {_id: 0},
-    {_id: 1, a: null},
-    {_id: 2, a: 9},
-    {_id: 3, a: [null, 9]},
-]));
-assert.commandWorked(foreignColl.insert([
-    {_id: 0},
-    {_id: 1, b: null},
-    {_id: 2, b: undefined},
-    {_id: 3, b: 9},
-    {_id: 4, b: [null, 9]},
-    {_id: 5, b: 42},
-    {_id: 6, b: [undefined, 9]},
-    {_id: 7, b: [9, 10]},
-]));
-
-const actualResults = localColl.aggregate([{
-    $lookup: {from: foreignColl.getName(), localField: "a", foreignField: "b", as: "lookupResults"}
-}]).toArray();
-
-const expectedResults = [
-    // Missing on the left-hand side results in a lookup with $eq:null semantics on the left-hand
-    // side. Namely, we expect this document to join with null, missing, and undefined, or arrays
-    // thereof.
-    {
-        _id: 0,
-        lookupResults: [
-            {_id: 0},
-            {_id: 1, b: null},
-            {_id: 2, b: undefined},
-            {_id: 4, b: [null, 9]},
-            {_id: 6, b: [undefined, 9]}
-        ]
-    },
-    // Null on the left-hand side is the same as the missing case above.
-    {
-        _id: 1,
-        a: null,
-        lookupResults: [
-            {_id: 0},
-            {_id: 1, b: null},
-            {_id: 2, b: undefined},
-            {_id: 4, b: [null, 9]},
-            {_id: 6, b: [undefined, 9]}
-        ]
-    },
-    // A "negative" test-case where the value being looked up is not nullish.
-    {
-        _id: 2,
-        a: 9,
-        lookupResults: [
-            {_id: 3, b: 9},
-            {_id: 4, b: [null, 9]},
-            {_id: 6, b: [undefined, 9]},
-            {_id: 7, b: [9, 10]},
-        ]
-    },
-    // Here we are looking up both null and a scalar. We expected missing, null, and undefined to
-    // match in addition to the matches due to the scalar.
-    {
-        _id: 3,
-        a: [null, 9],
-        lookupResults: [
-            {_id: 0},
-            {_id: 1, b: null},
-            {_id: 2, b: undefined},
-            {_id: 3, b: 9},
-            {_id: 4, b: [null, 9]},
-            {_id: 6, b: [undefined, 9]},
-            {_id: 7, b: [9, 10]},
-        ]
-    },
-];
-
-assertArrayEq({actual: actualResults, expected: expectedResults});
-
-// The results should not change there is an index available on the right-hand side.
-assert.commandWorked(foreignColl.createIndex({b: 1}));
-assertArrayEq({actual: actualResults, expected: expectedResults});
-
-// If the left-hand side collection has a value of undefined for "localField", then the query will
-// fail. This is a consequence of the fact that queries which explicitly compare to undefined, such
-// as {$eq:undefined}, are banned. Arguably this behavior could be improved, but we are unlikely to
-// change it given that the undefined BSON type has been deprecated for many years.
-assert.commandWorked(localColl.insert({a: undefined}));
-assert.throws(() => {
-    localColl.aggregate([{
-    $lookup: {from: foreignColl.getName(), localField: "a", foreignField: "b", as: "lookupResults"}
-}]).toArray();
-});
-}());