SERVER-30731 Add expr support in MatchExpression outside of aggregation

9 files changed, 346 insertions, 8 deletions

diff --git a/jstests/core/collation.js b/jstests/core/collation.js
index ac89db598b6..98da1ba8ebf 100644
--- a/jstests/core/collation.js
+++ b/jstests/core/collation.js
@@ -646,6 +646,14 @@
                       .collation({locale: "en_US", strength: 3})
                       .sort({a: 1});
         assert.eq(res.toArray(), [{a: "a"}, {a: "A"}, {a: "b"}, {a: "B"}]);
+
+        // Find should return correct results when collation specified and query contains $expr.
+        coll.drop();
+        assert.writeOK(coll.insert([{a: "A"}, {a: "B"}]));
+        assert.eq(1,
+                  coll.find({$expr: {$eq: ["$a", "a"]}})
+                      .collation({locale: "en_US", strength: 2})
+                      .itcount());
     }
 
     // Find should return correct results when no collation specified and collection has a default
@@ -692,6 +700,14 @@
                   .addOption(DBQuery.Option.oplogReplay)
                   .itcount());
 
+    // Find should return correct results for query containing $expr when no collation specified and
+    // collection has a default collation.
+    coll.drop();
+    assert.commandWorked(
+        db.createCollection(coll.getName(), {collation: {locale: "en_US", strength: 2}}));
+    assert.writeOK(coll.insert([{a: "A"}, {a: "B"}]));
+    assert.eq(1, coll.find({$expr: {$eq: ["$a", "a"]}}).itcount());
+
     if (db.getMongo().useReadCommands()) {
         // Find should return correct results when "simple" collation specified and collection has a
         // default collation.
diff --git a/jstests/core/doc_validation.js b/jstests/core/doc_validation.js
index 7a3dc5f038d..e1d02bdba9b 100644
--- a/jstests/core/doc_validation.js
+++ b/jstests/core/doc_validation.js
@@ -98,4 +98,22 @@
     assert.writeOK(coll.update({_id: 'invalid2'}, {$set: {a: 1}}));
     coll.drop();
 
+    // The validator is allowed to contain $expr.
+    assert.commandWorked(db.createCollection(collName, {validator: {$expr: {$eq: ["$a", 5]}}}));
+    assert.writeOK(coll.insert({a: 5}));
+    assertFailsValidation(coll.insert({a: 4}));
+    assert.commandWorked(
+        db.runCommand({"collMod": collName, "validator": {$expr: {$eq: ["$a", 4]}}}));
+    assert.writeOK(coll.insert({a: 4}));
+    assertFailsValidation(coll.insert({a: 5}));
+
+    // The validator can contain an $expr that may throw at runtime.
+    coll.drop();
+    assert.commandWorked(
+        db.createCollection(collName, {validator: {$expr: {$eq: ["$a", {$divide: [1, "$b"]}]}}}));
+    assert.writeOK(coll.insert({a: 1, b: 1}));
+    let res = coll.insert({a: 1, b: 0});
+    assert.writeError(res);
+    assert.eq(res.getWriteError().code, 16608);
+    assert.writeOK(coll.insert({a: -1, b: -1}));
 })();
diff --git a/jstests/core/doc_validation_invalid_validators.js b/jstests/core/doc_validation_invalid_validators.js
index e56977ac8e1..3cedaf0d537 100644
--- a/jstests/core/doc_validation_invalid_validators.js
+++ b/jstests/core/doc_validation_invalid_validators.js
@@ -22,6 +22,8 @@
     assert.commandFailed(db.createCollection(collName, {validator: {$geoNear: {place: "holder"}}}));
     assert.commandFailed(
         db.createCollection(collName, {validator: {$nearSphere: {place: "holder"}}}));
+    assert.commandFailed(
+        db.createCollection(collName, {validator: {$expr: {$eq: ["$a", "$$unbound"]}}}));
 
     // Verify we fail on admin, local and config databases.
     assert.commandFailed(
@@ -47,9 +49,8 @@
         db.runCommand({"collMod": collName, "validator": {$geoNear: {place: "holder"}}}));
     assert.commandFailed(
         db.runCommand({"collMod": collName, "validator": {$nearSphere: {place: "holder"}}}));
-
     assert.commandFailed(
-        db.runCommand({"collMod": collName, "validator": {$expr: {$eq: ["$a", 5]}}}));
+        db.runCommand({"collMod": collName, "validator": {$expr: {$eq: ["$a", "$$unbound"]}}}));
 
     coll.drop();
 
diff --git a/jstests/core/expr.js b/jstests/core/expr.js
new file mode 100644
index 00000000000..d2c6d9216ff
--- /dev/null
+++ b/jstests/core/expr.js
@@ -0,0 +1,239 @@
+// Tests for $expr in the CRUD commands.
+(function() {
+    "use strict";
+
+    const coll = db.expr;
+
+    const isMaster = db.runCommand("ismaster");
+    assert.commandWorked(isMaster);
+    const isMongos = (isMaster.msg === "isdbgrid");
+
+    //
+    // $expr in aggregate.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({a: 5}));
+    assert.eq(1, coll.aggregate([{$match: {$expr: {$eq: ["$a", 5]}}}]).itcount());
+    assert.throws(function() {
+        coll.aggregate([{$match: {$expr: {$eq: ["$a", "$$unbound"]}}}]);
+    });
+
+    //
+    // $expr in count.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({a: 5}));
+    assert.eq(1, coll.find({$expr: {$eq: ["$a", 5]}}).count());
+    assert.throws(function() {
+        coll.find({$expr: {$eq: ["$a", "$$unbound"]}}).count();
+    });
+
+    //
+    // $expr in distinct.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({a: 5}));
+    assert.eq(1, coll.distinct("a", {$expr: {$eq: ["$a", 5]}}).length);
+    assert.throws(function() {
+        coll.distinct("a", {$expr: {$eq: ["$a", "$$unbound"]}});
+    });
+
+    //
+    // $expr in find.
+    //
+
+    // $expr is allowed in query.
+    coll.drop();
+    assert.writeOK(coll.insert({a: 5}));
+    assert.eq(1, coll.find({$expr: {$eq: ["$a", 5]}}).itcount());
+
+    // $expr with unbound variable throws.
+    assert.throws(function() {
+        coll.find({$expr: {$eq: ["$a", "$$unbound"]}}).itcount();
+    });
+
+    // $expr is allowed in find with explain.
+    assert.commandWorked(coll.find({$expr: {$eq: ["$a", 5]}}).explain());
+
+    // $expr with unbound variable in find with explain throws.
+    assert.throws(function() {
+        coll.find({$expr: {$eq: ["$a", "$$unbound"]}}).explain();
+    });
+
+    // $expr is not allowed in $elemMatch projection.
+    coll.drop();
+    assert.writeOK(coll.insert({a: [{b: 5}]}));
+    assert.throws(function() {
+        coll.find({}, {a: {$elemMatch: {$expr: {$eq: ["$b", 5]}}}}).itcount();
+    });
+
+    //
+    // $expr in findAndModify.
+    //
+
+    // $expr is allowed in the query when upsert=false.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: 5}));
+    assert.eq({_id: 0, a: 5, b: 6},
+              coll.findAndModify(
+                  {query: {_id: 0, $expr: {$eq: ["$a", 5]}}, update: {$set: {b: 6}}, new: true}));
+
+    // $expr with unbound variable throws.
+    assert.throws(function() {
+        coll.findAndModify(
+            {query: {_id: 0, $expr: {$eq: ["$a", "$$unbound"]}}, update: {$set: {b: 6}}});
+    });
+
+    // $expr is not allowed in the query when upsert=true.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: 5}));
+    assert.throws(function() {
+        coll.findAndModify(
+            {query: {_id: 0, $expr: {$eq: ["$a", 5]}}, update: {$set: {b: 6}}, upsert: true});
+    });
+
+    // $expr is not allowed in $pull filter.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: [{b: 5}]}));
+    assert.throws(function() {
+        coll.findAndModify({query: {_id: 0}, update: {$pull: {a: {$expr: {$eq: ["$b", 5]}}}}});
+    });
+
+    // $expr is not allowed in arrayFilters.
+    if (db.getMongo().writeMode() === "commands") {
+        coll.drop();
+        assert.writeOK(coll.insert({_id: 0, a: [{b: 5}]}));
+        assert.throws(function() {
+            coll.findAndModify({
+                query: {_id: 0},
+                update: {$set: {"a.$[i].b": 6}},
+                arrayFilters: [{"i.b": 5, $expr: {$eq: ["$i.b", 5]}}]
+            });
+        });
+    }
+
+    //
+    // $expr in geoNear.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({geo: {type: "Point", coordinates: [0, 0]}, a: 5}));
+    assert.commandWorked(coll.ensureIndex({geo: "2dsphere"}));
+    assert.eq(1,
+              assert
+                  .commandWorked(db.runCommand({
+                      geoNear: coll.getName(),
+                      near: {type: "Point", coordinates: [0, 0]},
+                      spherical: true,
+                      query: {$expr: {$eq: ["$a", 5]}}
+                  }))
+                  .results.length);
+    assert.commandFailed(db.runCommand({
+        geoNear: coll.getName(),
+        near: {type: "Point", coordinates: [0, 0]},
+        spherical: true,
+        query: {$expr: {$eq: ["$a", "$$unbound"]}}
+    }));
+
+    //
+    // $expr in group.
+    //
+
+    // The group command is not permitted in sharded collections.
+    if (!isMongos) {
+        coll.drop();
+        assert.writeOK(coll.insert({a: 5}));
+        assert.eq([{a: 5, count: 1}], coll.group({
+            cond: {$expr: {$eq: ["$a", 5]}},
+            key: {a: 1},
+            initial: {count: 0},
+            reduce: function(curr, result) {
+                result.count += 1;
+            }
+        }));
+        assert.throws(function() {
+            coll.group({
+                cond: {$expr: {$eq: ["$a", "$$unbound"]}},
+                key: {a: 1},
+                initial: {count: 0},
+                reduce: function(curr, result) {
+                    result.count += 1;
+                }
+            });
+        });
+    }
+
+    //
+    // $expr in mapReduce.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({a: 5}));
+    let mapReduceOut = coll.mapReduce(
+        function() {
+            emit(this.a, 1);
+        },
+        function(key, values) {
+            return Array.sum(values);
+        },
+        {out: {inline: 1}, query: {$expr: {$eq: ["$a", 5]}}});
+    assert.commandWorked(mapReduceOut);
+    assert.eq(mapReduceOut.results.length, 1, tojson(mapReduceOut));
+    assert.throws(function() {
+        coll.mapReduce(
+            function() {
+                emit(this.a, 1);
+            },
+            function(key, values) {
+                return Array.sum(values);
+            },
+            {out: {inline: 1}, query: {$expr: {$eq: ["$a", "$$unbound"]}}});
+    });
+
+    //
+    // $expr in remove.
+    //
+
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: 5}));
+    let writeRes = coll.remove({_id: 0, $expr: {$eq: ["$a", 5]}});
+    assert.writeOK(writeRes);
+    assert.eq(1, writeRes.nRemoved);
+    assert.writeError(coll.remove({_id: 0, $expr: {$eq: ["$a", "$$unbound"]}}));
+
+    //
+    // $expr in update.
+    //
+
+    // $expr is allowed in the query when upsert=false.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: 5}));
+    assert.writeOK(coll.update({_id: 0, $expr: {$eq: ["$a", 5]}}, {$set: {b: 6}}));
+    assert.eq({_id: 0, a: 5, b: 6}, coll.findOne({_id: 0}));
+
+    // $expr with unbound variable fails.
+    assert.writeError(coll.update({_id: 0, $expr: {$eq: ["$a", "$$unbound"]}}, {$set: {b: 6}}));
+
+    // $expr is not allowed in the query when upsert=true.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: 5}));
+    assert.writeError(
+        coll.update({_id: 0, $expr: {$eq: ["$a", 5]}}, {$set: {b: 6}}, {upsert: true}));
+
+    // $expr is not allowed in $pull filter.
+    coll.drop();
+    assert.writeOK(coll.insert({_id: 0, a: [{b: 5}]}));
+    assert.writeError(coll.update({_id: 0}, {$pull: {a: {$expr: {$eq: ["$b", 5]}}}}));
+
+    // $expr is not allowed in arrayFilters.
+    if (db.getMongo().writeMode() === "commands") {
+        coll.drop();
+        assert.writeOK(coll.insert({_id: 0, a: [{b: 5}]}));
+        assert.writeError(coll.update({_id: 0},
+                                      {$set: {"a.$[i].b": 6}},
+                                      {arrayFilters: [{"i.b": 5, $expr: {$eq: ["$i.b", 5]}}]}));
+    }
+})();
diff --git a/jstests/core/index_filter_commands.js b/jstests/core/index_filter_commands.js
index d0973aacd69..644707fa199 100644
--- a/jstests/core/index_filter_commands.js
+++ b/jstests/core/index_filter_commands.js
@@ -234,3 +234,42 @@ assert.commandWorked(
 
 explain = t.find(queryAA).collation(collationEN).explain();
 assert(isCollscan(explain.queryPlanner.winningPlan), "Expected collscan: " + tojson(explain));
+
+//
+// Test that planCacheSetFilter and planCacheClearFilters allow queries containing $expr.
+//
+
+t.drop();
+assert.writeOK(t.insert({a: "a"}));
+assert.commandWorked(t.createIndex(indexA1, {name: "a_1"}));
+
+assert.commandWorked(t.runCommand(
+    "planCacheSetFilter", {query: {a: "a", $expr: {$eq: ["$a", "a"]}}, indexes: [indexA1]}));
+filters = getFilters();
+assert.eq(1, filters.length, tojson(filters));
+assert.eq({a: "a", $expr: {$eq: ["$a", "a"]}}, filters[0].query, tojson(filters[0]));
+
+assert.commandWorked(
+    t.runCommand("planCacheClearFilters", {query: {a: "a", $expr: {$eq: ["$a", "a"]}}}));
+filters = getFilters();
+assert.eq(0, filters.length, tojson(filters));
+
+//
+// Test that planCacheSetFilter and planCacheClearFilters do not allow queries containing $expr with
+// unbound variables.
+//
+
+t.drop();
+assert.writeOK(t.insert({a: "a"}));
+assert.commandWorked(t.createIndex(indexA1, {name: "a_1"}));
+
+assert.commandFailed(
+    t.runCommand("planCacheSetFilter",
+                 {query: {a: "a", $expr: {$eq: ["$a", "$$unbound"]}}, indexes: [indexA1]}));
+filters = getFilters();
+assert.eq(0, filters.length, tojson(filters));
+
+assert.commandFailed(
+    t.runCommand("planCacheClearFilters", {query: {a: "a", $expr: {$eq: ["$a", "$$unbound"]}}}));
+filters = getFilters();
+assert.eq(0, filters.length, tojson(filters));
diff --git a/jstests/core/index_partial_create_drop.js b/jstests/core/index_partial_create_drop.js
index 5875093c706..f3bc37efbd9 100644
--- a/jstests/core/index_partial_create_drop.js
+++ b/jstests/core/index_partial_create_drop.js
@@ -32,6 +32,8 @@
         partialFilterExpression:
             {$and: [{$and: [{x: {$lt: 2}}, {x: {$gt: 0}}]}, {x: {$exists: true}}]}
     }));
+    assert.commandFailed(
+        coll.createIndex({x: 1}, {partialFilterExpression: {$expr: {$eq: ["$x", 5]}}}));
 
     for (var i = 0; i < 10; i++) {
         assert.writeOK(coll.insert({x: i, a: i}));
diff --git a/jstests/core/list_collections_filter.js b/jstests/core/list_collections_filter.js
index 41d80d0a344..defd5c62bc0 100644
--- a/jstests/core/list_collections_filter.js
+++ b/jstests/core/list_collections_filter.js
@@ -84,6 +84,14 @@
     },
                         []);
 
+    // Filter with $expr.
+    testListCollections({$expr: {$eq: ["$name", "lists"]}}, ["lists"]);
+
+    // Filter with $expr with an unbound variable.
+    assert.throws(function() {
+        mydb.getCollectionInfos({$expr: {$eq: ["$name", "$$unbound"]}});
+    });
+
     // No extensions are allowed in filters.
     assert.throws(function() {
         mydb.getCollectionInfos({$text: {$search: "str"}});
@@ -99,8 +107,4 @@
         mydb.getCollectionInfos(
             {a: {$nearSphere: {$geometry: {type: "Point", coordinates: [0, 0]}}}});
     });
-
-    assert.throws(function() {
-        mydb.getCollectionInfos({$expr: {$eq: ["$a", 5]}});
-    });
 }());
diff --git a/jstests/core/list_databases.js b/jstests/core/list_databases.js
index ecc746ce775..a52cfec656b 100644
--- a/jstests/core/list_databases.js
+++ b/jstests/core/list_databases.js
@@ -62,6 +62,16 @@
     assert.eq(1, cmdRes.databases.length, tojson(cmdRes));
     verifyNameOnly(cmdRes);
 
+    // $expr in filter.
+    cmdRes = assert.commandWorked(db.adminCommand(
+        {listDatabases: 1, filter: {$expr: {$eq: ["$name", "jstest_list_databases_zap"]}}}));
+    assert.eq(1, cmdRes.databases.length, tojson(cmdRes));
+    assert.eq("jstest_list_databases_zap", cmdRes.databases[0].name, tojson(cmdRes));
+
+    // $expr with an unbound variable in filter.
+    assert.commandFailed(
+        db.adminCommand({listDatabases: 1, filter: {$expr: {$eq: ["$name", "$$unbound"]}}}));
+
     // No extensions are allowed in filters.
     assert.commandFailed(db.adminCommand({listDatabases: 1, filter: {$text: {$search: "str"}}}));
     assert.commandFailed(db.adminCommand({
@@ -76,6 +86,4 @@
         listDatabases: 1,
         filter: {a: {$nearSphere: {$geometry: {type: "Point", coordinates: [0, 0]}}}}
     }));
-
-    assert.commandFailed(db.adminCommand({listDatabases: 1, filter: {$expr: {$eq: ["$a", 5]}}}));
 }());
diff --git a/jstests/core/plan_cache_clear.js b/jstests/core/plan_cache_clear.js
index 8f9cf0ea302..bb44ecab549 100644
--- a/jstests/core/plan_cache_clear.js
+++ b/jstests/core/plan_cache_clear.js
@@ -37,6 +37,17 @@ assert.eq(1, getShapes().length, 'unexpected cache size after running 2nd query'
 assert.commandWorked(t.runCommand('planCacheClear', {query: {a: 1, b: 1}}));
 assert.eq(0, getShapes().length, 'unexpected cache size after dropping 2nd query from cache');
 
+// planCacheClear can clear $expr queries.
+assert.eq(1, t.find({a: 1, b: 1, $expr: {$eq: ['$a', 1]}}).itcount(), 'unexpected document count');
+assert.eq(1, getShapes().length, 'unexpected cache size after running 2nd query');
+assert.commandWorked(
+    t.runCommand('planCacheClear', {query: {a: 1, b: 1, $expr: {$eq: ['$a', 1]}}}));
+assert.eq(0, getShapes().length, 'unexpected cache size after dropping 2nd query from cache');
+
+// planCacheClear fails with an $expr query with an unbound variable.
+assert.commandFailed(
+    t.runCommand('planCacheClear', {query: {a: 1, b: 1, $expr: {$eq: ['$a', '$$unbound']}}}));
+
 // Insert two more shapes into the cache.
 assert.eq(1, t.find({a: 1, b: 1}).itcount(), 'unexpected document count');
 assert.eq(1, t.find({a: 1, b: 1}, {_id: 0, a: 1}).itcount(), 'unexpected document count');