diff options
18 files changed, 1047 insertions, 41 deletions
diff --git a/jstests/aggregation/sources/graphLookup/error.js b/jstests/aggregation/sources/graphLookup/error.js index d05313200ca..42d1203238c 100644 --- a/jstests/aggregation/sources/graphLookup/error.js +++ b/jstests/aggregation/sources/graphLookup/error.js @@ -233,7 +233,7 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". restrictSearchWithMatch: {$not: {a: 1}} } }; - assertErrorCode(local, pipeline, 40186, "unable to parse match expression"); + assert.throws(() => local.aggregate(pipeline), [], "unable to parse match expression"); // $where and $text cannot be used inside $graphLookup. pipeline = { @@ -246,7 +246,7 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". restrictSearchWithMatch: {$where: "3 > 2"} } }; - assertErrorCode(local, pipeline, 40186, "cannot use $where inside $graphLookup"); + assert.throws(() => local.aggregate(pipeline), [], "cannot use $where inside $graphLookup"); pipeline = { $graphLookup: { @@ -258,7 +258,7 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". restrictSearchWithMatch: {$text: {$search: "some text"}} } }; - assertErrorCode(local, pipeline, 40186, "cannot use $text inside $graphLookup"); + assert.throws(() => local.aggregate(pipeline), [], "cannot use $text inside $graphLookup"); pipeline = { $graphLookup: { @@ -272,7 +272,7 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". } } }; - assertErrorCode(local, pipeline, 40186, "cannot use $near inside $graphLookup"); + assert.throws(() => local.aggregate(pipeline), [], "cannot use $near inside $graphLookup"); pipeline = { $graphLookup: { @@ -293,7 +293,8 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". } } }; - assertErrorCode(local, pipeline, 40186, "cannot use $near inside $graphLookup at any depth"); + assert.throws( + () => local.aggregate(pipeline), [], "cannot use $near inside $graphLookup at any depth"); let foreign = db.foreign; foreign.drop(); @@ -310,7 +311,7 @@ load("jstests/aggregation/extras/utils.js"); // For "assertErrorCode". restrictSearchWithMatch: {$expr: {$eq: ["$x", "$$unbound"]}} } }; - assertErrorCode(local, pipeline, 40186, "cannot use $expr with unbound variable"); + assert.throws(() => local.aggregate(pipeline), [], "cannot use $expr with unbound variable"); // Test a restrictSearchWithMatchExpression that throws at runtime. pipeline = { diff --git a/jstests/core/index_partial_create_drop.js b/jstests/core/index_partial_create_drop.js index f3bc37efbd9..106a658c0f6 100644 --- a/jstests/core/index_partial_create_drop.js +++ b/jstests/core/index_partial_create_drop.js @@ -32,8 +32,11 @@ partialFilterExpression: {$and: [{$and: [{x: {$lt: 2}}, {x: {$gt: 0}}]}, {x: {$exists: true}}]} })); + // Use of $expr is banned in a partial index filter. assert.commandFailed( coll.createIndex({x: 1}, {partialFilterExpression: {$expr: {$eq: ["$x", 5]}}})); + assert.commandFailed(coll.createIndex( + {x: 1}, {partialFilterExpression: {$expr: {$eq: [{$trim: {input: "$x"}}, "hi"]}}})); for (var i = 0; i < 10; i++) { assert.writeOK(coll.insert({x: i, a: i})); diff --git a/jstests/multiVersion/collection_validator_feature_compatibility_version.js b/jstests/multiVersion/collection_validator_feature_compatibility_version.js new file mode 100644 index 00000000000..046ea6c55be --- /dev/null +++ b/jstests/multiVersion/collection_validator_feature_compatibility_version.js @@ -0,0 +1,285 @@ +/** + * Test that mongod will not allow creation of collection validators using 4.0 query features when + * the feature compatibility version is older than 4.0. + * + * TODO SERVER-33321: Remove FCV 3.6 validation during the 4.1 development cycle. + * + * We restart mongod during the test and expect it to have the same data after restarting. + * @tags: [requires_persistence] + */ + +(function() { + "use strict"; + + const testName = "collection_validator_feature_compatibility_version"; + const dbpath = MongoRunner.dataPath + testName; + + // In order to avoid restarting the server for each test case, we declare all the test cases up + // front, and test them all at once. + const testCases = [ + { + validator: {$expr: {$eq: [{$trim: {input: "$a"}}, "good"]}}, + nonMatchingDocument: {a: "bad"} + }, + { + validator: {$expr: {$eq: [{$ltrim: {input: "$a"}}, "good"]}}, + nonMatchingDocument: {a: "bad"} + }, + { + validator: {$expr: {$eq: [{$rtrim: {input: "$a"}}, "good"]}}, + nonMatchingDocument: {a: "bad"} + }, + { + validator: { + $expr: { + $eq: [ + // The 'format' option was added in 4.0. + {$dateFromString: {dateString: "2018-02-08", format: "$format"}}, + new Date("2018-02-08") + ] + } + }, + // Swap the month and day so it doesn't match. + nonMatchingDocument: {format: "%Y-%d-%m"} + }, + { + validator: { + $expr: { + $eq: [ + // The 'onNull' option was added in 4.0. + { + $dateFromString: + {dateString: "$dateString", onNull: new Date("1970-01-01")} + }, + new Date("2018-02-08") + ] + } + }, + nonMatchingDocument: {dateString: null} + }, + { + validator: { + $expr: { + $eq: [ + // The 'onError' option was added in 4.0. + { + $dateFromString: + {dateString: "$dateString", onError: new Date("1970-01-01")} + }, + new Date("2018-02-08") + ] + } + }, + nonMatchingDocument: {dateString: "Not a date"} + }, + { + validator: { + $expr: { + $eq: [ + // The 'onNull' option was added in 4.0. + {$dateToString: {date: "$date", format: "%Y-%m-%d", onNull: "null input"}}, + "2018-02-08" + ] + } + }, + nonMatchingDocument: {date: null} + }, + { + validator: { + $expr: { + $eq: [ + // The 'format' option was made optional in 4.0. + {$dateToString: {date: "$date"}}, + "2018-02-08T00:00:00.000Z" + ] + } + }, + nonMatchingDocument: {date: new Date("2018-02-07")} + }, + { + validator: {$expr: {$eq: [{$convert: {input: "$a", to: "int"}}, 2018]}}, + nonMatchingDocument: {a: "2017"} + }, + { + validator: {$expr: {$eq: [{$convert: {input: "$a", to: "int", onNull: 0}}, 2018]}}, + nonMatchingDocument: {a: null} + } + ]; + + let conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "latest"}); + assert.neq(null, conn, "mongod was unable to start up"); + + let testDB = conn.getDB(testName); + + let adminDB = conn.getDB("admin"); + + // Explicitly set feature compatibility version 4.0. + assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "4.0"})); + + testCases.forEach(function(test, i) { + // Create a collection with a validator using 4.0 query features. + const coll = testDB["coll" + i]; + assert.commandWorked( + testDB.createCollection(coll.getName(), {validator: test.validator}), + `Expected to be able to create collection with validator ${tojson(test.validator)}`); + + // The validator should cause this insert to fail. + assert.writeErrorWithCode( + coll.insert(test.nonMatchingDocument), + ErrorCodes.DocumentValidationFailure, + `Expected document ${tojson(test.nonMatchingDocument)} to fail validation for ` + + `collection with validator ${tojson(test.validator)}`); + + // Set a validator using 4.0 query features on an existing collection. + coll.drop(); + assert.commandWorked(testDB.createCollection(coll.getName())); + assert.commandWorked( + testDB.runCommand({collMod: coll.getName(), validator: test.validator}), + `Expected to be able to modify collection validator to be ${tojson(test.validator)}`); + + // Another failing update. + assert.writeErrorWithCode( + coll.insert(test.nonMatchingDocument), + ErrorCodes.DocumentValidationFailure, + `Expected document ${tojson(test.nonMatchingDocument)} to fail validation for ` + + `collection with validator ${tojson(test.validator)}`); + }); + + // Set the feature compatibility version to 3.6. + assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.6"})); + + testCases.forEach(function(test, i) { + // The validator is already in place, so it should still cause this insert to fail. + const coll = testDB["coll" + i]; + assert.writeErrorWithCode( + coll.insert(test.nonMatchingDocument), + ErrorCodes.DocumentValidationFailure, + `Expected document ${tojson(test.nonMatchingDocument)} to fail validation for ` + + `collection with validator ${tojson(test.validator)}`); + + // Trying to create a new collection with a validator using 4.0 query features should fail + // while feature compatibility version is 3.6. + let res = testDB.createCollection("other", {validator: test.validator}); + assert.commandFailedWithCode( + res, + ErrorCodes.QueryFeatureNotAllowed, + `Expected *not* to be able to create collection with validator ${tojson(test.validator)}`); + assert( + res.errmsg.match(/feature compatibility version/), + `Expected error message from createCollection with validator ` + + `${tojson(test.validator)} to reference 'feature compatibility version' but got: ` + + res.errmsg); + + // Trying to update a collection with a validator using 4.0 query features should also fail. + res = testDB.runCommand({collMod: coll.getName(), validator: test.validator}); + assert.commandFailedWithCode( + res, + ErrorCodes.QueryFeatureNotAllowed, + `Expected to be able to create collection with validator ${tojson(test.validator)}`); + assert( + res.errmsg.match(/feature compatibility version/), + `Expected error message from createCollection with validator ` + + `${tojson(test.validator)} to reference 'feature compatibility version' but got: ` + + res.errmsg); + }); + + MongoRunner.stopMongod(conn); + + // If we try to start up a 3.6 mongod, it will fail, because it will not be able to parse + // the validator using 4.0 query features. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "3.6", noCleanData: true}); + assert.eq( + null, conn, "mongod 3.6 started, even with a validator using 4.0 query features in place."); + + // Starting up a 4.0 mongod, however, should succeed, even though the feature compatibility + // version is still set to 3.6. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "latest", noCleanData: true}); + assert.neq(null, conn, "mongod was unable to start up"); + + adminDB = conn.getDB("admin"); + testDB = conn.getDB(testName); + + // And the validator should still work. + testCases.forEach(function(test, i) { + const coll = testDB["coll" + i]; + assert.writeErrorWithCode( + coll.insert(test.nonMatchingDocument), + ErrorCodes.DocumentValidationFailure, + `Expected document ${tojson(test.nonMatchingDocument)} to fail validation for ` + + `collection with validator ${tojson(test.validator)}`); + + // Remove the validator. + assert.commandWorked(testDB.runCommand({collMod: coll.getName(), validator: {}})); + }); + + MongoRunner.stopMongod(conn); + + // Now, we should be able to start up a 3.6 mongod. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "3.6", noCleanData: true}); + assert.neq( + null, + conn, + "mongod 3.6 failed to start, even after we removed the validator using 4.0 query features"); + + MongoRunner.stopMongod(conn); + + // The rest of the test uses mongod 4.0. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "latest", noCleanData: true}); + assert.neq(null, conn, "mongod was unable to start up"); + + adminDB = conn.getDB("admin"); + testDB = conn.getDB(testName); + + // Set the feature compatibility version back to 4.0. + assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "4.0"})); + + testCases.forEach(function(test, i) { + const coll = testDB["coll2" + i]; + + // Now we should be able to create a collection with a validator using 4.0 query features + // again. + assert.commandWorked( + testDB.createCollection(coll.getName(), {validator: test.validator}), + `Expected to be able to create collection with validator ${tojson(test.validator)}`); + + // And we should be able to modify a collection to have a validator using 4.0 query + // features. + assert.commandWorked( + testDB.runCommand({collMod: coll.getName(), validator: test.validator}), + `Expected to be able to modify collection validator to be ${tojson(test.validator)}`); + }); + + // Set the feature compatibility version to 3.6 and then restart with + // internalValidateFeaturesAsMaster=false. + assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: "3.6"})); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({ + dbpath: dbpath, + binVersion: "latest", + noCleanData: true, + setParameter: "internalValidateFeaturesAsMaster=false" + }); + assert.neq(null, conn, "mongod was unable to start up"); + + testDB = conn.getDB(testName); + + testCases.forEach(function(test, i) { + const coll = testDB["coll3" + i]; + // Even though the feature compatibility version is 3.6, we should still be able to add a + // validator using 4.0 query features, because internalValidateFeaturesAsMaster is false. + assert.commandWorked( + testDB.createCollection(coll.getName(), {validator: test.validator}), + `Expected to be able to create collection with validator ${tojson(test.validator)}`); + + // We should also be able to modify a collection to have a validator using 4.0 query + // features. + coll.drop(); + assert.commandWorked(testDB.createCollection(coll.getName())); + assert.commandWorked( + testDB.runCommand({collMod: coll.getName(), validator: test.validator}), + `Expected to be able to modify collection validator to be ${tojson(test.validator)}`); + }); + + MongoRunner.stopMongod(conn); + +}()); diff --git a/jstests/multiVersion/view_definition_feature_compatibility_version.js b/jstests/multiVersion/view_definition_feature_compatibility_version.js new file mode 100644 index 00000000000..290fef89f33 --- /dev/null +++ b/jstests/multiVersion/view_definition_feature_compatibility_version.js @@ -0,0 +1,248 @@ +/** + * Test that mongod will not allow creation of a view using 4.0 aggregation features when the + * feature compatibility version is older than 4.0. + * + * TODO SERVER-33321: Remove FCV 3.6 validation during the 4.1 development cycle. + * + * We restart mongod during the test and expect it to have the same data after restarting. + * @tags: [requires_persistence] + */ + +(function() { + "use strict"; + + const testName = "view_definition_feature_compatibility_version_multiversion"; + const dbpath = MongoRunner.dataPath + testName; + + // In order to avoid restarting the server for each test case, we declare all the test cases up + // front, and test them all at once. + const pipelinesWithNewFeatures = [ + [{$project: {trimmed: {$trim: {input: " hi "}}}}], + [{$project: {trimmed: {$ltrim: {input: " hi "}}}}], + [{$project: {trimmed: {$rtrim: {input: " hi "}}}}], + // The 'format' option was added in 4.0. + [{ + $project: { + dateFromStringWithFormat: + {$dateFromString: {dateString: "2018-02-08", format: "$format"}} + } + }], + // The 'onNull' option was added in 4.0. + [{ + $project: { + dateFromStringWithOnNull: { + $dateFromString: {dateString: "$dateString", onNull: new Date("1970-01-01")} + } + } + }], + // The 'onError' option was added in 4.0. + [{ + $project: { + dateFromStringWithOnError: { + $dateFromString: + {dateString: "$dateString", onError: new Date("1970-01-01")} + } + } + }], + // The 'onNull' option was added in 4.0. + [{ + $project: { + dateToStringWithOnNull: + {$dateToString: {date: "$date", format: "%Y-%m-%d", onNull: "null input"}} + } + }], + // The 'format' option was made optional in 4.0. + [{$project: {dateToStringWithoutFormat: {$dateToString: {date: "$date"}}}}], + [{$project: {conversion: {$convert: {input: "$a", to: "int"}}}}], + [{$project: {conversionWithOnNull: {$convert: {input: "$a", to: "int", onNull: 0}}}}], + // Test using one of the prohibited expressions inside of an $expr within a MatchExpression + // embedded in the pipeline. + [{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}], + [{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }], + [{$facet: {withinMatch: [{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}]}}], + [{ + $facet: { + withinGraphLookup: [{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: + {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }] + } + }], + [{ + $facet: { + withinMatch: [{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}], + withinGraphLookup: [{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: + {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }] + } + }] + ]; + + let conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "latest"}); + assert.neq(null, conn, "mongod was unable to start up"); + let testDB = conn.getDB(testName); + + // Explicitly set feature compatibility version 4.0. + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "4.0"})); + + // Test that we are able to create a new view with any of the new features. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandWorked( + testDB.createView("firstView" + i, "coll", pipe), + `Expected to be able to create view with pipeline ${tojson(pipe)} while in FCV 4.0`)); + + // Test that we are able to create a new view with any of the new features. + pipelinesWithNewFeatures.forEach(function(pipe, i) { + assert(testDB["firstView" + i].drop(), `Drop of view with pipeline ${tojson(pipe)} failed`); + assert.commandWorked(testDB.createView("firstView" + i, "coll", [])); + assert.commandWorked( + testDB.runCommand({collMod: "firstView" + i, viewOn: "coll", pipeline: pipe}), + `Expected to be able to modify view to use pipeline ${tojson(pipe)} while in FCV 4.0`); + }); + + // Create an empty view which we will attempt to update to use 4.0 query features under + // feature compatibility mode 3.6. + assert.commandWorked(testDB.createView("emptyView", "coll", [])); + + // Set the feature compatibility version to 3.6. + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.6"})); + + // Read against an existing view using 4.0 query features should not fail. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandWorked(testDB.runCommand({find: "firstView" + i}), + `Failed to query view with pipeline ${tojson(pipe)}`)); + + // Trying to create a new view using 4.0 query features should fail. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandFailedWithCode( + testDB.createView("view_fail" + i, "coll", pipe), + ErrorCodes.QueryFeatureNotAllowed, + `Expected *not* to be able to create view with pipeline ${tojson(pipe)} while in FCV 3.6`)); + + // Trying to update existing view to use 4.0 query features should also fail. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandFailedWithCode( + testDB.runCommand({collMod: "emptyView", viewOn: "coll", pipeline: pipe}), + ErrorCodes.QueryFeatureNotAllowed, + `Expected *not* to be able to modify view to use pipeline ${tojson(pipe)} while in FCV 3.6`)); + + MongoRunner.stopMongod(conn); + + // Starting up a 3.6 mongod with 4.0 query features will succeed. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "3.6", noCleanData: true}); + assert.neq(null, conn, "mongod 3.6 was unable to start up"); + testDB = conn.getDB(testName); + + // Reads will fail against views with 4.0 query features when running a 3.6 binary. + // Not checking the code returned on failure as it is not uniform across the various + // 'pipeline' arguments tested. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandFailed( + testDB.runCommand({find: "firstView" + i}), + `Expected read against view with pipeline ${tojson(pipe)} to fail on 3.6 binary`)); + + // Test that a read against a view that does not contain 4.0 query features succeeds. + assert.commandWorked(testDB.runCommand({find: "emptyView"})); + + MongoRunner.stopMongod(conn); + + // Starting up a 4.0 mongod should succeed, even though the feature compatibility version is + // still set to 3.6. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "latest", noCleanData: true}); + assert.neq(null, conn, "mongod was unable to start up"); + testDB = conn.getDB(testName); + + // Read against an existing view using 4.0 query features should not fail. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert.commandWorked(testDB.runCommand({find: "firstView" + i}), + `Failed to query view with pipeline ${tojson(pipe)}`)); + + // Set the feature compatibility version back to 4.0. + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "4.0"})); + + pipelinesWithNewFeatures.forEach(function(pipe, i) { + assert.commandWorked(testDB.runCommand({find: "firstView" + i}), + `Failed to query view with pipeline ${tojson(pipe)}`); + // Test that we are able to create a new view with any of the new features. + assert.commandWorked( + testDB.createView("secondView" + i, "coll", pipe), + `Expected to be able to create view with pipeline ${tojson(pipe)} while in FCV 4.0`); + + // Test that we are able to update an existing view to use any of the new features. + assert(testDB["secondView" + i].drop(), + `Drop of view with pipeline ${tojson(pipe)} failed`); + assert.commandWorked(testDB.createView("secondView" + i, "coll", [])); + assert.commandWorked( + testDB.runCommand({collMod: "secondView" + i, viewOn: "coll", pipeline: pipe}), + `Expected to be able to modify view to use pipeline ${tojson(pipe)} while in FCV 4.0`); + }); + + // Set the feature compatibility version to 3.6 and then restart with + // internalValidateFeaturesAsMaster=false. + assert.commandWorked(testDB.adminCommand({setFeatureCompatibilityVersion: "3.6"})); + MongoRunner.stopMongod(conn); + conn = MongoRunner.runMongod({ + dbpath: dbpath, + binVersion: "latest", + noCleanData: true, + setParameter: "internalValidateFeaturesAsMaster=false" + }); + assert.neq(null, conn, "mongod was unable to start up"); + testDB = conn.getDB(testName); + + pipelinesWithNewFeatures.forEach(function(pipe, i) { + // Even though the feature compatibility version is 3.6, we should still be able to create a + // view using 4.0 query features, because internalValidateFeaturesAsMaster is false. + assert.commandWorked( + testDB.createView("thirdView" + i, "coll", pipe), + `Expected to be able to create view with pipeline ${tojson(pipe)} while in FCV 3.6 ` + + `with internalValidateFeaturesAsMaster=false`); + + // We should also be able to modify a view to use 4.0 query features. + assert(testDB["thirdView" + i].drop(), `Drop of view with pipeline ${tojson(pipe)} failed`); + assert.commandWorked(testDB.createView("thirdView" + i, "coll", [])); + assert.commandWorked( + testDB.runCommand({collMod: "thirdView" + i, viewOn: "coll", pipeline: pipe}), + `Expected to be able to modify view to use pipeline ${tojson(pipe)} while in FCV 3.6 ` + + `with internalValidateFeaturesAsMaster=false`); + }); + + MongoRunner.stopMongod(conn); + + // Starting up a 3.6 mongod with 4.0 query features. + conn = MongoRunner.runMongod({dbpath: dbpath, binVersion: "3.6", noCleanData: true}); + assert.neq(null, conn, "mongod 3.6 was unable to start up"); + testDB = conn.getDB(testName); + + // Existing views with 4.0 query features can be dropped. + pipelinesWithNewFeatures.forEach( + (pipe, i) => assert(testDB["firstView" + i].drop(), + `Drop of view with pipeline ${tojson(pipe)} failed`)); + assert(testDB.system.views.drop(), "Drop of system.views collection failed"); + + MongoRunner.stopMongod(conn); +}()); diff --git a/jstests/replsets/collection_validator_initial_sync_with_feature_compatibility.js b/jstests/replsets/collection_validator_initial_sync_with_feature_compatibility.js new file mode 100644 index 00000000000..6a77143b378 --- /dev/null +++ b/jstests/replsets/collection_validator_initial_sync_with_feature_compatibility.js @@ -0,0 +1,145 @@ +/** + * Test that a new replica set member can successfully sync a collection with a validator using 4.0 + * aggregation features, even when the replica set was downgraded to feature compatibility version + * 3.6. + * + * TODO SERVER-33321: Remove FCV 3.6 validation during the 4.1 development cycle. + * + * We restart the secondary as part of this test with the expectation that it still has the same + * data after the restart. + * @tags: [requires_persistence] + */ + +load("jstests/replsets/rslib.js"); + +(function() { + "use strict"; + const testName = "collection_validator_initial_sync_with_feature_compatibility"; + + function testValidator(validator, nonMatchingDocument) { + // + // Create a single-node replica set. + // + let replTest = new ReplSetTest({name: testName, nodes: 1}); + + replTest.startSet(); + replTest.initiate(); + + let primary = replTest.getPrimary(); + let testDB = primary.getDB("test"); + + // + // Explicitly set the replica set to feature compatibility version 4.0. + // + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: "4.0"})); + + // + // Create a collection with a validator using 4.0 query features. + // + assert.commandWorked(testDB.createCollection("coll", {validator: validator})); + + // + // Downgrade the replica set to feature compatibility version 3.6. + // + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: "3.6"})); + + // + // Add a new member to the replica set. + // + let secondaryDBPath = MongoRunner.dataPath + testName + "_secondary"; + resetDbpath(secondaryDBPath); + let secondary = replTest.add({dbpath: secondaryDBPath}); + replTest.reInitiate(secondary); + reconnect(primary); + reconnect(secondary); + + // + // Once the new member completes its initial sync, stop it, remove it from the replica set, + // and start it back up as an individual instance. + // + replTest.waitForState(secondary, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); + + replTest.stopSet(undefined /* send default signal */, + true /* don't clear data directory */); + + secondary = MongoRunner.runMongod({dbpath: secondaryDBPath, noCleanData: true}); + assert.neq(null, secondary, "mongod was unable to start up"); + + // + // Verify that the validator synced to the new member by attempting to insert a document + // that does not validate and checking that the insert fails. + // + let secondaryDB = secondary.getDB("test"); + assert.writeError(secondaryDB.coll.insert(nonMatchingDocument), + ErrorCodes.DocumentValidationFailure); + + // + // Verify that, even though the existing validator still works, it is not possible to create + // a new validator using 4.0 query features because of feature compatibility version 3.6. + // + assert.commandFailedWithCode( + secondaryDB.runCommand({collMod: "coll", validator: validator}), + ErrorCodes.QueryFeatureNotAllowed); + + MongoRunner.stopMongod(secondary); + } + + // Ban the use of expressions that were introduced or had their parsing modified in 4.0. + testValidator({$expr: {$eq: [{$trim: {input: "$a"}}, "good"]}}, {a: "bad"}); + testValidator({$expr: {$eq: [{$ltrim: {input: "$a"}}, "good"]}}, {a: "bad"}); + testValidator({$expr: {$eq: [{$rtrim: {input: "$a"}}, "good"]}}, {a: "bad"}); + testValidator({ + $expr: { + $eq: [ + // The 'format' option was added in 4.0. + {$dateFromString: {dateString: "2018-02-08", format: "$format"}}, + new Date("2018-02-08") + ] + } + }, + // Swap the month and day so it doesn't match. + {format: "%Y-%d-%m"}); + testValidator({ + $expr: { + $eq: [ + // The 'onNull' option was added in 4.0. + {$dateFromString: {dateString: "$dateString", onNull: new Date("1970-01-01")}}, + new Date("2018-02-08") + ] + } + }, + {dateString: null}); + testValidator({ + $expr: { + $eq: [ + // The 'onError' option was added in 4.0. + {$dateFromString: {dateString: "$dateString", onError: new Date("1970-01-01")}}, + new Date("2018-02-08") + ] + } + }, + {dateString: "Not a date"}); + testValidator({ + $expr: { + $eq: [ + // The 'onNull' option was added in 4.0. + {$dateToString: {date: "$date", format: "%Y-%m-%d", onNull: "null input"}}, + "2018-02-08" + ] + } + }, + {date: null}); + testValidator({ + $expr: { + $eq: [ + // The 'format' option was made optional in 4.0. + {$dateToString: {date: "$date"}}, + "2018-02-08T00:00:00.000Z" + ] + } + }, + {date: new Date("2018-02-07")}); + testValidator({$expr: {$eq: [{$convert: {input: "$a", to: "int"}}, 2018]}}, {a: "2017"}); + testValidator({$expr: {$eq: [{$convert: {input: "$a", to: "int", onNull: 0}}, 2018]}}, + {a: null}); +}()); diff --git a/jstests/replsets/view_definition_initial_sync_with_feature_compatibility.js b/jstests/replsets/view_definition_initial_sync_with_feature_compatibility.js new file mode 100644 index 00000000000..79d9c676b1a --- /dev/null +++ b/jstests/replsets/view_definition_initial_sync_with_feature_compatibility.js @@ -0,0 +1,166 @@ +/** + * Test that a new replica set member can successfully sync a collection with a view using 4.0 + * aggregation features, even when the replica set was downgraded to feature compatibility version + * 3.6. + * + * TODO SERVER-33321: Remove FCV 3.6 validation during the 4.1 development cycle. + * + * We restart the secondary as part of this test with the expectation that it still has the same + * data after the restart. + * @tags: [requires_persistence] + */ + +load("jstests/replsets/rslib.js"); + +(function() { + "use strict"; + const testName = "view_definition_initial_sync_with_feature_compatibility"; + + function testView(pipeline) { + // + // Create a single-node replica set. + // + let replTest = new ReplSetTest({name: testName, nodes: 1}); + + replTest.startSet(); + replTest.initiate(); + + let primary = replTest.getPrimary(); + let testDB = primary.getDB("test"); + + // + // Explicitly set the replica set to feature compatibility version 4.0. + // + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: "4.0"})); + + // + // Create a view using 4.0 query features. + // + assert.commandWorked(testDB.createView("view1", "coll", pipeline)); + + // + // Downgrade the replica set to feature compatibility version 3.6. + // + assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: "3.6"})); + + // + // Add a new member to the replica set. + // + let secondaryDBPath = MongoRunner.dataPath + testName + "_secondary"; + resetDbpath(secondaryDBPath); + let secondary = replTest.add({dbpath: secondaryDBPath}); + replTest.reInitiate(secondary); + reconnect(primary); + reconnect(secondary); + + // + // Once the new member completes its initial sync, stop it, remove it from the replica + // set, and start it back up as an individual instance. + // + replTest.waitForState(secondary, [ReplSetTest.State.PRIMARY, ReplSetTest.State.SECONDARY]); + + replTest.stopSet(undefined /* send default signal */, + true /* don't clear data directory */); + + secondary = MongoRunner.runMongod({dbpath: secondaryDBPath, noCleanData: true}); + assert.neq(null, secondary, "mongod was unable to start up"); + + // + // Verify that the view synced to the new member. + // + let secondaryDB = secondary.getDB("test"); + assert.eq(secondaryDB.system.views.findOne({_id: "test.view1"}, {_id: 1}), + {_id: "test.view1"}); + + // + // Verify that, even though a view using 4.0 query features exists, it is not possible to + // create a new view using 4.0 query features because of feature compatibility version 3.6. + // + assert.commandFailedWithCode(secondaryDB.createView("view2", "coll", pipeline), + ErrorCodes.QueryFeatureNotAllowed); + + MongoRunner.stopMongod(secondary); + } + + testView([{$project: {trimmed: {$trim: {input: " hi "}}}}]); + testView([{$project: {trimmed: {$ltrim: {input: " hi "}}}}]); + testView([{$project: {trimmed: {$rtrim: {input: " hi "}}}}]); + testView([{ + $project: { + dateFromStringWithFormat: + // The 'format' option was added in 4.0. + {$dateFromString: {dateString: "2018-02-08", format: "$format"}} + } + }]); + testView([{ + $project: { + dateFromStringWithOnNull: { + // The 'onNull' option was added in 4.0. + $dateFromString: {dateString: "$dateString", onNull: new Date("1970-01-01")} + } + } + }]); + testView([{ + $project: { + dateFromStringWithOnError: { + // The 'onError' option was added in 4.0. + $dateFromString: {dateString: "$dateString", onError: new Date("1970-01-01")} + } + } + }]); + testView([{ + $project: { + dateToStringWithOnNull: + // The 'onNull' option was added in 4.0. + {$dateToString: {date: "$date", format: "%Y-%m-%d", onNull: "null input"}} + } + }]); + // The 'format' option was made optional in 4.0. + testView([{$project: {dateToStringWithoutFormat: {$dateToString: {date: "$date"}}}}]); + testView([{$project: {conversion: {$convert: {input: "$a", to: "int"}}}}]); + testView([{$project: {conversionWithOnNull: {$convert: {input: "$a", to: "int", onNull: 0}}}}]); + + // Test using one of the prohibited expressions inside of an $expr within a MatchExpression + // embedded in the pipeline. + testView([{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}]); + testView([{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }]); + testView([{$facet: {withinMatch: [{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}]}}]); + testView([{ + $facet: { + withinGraphLookup: [{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }] + } + }]); + testView([{ + $facet: { + withinMatch: [{$match: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}}}], + withinGraphLookup: [{ + $graphLookup: { + from: "foreign", + startWith: "$start", + connectFromField: "to", + connectToField: "_id", + as: "results", + restrictSearchWithMatch: {$expr: {$eq: [{$trim: {input: "$a"}}, "hi"]}} + } + }] + } + }]); +}()); diff --git a/src/mongo/db/catalog/coll_mod.cpp b/src/mongo/db/catalog/coll_mod.cpp index 4ef185fde2c..b004bf6d837 100644 --- a/src/mongo/db/catalog/coll_mod.cpp +++ b/src/mongo/db/catalog/coll_mod.cpp @@ -175,9 +175,23 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx, } } else if (fieldName == "validator" && !isView) { - MatchExpressionParser::AllowedFeatureSet allowedFeatures = - MatchExpressionParser::kDefaultSpecialFeatures; - auto statusW = coll->parseValidator(opCtx, e.Obj(), allowedFeatures); + // Save this to a variable to avoid reading the atomic variable multiple times. + const auto currentFCV = serverGlobalParams.featureCompatibility.getVersion(); + + // If the feature compatibility version is not 4.0, and we are validating features as + // master, ban the use of new agg features introduced in 4.0 to prevent them from being + // persisted in the catalog. + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion; + if (serverGlobalParams.validateFeaturesAsMaster.load() && + currentFCV != + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) { + maxFeatureCompatibilityVersion = currentFCV; + } + auto statusW = coll->parseValidator(opCtx, + e.Obj(), + MatchExpressionParser::kDefaultSpecialFeatures, + maxFeatureCompatibilityVersion); if (!statusW.isOK()) { return statusW.getStatus(); } diff --git a/src/mongo/db/catalog/collection.h b/src/mongo/db/catalog/collection.h index 755da32b65e..042c7a3a0c5 100644 --- a/src/mongo/db/catalog/collection.h +++ b/src/mongo/db/catalog/collection.h @@ -290,7 +290,9 @@ public: virtual StatusWithMatchExpression parseValidator( OperationContext* opCtx, const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) const = 0; + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion = boost::none) const = 0; virtual Status setValidator(OperationContext* opCtx, BSONObj validator) = 0; @@ -618,8 +620,11 @@ public: inline StatusWithMatchExpression parseValidator( OperationContext* opCtx, const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) const { - return this->_impl().parseValidator(opCtx, validator, allowedFeatures); + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion) const { + return this->_impl().parseValidator( + opCtx, validator, allowedFeatures, maxFeatureCompatibilityVersion); } static StatusWith<ValidationLevel> parseValidationLevel(StringData); diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 28a16f136db..dae6600d637 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -268,7 +268,9 @@ Status CollectionImpl::checkValidation(OperationContext* opCtx, const BSONObj& d StatusWithMatchExpression CollectionImpl::parseValidator( OperationContext* opCtx, const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) const { + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion) const { if (validator.isEmpty()) return {nullptr}; @@ -294,6 +296,9 @@ StatusWithMatchExpression CollectionImpl::parseValidator( // owned by the Collection and will outlive the OperationContext they were created under. expCtx->opCtx = nullptr; + // Enforce a maximum feature version if requested. + expCtx->maxFeatureCompatibilityVersion = maxFeatureCompatibilityVersion; + auto statusWithMatcher = MatchExpressionParser::parse(validator, expCtx, ExtensionsCallbackNoop(), allowedFeatures); if (!statusWithMatcher.isOK()) diff --git a/src/mongo/db/catalog/collection_impl.h b/src/mongo/db/catalog/collection_impl.h index 1950e1570b9..38c58976026 100644 --- a/src/mongo/db/catalog/collection_impl.h +++ b/src/mongo/db/catalog/collection_impl.h @@ -286,7 +286,9 @@ public: StatusWithMatchExpression parseValidator( OperationContext* opCtx, const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) const final; + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion = boost::none) const final; static StatusWith<ValidationLevel> parseValidationLevel(StringData); static StatusWith<ValidationAction> parseValidationAction(StringData); diff --git a/src/mongo/db/catalog/collection_mock.h b/src/mongo/db/catalog/collection_mock.h index 785256358aa..2b435027c64 100644 --- a/src/mongo/db/catalog/collection_mock.h +++ b/src/mongo/db/catalog/collection_mock.h @@ -223,7 +223,9 @@ public: StatusWithMatchExpression parseValidator( OperationContext* opCtx, const BSONObj& validator, - MatchExpressionParser::AllowedFeatureSet allowedFeatures) const { + MatchExpressionParser::AllowedFeatureSet allowedFeatures, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion) const { std::abort(); } diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index 858a579982e..1c7d4b0157b 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -1037,6 +1037,17 @@ auto mongo::userCreateNSImpl(OperationContext* opCtx, if (!collectionOptions.validator.isEmpty()) { boost::intrusive_ptr<ExpressionContext> expCtx( new ExpressionContext(opCtx, collator.get())); + + // Save this to a variable to avoid reading the atomic variable multiple times. + const auto currentFCV = serverGlobalParams.featureCompatibility.getVersion(); + + // If the feature compatibility version is not 4.0, and we are validating features as + // master, ban the use of new agg features introduced in 4.0 to prevent them from being + // persisted in the catalog. + if (serverGlobalParams.validateFeaturesAsMaster.load() && + currentFCV != ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) { + expCtx->maxFeatureCompatibilityVersion = currentFCV; + } auto statusWithMatcher = MatchExpressionParser::parse(collectionOptions.validator, std::move(expCtx)); diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index d91f36f7a1c..680094f0971 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -184,11 +184,12 @@ env.Library( 'expression.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/query/datetime/date_time_support', + '$BUILD_DIR/mongo/db/server_options_core', + '$BUILD_DIR/mongo/util/summation', 'dependencies', 'document_value', 'expression_context', - '$BUILD_DIR/mongo/db/query/datetime/date_time_support', - '$BUILD_DIR/mongo/util/summation', ] ) diff --git a/src/mongo/db/pipeline/document_source_graph_lookup.cpp b/src/mongo/db/pipeline/document_source_graph_lookup.cpp index 07fac74d963..46eec709cd3 100644 --- a/src/mongo/db/pipeline/document_source_graph_lookup.cpp +++ b/src/mongo/db/pipeline/document_source_graph_lookup.cpp @@ -545,14 +545,9 @@ intrusive_ptr<DocumentSource> DocumentSourceGraphLookUp::createFromBson( // We don't need to keep ahold of the MatchExpression, but we do need to ensure that // the specified object is parseable and does not contain extensions. - auto parsedMatchExpression = - MatchExpressionParser::parse(argument.embeddedObject(), expCtx); - - uassert(40186, - str::stream() - << "Failed to parse 'restrictSearchWithMatch' option to $graphLookup: " - << parsedMatchExpression.getStatus().reason(), - parsedMatchExpression.isOK()); + uassertStatusOKWithContext( + MatchExpressionParser::parse(argument.embeddedObject(), expCtx), + "Failed to parse 'restrictSearchWithMatch' option to $graphLookup"); additionalFilter = argument.embeddedObject().getOwned(); continue; diff --git a/src/mongo/db/pipeline/expression.cpp b/src/mongo/db/pipeline/expression.cpp index 016749ca29b..318933b3c20 100644 --- a/src/mongo/db/pipeline/expression.cpp +++ b/src/mongo/db/pipeline/expression.cpp @@ -36,6 +36,7 @@ #include <cstdio> #include <vector> +#include "mongo/db/commands/feature_compatibility_version_command_parser.h" #include "mongo/db/jsobj.h" #include "mongo/db/pipeline/document.h" #include "mongo/db/pipeline/expression_context.h" @@ -104,15 +105,23 @@ intrusive_ptr<Expression> Expression::parseObject( } namespace { -StringMap<Parser> parserMap; +struct ParserRegistration { + Parser parser; + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> requiredMinVersion; +}; + +StringMap<ParserRegistration> parserMap; } -void Expression::registerExpression(string key, Parser parser) { +void Expression::registerExpression( + string key, + Parser parser, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> requiredMinVersion) { auto op = parserMap.find(key); massert(17064, str::stream() << "Duplicate expression (" << key << ") registered.", op == parserMap.end()); - parserMap[key] = parser; + parserMap[key] = {parser, requiredMinVersion}; } intrusive_ptr<Expression> Expression::parseExpression( @@ -127,11 +136,26 @@ intrusive_ptr<Expression> Expression::parseExpression( // Look up the parser associated with the expression name. const char* opName = obj.firstElementFieldName(); - auto op = parserMap.find(opName); + auto it = parserMap.find(opName); uassert(ErrorCodes::InvalidPipelineOperator, str::stream() << "Unrecognized expression '" << opName << "'", - op != parserMap.end()); - return op->second(expCtx, obj.firstElement(), vps); + it != parserMap.end()); + + // Make sure we are allowed to use this expression under the current feature compatibility + // version. + auto& entry = it->second; + uassert( + ErrorCodes::QueryFeatureNotAllowed, + // TODO SERVER-31968 we would like to include the current version and the required minimum + // version in this error message, but using FeatureCompatibilityVersion::toString() would + // introduce a dependency cycle. + str::stream() << opName + << " is not allowed in the current feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + !expCtx->maxFeatureCompatibilityVersion || !entry.requiredMinVersion || + (*entry.requiredMinVersion <= *expCtx->maxFeatureCompatibilityVersion)); + return entry.parser(expCtx, obj.firstElement(), vps); } Expression::ExpressionVector ExpressionNary::parseArguments( @@ -1302,6 +1326,34 @@ intrusive_ptr<Expression> ExpressionDateFromString::parse( } } + // The 'format', 'onNull' and 'onError' options were introduced in 4.0, and should not be + // allowed in contexts where the maximum feature version is <= 4.0. + if (expCtx->maxFeatureCompatibilityVersion && + *expCtx->maxFeatureCompatibilityVersion < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) { + uassert( + ErrorCodes::QueryFeatureNotAllowed, + str::stream() << "\"format\" option to $dateFromString is not allowed with the current " + "feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + !formatElem); + uassert( + ErrorCodes::QueryFeatureNotAllowed, + str::stream() << "\"onNull\" option to $dateFromString is not allowed with the current " + "feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + !onNullElem); + uassert(ErrorCodes::QueryFeatureNotAllowed, + str::stream() + << "\"onError\" option to $dateFromString is not allowed with the current " + "feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + !onErrorElem); + } + uassert(40542, "Missing 'dateString' parameter to $dateFromString", dateStringElem); return new ExpressionDateFromString( @@ -1622,6 +1674,28 @@ intrusive_ptr<Expression> ExpressionDateToString::parse( } } + // The 'onNull' option was introduced in 4.0, and should not be allowed in contexts where the + // maximum feature version is <= 4.0. Similarly, the 'format' option was made optional in 4.0, + // so should be required in such scenarios. + if (expCtx->maxFeatureCompatibilityVersion && + *expCtx->maxFeatureCompatibilityVersion < + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) { + uassert( + ErrorCodes::QueryFeatureNotAllowed, + str::stream() << "\"onNull\" option to $dateToString is not allowed with the current " + "feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + !onNullElem); + + uassert(ErrorCodes::QueryFeatureNotAllowed, + str::stream() << "\"format\" option to $dateToString is required with the current " + "feature compatibility version. See " + << feature_compatibility_version::kDochubLink + << " for more information.", + formatElem); + } + uassert(18628, "Missing 'date' parameter to $dateToString", !dateElem.eoo()); return new ExpressionDateToString(expCtx, @@ -4325,9 +4399,21 @@ const char* ExpressionToUpper::getOpName() const { /* -------------------------- ExpressionTrim ------------------------------ */ -REGISTER_EXPRESSION(trim, ExpressionTrim::parse); -REGISTER_EXPRESSION(ltrim, ExpressionTrim::parse); -REGISTER_EXPRESSION(rtrim, ExpressionTrim::parse); +REGISTER_EXPRESSION_WITH_MIN_VERSION( + trim, + ExpressionTrim::parse, + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40); + +REGISTER_EXPRESSION_WITH_MIN_VERSION( + ltrim, + ExpressionTrim::parse, + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40); + +REGISTER_EXPRESSION_WITH_MIN_VERSION( + rtrim, + ExpressionTrim::parse, + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40); + intrusive_ptr<Expression> ExpressionTrim::parse( const boost::intrusive_ptr<ExpressionContext>& expCtx, BSONElement expr, @@ -5074,7 +5160,11 @@ private: const double ExpressionConvert::kLongLongMaxPlusOneAsDouble = scalbn(1, std::numeric_limits<long long>::digits); -REGISTER_EXPRESSION(convert, ExpressionConvert::parse); +REGISTER_EXPRESSION_WITH_MIN_VERSION( + convert, + ExpressionConvert::parse, + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40); + intrusive_ptr<Expression> ExpressionConvert::parse( const boost::intrusive_ptr<ExpressionContext>& expCtx, BSONElement expr, diff --git a/src/mongo/db/pipeline/expression.h b/src/mongo/db/pipeline/expression.h index 6f3f7c6126d..108447cc784 100644 --- a/src/mongo/db/pipeline/expression.h +++ b/src/mongo/db/pipeline/expression.h @@ -44,6 +44,7 @@ #include "mongo/db/pipeline/value.h" #include "mongo/db/pipeline/variables.h" #include "mongo/db/query/datetime/date_time_support.h" +#include "mongo/db/server_options.h" #include "mongo/stdx/functional.h" #include "mongo/util/intrusive_counter.h" #include "mongo/util/mongoutils/str.h" @@ -56,14 +57,31 @@ class BSONObjBuilder; class DocumentSource; /** - * Registers an Parser so it can be called from parseExpression and friends. + * Registers a Parser so it can be called from parseExpression and friends. * * As an example, if your expression looks like {"$foo": [1,2,3]} you would add this line: * REGISTER_EXPRESSION(foo, ExpressionFoo::parse); + * + * An expression registered this way can be used in any featureCompatibilityVersion. */ #define REGISTER_EXPRESSION(key, parser) \ MONGO_INITIALIZER(addToExpressionParserMap_##key)(InitializerContext*) { \ - Expression::registerExpression("$" #key, (parser)); \ + Expression::registerExpression("$" #key, (parser), boost::none); \ + return Status::OK(); \ + } + +/** + * Registers a Parser so it can be called from parseExpression and friends. Use this version if your + * expression can only be persisted to a catalog data structure in a feature compatibility version + * >= X. + * + * As an example, if your expression looks like {"$foo": [1,2,3]}, and can only be used in a feature + * compatibility version >= X, you would add this line: + * REGISTER_EXPRESSION_WITH_MIN_VERSION(foo, ExpressionFoo::parse, X); + */ +#define REGISTER_EXPRESSION_WITH_MIN_VERSION(key, parser, minVersion) \ + MONGO_INITIALIZER(addToExpressionParserMap_##key)(InitializerContext*) { \ + Expression::registerExpression("$" #key, (parser), (minVersion)); \ return Status::OK(); \ } @@ -208,7 +226,10 @@ public: * DO NOT call this method directly. Instead, use the REGISTER_EXPRESSION macro defined in this * file. */ - static void registerExpression(std::string key, Parser parser); + static void registerExpression( + std::string key, + Parser parser, + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> requiredMinVersion); protected: Expression(const boost::intrusive_ptr<ExpressionContext>& expCtx) : _expCtx(expCtx) { diff --git a/src/mongo/db/pipeline/expression_context.h b/src/mongo/db/pipeline/expression_context.h index 2bafa8253db..9a37f462fe8 100644 --- a/src/mongo/db/pipeline/expression_context.h +++ b/src/mongo/db/pipeline/expression_context.h @@ -202,6 +202,10 @@ public: // Tracks the depth of nested aggregation sub-pipelines. Used to enforce depth limits. size_t subPipelineDepth = 0; + // If set, this will disallow use of features introduced in versions above the provided version. + boost::optional<ServerGlobalParams::FeatureCompatibility::Version> + maxFeatureCompatibilityVersion; + protected: static const int kInterruptCheckPeriod = 128; diff --git a/src/mongo/db/views/view_catalog.cpp b/src/mongo/db/views/view_catalog.cpp index 8c59710941e..1a029f0d5c4 100644 --- a/src/mongo/db/views/view_catalog.cpp +++ b/src/mongo/db/views/view_catalog.cpp @@ -43,9 +43,6 @@ #include "mongo/db/operation_context.h" #include "mongo/db/pipeline/aggregation_request.h" #include "mongo/db/pipeline/document_source.h" -#include "mongo/db/pipeline/document_source_facet.h" -#include "mongo/db/pipeline/document_source_lookup.h" -#include "mongo/db/pipeline/document_source_match.h" #include "mongo/db/pipeline/expression_context.h" #include "mongo/db/pipeline/lite_parsed_pipeline.h" #include "mongo/db/pipeline/pipeline.h" @@ -251,6 +248,17 @@ StatusWith<stdx::unordered_set<NamespaceString>> ViewCatalog::_validatePipeline_ // pipeline that will require a real implementation. std::make_shared<StubMongoProcessInterface>(), std::move(resolvedNamespaces)); + + // Save this to a variable to avoid reading the atomic variable multiple times. + auto currentFCV = serverGlobalParams.featureCompatibility.getVersion(); + + // If the feature compatibility version is not 4.0, and we are validating features as master, + // ban the use of new agg features introduced in 4.0 to prevent them from being persisted in the + // catalog. + if (serverGlobalParams.validateFeaturesAsMaster.load() && + currentFCV != ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo40) { + expCtx->maxFeatureCompatibilityVersion = currentFCV; + } auto pipelineStatus = Pipeline::parse(viewDef.pipeline(), std::move(expCtx)); if (!pipelineStatus.isOK()) { return pipelineStatus.getStatus(); |