SERVER-52874 Implement validateDBMetadata command

13 files changed, 273 insertions, 12 deletions

diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index bf36b923932..5227dda8ebc 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -6003,6 +6003,77 @@ var authCommandsLib = {
               },
             ]
         },
+        {
+          testname: "validate_db_metadata_command_specific_db",
+          command: {
+              validateDBMetadata: 1,
+              db: secondDbName,
+              collection: "test",
+              apiParameters: {version: "1", strict: true}
+          },
+          skipSharded: true,
+          setup: function(db) {
+              assert.commandWorked(db.getSiblingDB(firstDbName).createCollection("test"));
+              assert.commandWorked(db.getSiblingDB(secondDbName).createCollection("test"));
+              assert.commandWorked(db.getSiblingDB("ThirdDB").createCollection("test"));
+          },
+          teardown: function(db) {
+              assert.commandWorked(db.getSiblingDB(firstDbName).dropDatabase());
+              assert.commandWorked(db.getSiblingDB(secondDbName).dropDatabase());
+              assert.commandWorked(db.getSiblingDB("ThirdDB").dropDatabase());
+          },
+          testcases: [
+              {
+                  runOnDb: secondDbName,
+                  privileges: [{resource: {db: secondDbName, collection: ""}, actions: ["validate"]}]
+              },
+              {
+                  // Need to have permission on firstDBName to be able to the command on the db.
+                  runOnDb: firstDbName,
+                  privileges: [{resource: {db: secondDbName, collection: ""}, actions: ["validate"]}],
+                  expectAuthzFailure: true
+              },
+              {
+                  runOnDb: firstDbName,
+                  privileges: [
+                      {resource: {db: firstDbName, collection: ""}, actions: ["validate"]},
+                      {resource: {db: secondDbName, collection: ""}, actions: ["validate"]}
+                  ]
+              },
+          ]
+      },
+      {
+          testname: "validate_db_metadata_command_all_dbs",
+          command: {validateDBMetadata: 1, apiParameters: {version: "1", strict: true}},
+          skipSharded: true,
+          setup: function(db) {
+              assert.commandWorked(db.getSiblingDB(firstDbName).createCollection("test"));
+              assert.commandWorked(db.getSiblingDB(secondDbName).createCollection("test"));
+          },
+          teardown: function(db) {
+              assert.commandWorked(db.getSiblingDB(firstDbName).dropDatabase());
+              assert.commandWorked(db.getSiblingDB(secondDbName).dropDatabase());
+          },
+          testcases: [
+              {
+                  // Since the command didn't specify a 'db', it validates all dbs and hence require
+                  // permission to run on all dbs.
+                  runOnDb: secondDbName,
+                  privileges: [{resource: {db: secondDbName, collection: ""}, actions: ["validate"]}],
+                  expectAuthzFailure: true
+              },
+              {
+                  runOnDb: secondDbName,
+                  privileges: [
+                      {resource: {db: "admin", collection: ""}, actions: ["validate"]},
+                      {resource: {db: "config", collection: ""}, actions: ["validate"]},
+                      {resource: {db: "local", collection: ""}, actions: ["validate"]},
+                      {resource: {db: firstDbName, collection: ""}, actions: ["validate"]},
+                      {resource: {db: secondDbName, collection: ""}, actions: ["validate"]}
+                  ]
+              },
+          ]
+      },
     ],
 
     /************* SHARED TEST LOGIC ****************/
diff --git a/jstests/core/api_version_pipeline_stages.js b/jstests/core/api_version_pipeline_stages.js
index b72f7336d3a..425e08730cd 100644
--- a/jstests/core/api_version_pipeline_stages.js
+++ b/jstests/core/api_version_pipeline_stages.js
@@ -26,6 +26,9 @@ const pipelines = [
     [{$listLocalSessions: {}}],
     [{$listSessions: {}}],
     [{$planCacheStats: {}}],
+    [{$unionWith: {coll: "coll2", pipeline: [{$collStats: {count: {}}}]}}],
+    [{$lookup: {from: "coll2", pipeline: [{$indexStats: {}}]}}],
+    [{$facet: {field1: [], field2: [{$indexStats: {}}]}}],
 ];
 
 for (let pipeline of pipelines) {
diff --git a/jstests/core/api_version_test_expression.js b/jstests/core/api_version_test_expression.js
index 5259fdc3dfa..fc2488f321f 100644
--- a/jstests/core/api_version_test_expression.js
+++ b/jstests/core/api_version_test_expression.js
@@ -135,16 +135,27 @@ assert.commandFailedWithCode(
         {aggregate: "view", pipeline: pipeline, cursor: {}, apiVersion: "1", apiStrict: true}),
     ErrorCodes.APIStrictError);
 
-// Create a view with {unstable: true}.
+// Create a view with 'unstable' parameter should fail with 'apiStrict'.
 db.unstableView.drop();
-assert.commandWorked(db.runCommand({
+assert.commandFailedWithCode(db.runCommand({
     create: "unstableView",
     viewOn: collName,
     pipeline: pipeline,
     apiStrict: true,
     apiVersion: "1"
+}),
+                             ErrorCodes.APIStrictError);
+
+// Create a view with 'unstable' should be allowed without 'apiStrict'.
+assert.commandWorked(db.runCommand({
+    create: "unstableView",
+    viewOn: collName,
+    pipeline: pipeline,
+    apiVersion: "1",
+    apiStrict: false
 }));
 assert.commandWorked(db.runCommand({aggregate: "unstableView", pipeline: [], cursor: {}}));
+
 // This commmand will fail even with the empty pipeline because of the view.
 assert.commandFailedWithCode(
     db.runCommand(
@@ -198,18 +209,13 @@ assert.commandWorked(db[validatedCollName].runCommand({
 
 // Test that API version parameters are inherited into the inner command of the explain command.
 function checkExplainInnerCommandGetsAPIVersionParameters(explainedCmd, errCode) {
-    let explainRes = db.runCommand(
-        {explain: explainedCmd, apiVersion: "1", apiDeprecationErrors: true, apiStrict: true});
-
-    assert(explainRes.hasOwnProperty('executionStats'), explainRes);
-    const execStats = explainRes['executionStats'];
-
-    // 'execStats' will return APIStrictError if the inner command gets the APIVersionParameters.
-    assert.eq(execStats['executionSuccess'], false, execStats);
-    assert.eq(execStats['errorCode'], errCode, execStats);
+    assert.commandFailedWithCode(
+        db.runCommand(
+            {explain: explainedCmd, apiVersion: "1", apiDeprecationErrors: true, apiStrict: true}),
+        errCode);
 
     // If 'apiStrict: false' the inner aggregate command will execute successfully.
-    explainRes = db.runCommand({explain: explainedCmd, apiVersion: "1", apiStrict: false});
+    const explainRes = db.runCommand({explain: explainedCmd, apiVersion: "1", apiStrict: false});
     assert(explainRes.hasOwnProperty('executionStats'), explainRes);
     assert.eq(explainRes['executionStats']['executionSuccess'], true, explainRes);
 }
@@ -234,4 +240,7 @@ findCmd = {
     projection: {v: {$_testApiVersion: {deprecated: true}}}
 };
 checkExplainInnerCommandGetsAPIVersionParameters(findCmd, ErrorCodes.APIDeprecationError);
+
+db[validatedCollName].drop();
+db.unstableView.drop();
 })();
diff --git a/jstests/core/validate_db_metadata_command.js b/jstests/core/validate_db_metadata_command.js
new file mode 100644
index 00000000000..c0a3e21e6a8
--- /dev/null
+++ b/jstests/core/validate_db_metadata_command.js
@@ -0,0 +1,126 @@
+/**
+ * Tests the validateDBMetaData commands with various input parameters.
+ * @tags: [
+ *   requires_fcv_49,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/fixture_helpers.js");  // For FixtureHelpers.
+
+const dbName = jsTestName();
+
+const testDB = db.getSiblingDB(dbName);
+assert.commandWorked(testDB.dropDatabase());
+const coll1 = testDB.coll1;
+
+// Drop all the unstable data that the other tests might have created. This will ensure that the
+// validateDBMetadata command is validating only the data generated by this test.
+(function dropAllUnstableData() {
+    const listDBRes = assert.commandWorked(db.adminCommand({listDatabases: 1, nameOnly: true}));
+    for (let listDBOutput of listDBRes.databases) {
+        // Skip non-user databases.
+        if (Array.contains(["admin", "config", "local", "$external"], listDBOutput.name)) {
+            continue;
+        }
+        const currentDB = db.getSiblingDB(listDBOutput.name);
+        for (let collInfo of currentDB.getCollectionInfos()) {
+            if (collInfo.type == "collection" && !collInfo.name.startsWith("system")) {
+                assert.commandWorked(currentDB[collInfo.name].dropIndexes());
+            }
+        }
+    }
+})();
+
+// Verify that the 'apiParameters' field is required.
+const res = assert.commandFailedWithCode(testDB.runCommand({validateDBMetadata: 1}), 40414);
+
+function validate({dbName, coll, apiStrict, error}) {
+    dbName = dbName ? dbName : null;
+    coll = coll ? coll : null;
+    const res = assert.commandWorked(testDB.runCommand({
+        validateDBMetadata: 1,
+        db: dbName,
+        collection: coll,
+        apiParameters: {version: "1", strict: apiStrict}
+    }));
+
+    assert(res.apiVersionErrors);
+    const foundError = res.apiVersionErrors.length > 0;
+
+    // Verify that 'apiVersionErrors' is not empty when 'error' is true, and vice versa.
+    assert((!error && !foundError) || (error && foundError), res);
+
+    if (error) {
+        for (let apiError of res.apiVersionErrors) {
+            assert(apiError.ns);
+            if (error.code) {
+                assert.eq(apiError.code, error.code);
+            }
+
+            if (FixtureHelpers.isMongos(testDB)) {
+                // Check that every error has an additional 'shard' field on sharded clusters.
+                assert(apiError.shard);
+            }
+        }
+    }
+}
+
+//
+// Tests for indexes.
+//
+assert.commandWorked(coll1.createIndex({p: "text"}));
+
+validate({apiStrict: false});
+
+// All dbs but different collection name.
+validate({coll: "coll2", apiStrict: true});
+
+// Different db, and collection which has unstable index should not error.
+validate({dbName: "new", coll: "coll1", apiStrict: true});
+validate({
+    dbName: "new",
+    apiStrict: true,
+});
+
+// Cases where the command returns an error.
+validate({apiStrict: true, error: true});
+validate({coll: "coll1", apiStrict: true, error: true});
+validate({
+    dbName: testDB.getName(),
+    coll: "coll1",
+    apiStrict: true,
+    error: {code: ErrorCodes.APIStrictError}
+});
+validate({dbName: testDB.getName(), apiStrict: true, error: true});
+
+//
+// Tests for views.
+//
+assert.commandWorked(coll1.dropIndexes());
+validate({apiStrict: true});
+
+// Create a view which uses unstable expression and verify that validateDBMetadata commands throws
+// an assertion.
+const view =
+    testDB.createView("view1", "coll2", [{$project: {v: {$_testApiVersion: {unstable: true}}}}]);
+
+validate({apiStrict: true, error: true});
+validate({dbName: dbName, apiStrict: true, error: true});
+
+validate({dbName: "otherDB", apiStrict: true});
+validate({dbName: dbName, coll: "coll", apiStrict: true});
+
+// With view name in the input.
+validate({coll: "view1", apiStrict: true, error: {code: ErrorCodes.APIStrictError}});
+validate(
+    {dbName: dbName, coll: "view1", apiStrict: true, error: {code: ErrorCodes.APIStrictError}});
+
+validate({dbName: "new", coll: "view1", apiStrict: true});
+
+// Collection named same as the view name in another db.
+const testDB2 = db.getSiblingDB("testDB2");
+const collWithViewName = testDB2.view1;
+validate({coll: "view1", apiStrict: true, error: {code: ErrorCodes.APIStrictError}});
+}());
\ No newline at end of file
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 93dc1803ffc..acd53276fc6 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -578,6 +578,8 @@ let viewsCommandTests = {
     updateZoneKeyRange: {skip: isUnrelated},
     usersInfo: {skip: isUnrelated},
     validate: {command: {validate: "view"}, expectFailure: true},
+    validateDBMetadata:
+        {command: {validateDBMetadata: 1, apiParameters: {version: "1", strict: true}}},
     waitForOngoingChunkSplits: {skip: isUnrelated},
     voteCommitImportCollection: {skip: isUnrelated},
     voteCommitIndexBuild: {skip: isUnrelated},
diff --git a/jstests/libs/parallelTester.js b/jstests/libs/parallelTester.js
index c17dd67c0bb..511ba98b1cd 100644
--- a/jstests/libs/parallelTester.js
+++ b/jstests/libs/parallelTester.js
@@ -221,6 +221,9 @@ if (typeof _threadInject != "undefined") {
             // This test updates global memory usage counters in the bucket catalog in a way that
             // may affect other time-series tests running concurrently.
             "timeseries/timeseries_idle_buckets.js",
+
+            // Assumes that other tests are not creating API version 1 incompatible data.
+            "validate_db_metadata_command.js",
         ]);
 
         // Get files, including files in subdirectories.
diff --git a/jstests/noPassthrough/validate_db_metadata_limits.js b/jstests/noPassthrough/validate_db_metadata_limits.js
new file mode 100644
index 00000000000..dce8af08b6f
--- /dev/null
+++ b/jstests/noPassthrough/validate_db_metadata_limits.js
@@ -0,0 +1,28 @@
+/**
+ * Tests to verify that the validateDBMetadata command returns response correctly when the expected
+ * output data is larger than the max BSON size.
+ */
+(function() {
+"use strict";
+
+const conn = MongoRunner.runMongod();
+const testDB = conn.getDB("validate_db_metadaba");
+const coll = testDB.getCollection("test");
+
+for (let i = 0; i < 100; i++) {
+    // Create a large index name. As the index name is returned in the output validateDBMetadata
+    // command, it can cause the output size to exceed max BSON size.
+    let largeName = "a".repeat(200000);
+    assert.commandWorked(testDB.runCommand(
+        {createIndexes: "test" + i, indexes: [{key: {p: 1}, name: largeName, sparse: true}]}));
+}
+
+const res = assert.commandWorked(
+    testDB.runCommand({validateDBMetadata: 1, apiParameters: {version: "1", strict: true}}));
+
+assert(res.hasMoreErrors, res);
+assert(res.apiVersionErrors, res);
+assert(res.apiVersionErrors.length < 100, res);
+
+MongoRunner.stopMongod(conn);
+})();
\ No newline at end of file
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index 78a557b4649..2e117b7138d 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -311,6 +311,11 @@ const allCommands = {
     updateUser: {skip: isPrimaryOnly},
     usersInfo: {skip: isPrimaryOnly},
     validate: {skip: isNotAUserDataRead},
+    validateDBMetadata: {
+        command: {validateDBMetadata: 1, apiParameters: {version: "1", strict: true}},
+        expectFailure: true,
+        expectedErrorCode: ErrorCodes.NotPrimaryOrSecondary,
+    },
     voteCommitImportCollection: {skip: isNotAUserDataRead},
     voteCommitIndexBuild: {skip: isNotAUserDataRead},
     waitForFailPoint: {skip: isNotAUserDataRead},
diff --git a/jstests/sharding/database_versioning_all_commands.js b/jstests/sharding/database_versioning_all_commands.js
index 2474c572502..1e5973d3654 100644
--- a/jstests/sharding/database_versioning_all_commands.js
+++ b/jstests/sharding/database_versioning_all_commands.js
@@ -666,6 +666,16 @@ let testCases = {
             },
         }
     },
+    validateDBMetadata: {
+        run: {
+            // validateDBMetadata is always broadcast to all shards.
+            sendsDbVersion: false,
+            explicitlyCreateCollection: true,
+            command: function(dbName, collName) {
+                return {validateDBMetadata: 1, apiParameters: {version: "1"}};
+            },
+        }
+    },
     waitForFailPoint: {skip: "executes locally on mongos (not sent to any remote node)"},
     whatsmyuri: {skip: "executes locally on mongos (not sent to any remote node)"},
 };
diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js
index 78881616721..f0beeb00a2c 100644
--- a/jstests/sharding/read_write_concern_defaults_application.js
+++ b/jstests/sharding/read_write_concern_defaults_application.js
@@ -663,6 +663,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "does not accept read or write concern"},
     usersInfo: {skip: "does not accept read or write concern"},
     validate: {skip: "does not accept read or write concern"},
+    validateDBMetadata: {skip: "does not accept read or write concern"},
     voteCommitImportCollection: {skip: "internal command"},
     voteCommitIndexBuild: {skip: "internal command"},
     waitForFailPoint: {skip: "does not accept read or write concern"},
diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js
index b207d25d91c..62c5558db53 100644
--- a/jstests/sharding/safe_secondary_reads_drop_recreate.js
+++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js
@@ -309,6 +309,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    validateDBMetadata: {skip: "does not return user data"},
     waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
index f90dbb90f21..526e0617cf4 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js
@@ -381,6 +381,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    validateDBMetadata: {skip: "does not return user data"},
     waitForFailPoint: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}
diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
index a81d7c0b47e..039071310a2 100644
--- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
+++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js
@@ -316,6 +316,7 @@ let testCases = {
     updateZoneKeyRange: {skip: "primary only"},
     usersInfo: {skip: "primary only"},
     validate: {skip: "does not return user data"},
+    validateDBMetadata: {skip: "does not return user data"},
     waitForOngoingChunkSplits: {skip: "does not return user data"},
     waitForFailPoint: {skip: "does not return user data"},
     whatsmyuri: {skip: "does not return user data"}