summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--jstests/core/views/views_all_commands.js1
-rw-r--r--jstests/replsets/db_reads_while_recovering_all_commands.js1
-rw-r--r--jstests/sharding/read_write_concern_defaults_application.js1
-rw-r--r--jstests/sharding/run_restore.js162
-rw-r--r--src/mongo/db/namespace_string.cpp5
-rw-r--r--src/mongo/db/namespace_string.h3
-rw-r--r--src/mongo/db/s/SConscript1
-rw-r--r--src/mongo/db/s/config/configsvr_run_restore_command.cpp274
8 files changed, 448 insertions, 0 deletions
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 8996ec0243d..d55c4f07f1f 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -114,6 +114,7 @@ let viewsCommandTests = {
_configsvrRemoveTags: {skip: isAnInternalCommand},
_configsvrRepairShardedCollectionChunksHistory: {skip: isAnInternalCommand},
_configsvrReshardCollection: {skip: isAnInternalCommand},
+ _configsvrRunRestore: {skip: isAnInternalCommand},
_configsvrSetAllowMigrations: {skip: isAnInternalCommand},
_configsvrSetClusterParameter: {skip: isAnInternalCommand},
_configsvrSetUserWriteBlockMode: {skip: isAnInternalCommand},
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index eca3081b7b9..8c884d6e9c5 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -51,6 +51,7 @@ const allCommands = {
_configsvrRepairShardedCollectionChunksHistory: {skip: isPrimaryOnly},
_configsvrRenameCollectionMetadata: {skip: isPrimaryOnly},
_configsvrReshardCollection: {skip: isPrimaryOnly},
+ _configsvrRunRestore: {skip: isPrimaryOnly},
_configsvrSetAllowMigrations: {skip: isPrimaryOnly},
_configsvrSetClusterParameter: {skip: isPrimaryOnly},
_configsvrSetUserWriteBlockMode: {skip: isPrimaryOnly},
diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js
index 97ab87c0de1..b43c17181c4 100644
--- a/jstests/sharding/read_write_concern_defaults_application.js
+++ b/jstests/sharding/read_write_concern_defaults_application.js
@@ -117,6 +117,7 @@ let testCases = {
_configsvrRenameCollectionMetadata: {skip: "internal command"},
_configsvrRepairShardedCollectionChunksHistory: {skip: "internal command"},
_configsvrReshardCollection: {skip: "internal command"},
+ _configsvrRunRestore: {skip: "internal command"},
_configsvrSetAllowMigrations: {skip: "internal command"},
_configsvrSetClusterParameter: {skip: "internal command"},
_configsvrSetUserWriteBlockMode: {skip: "internal command"},
diff --git a/jstests/sharding/run_restore.js b/jstests/sharding/run_restore.js
new file mode 100644
index 00000000000..a314c2bca58
--- /dev/null
+++ b/jstests/sharding/run_restore.js
@@ -0,0 +1,162 @@
+/**
+ * Tests that the "_configsvrRunRestore" command removes documents in config collections not
+ * referenced in the "local.system.collections_to_restore" collection.
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/feature_flag_util.js");
+
+const s =
+ new ShardingTest({name: "runRestore", shards: 2, mongos: 1, config: 1, other: {chunkSize: 1}});
+
+let mongos = s.s0;
+let db = s.getDB("test");
+if (!FeatureFlagUtil.isEnabled(db, "SelectiveBackup")) {
+ jsTestLog("Skipping as featureFlagSelectiveBackup is not enabled");
+ s.stop();
+ return;
+}
+
+s.adminCommand({enablesharding: "test"});
+s.ensurePrimaryShard("test", s.shard1.shardName);
+s.adminCommand({shardcollection: "test.a", key: {x: 1}});
+s.adminCommand({shardcollection: "test.b", key: {x: 1}});
+
+s.adminCommand({enablesharding: "unusedDB"});
+s.ensurePrimaryShard("unusedDB", s.shard0.shardName);
+
+let primary = s.getPrimaryShard("test").getDB("test");
+let primaryName = s.getPrimaryShard("test").shardName;
+
+let secondary = s.getOther(primary).getDB("test");
+let secondaryName = s.getOther(primary).shardName;
+
+for (let i = 0; i < 6; i++) {
+ assert.commandWorked(db.getCollection("a").insert({x: i}));
+ assert.commandWorked(db.getCollection("b").insert({x: i}));
+
+ // Split chunks we just inserted.
+ assert.commandWorked(mongos.adminCommand({split: "test.a", middle: {x: i}}));
+ assert.commandWorked(mongos.adminCommand({split: "test.b", middle: {x: i}}));
+}
+
+const aCollUUID =
+ mongos.getDB("config").getCollection("collections").find({_id: "test.a"}).toArray()[0].uuid;
+const bCollUUID =
+ mongos.getDB("config").getCollection("collections").find({_id: "test.b"}).toArray()[0].uuid;
+
+for (const uuid of [aCollUUID, bCollUUID]) {
+ assert.eq(7,
+ mongos.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: uuid, shard: primaryName})
+ .count());
+ assert.eq(0,
+ mongos.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: uuid, shard: secondaryName})
+ .count());
+}
+
+// Move chunks between shards.
+for (const x of [0, 2, 4]) {
+ assert.commandWorked(s.s0.adminCommand(
+ {moveChunk: "test.a", find: {x: x}, to: secondary.getMongo().name, _waitForDelete: true}));
+ assert.commandWorked(s.s0.adminCommand(
+ {moveChunk: "test.b", find: {x: x}, to: secondary.getMongo().name, _waitForDelete: true}));
+}
+
+// Check config collection counts.
+for (const uuid of [aCollUUID, bCollUUID]) {
+ assert.eq(4,
+ mongos.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: uuid, shard: primaryName})
+ .count());
+ assert.eq(3,
+ mongos.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: uuid, shard: secondaryName})
+ .count());
+}
+
+assert.eq(1, mongos.getDB("config").getCollection("collections").find({_id: "test.a"}).count());
+assert.eq(1, mongos.getDB("config").getCollection("collections").find({_id: "test.b"}).count());
+
+assert.eq(1, mongos.getDB("config").getCollection("locks").find({_id: "test"}).count());
+assert.eq(1, mongos.getDB("config").getCollection("locks").find({_id: "test.a"}).count());
+assert.eq(1, mongos.getDB("config").getCollection("locks").find({_id: "test.b"}).count());
+assert.eq(1, mongos.getDB("config").getCollection("locks").find({_id: "unusedDB"}).count());
+
+assert.eq(1, mongos.getDB("config").getCollection("databases").find({_id: "test"}).count());
+assert.eq(1, mongos.getDB("config").getCollection("databases").find({_id: "unusedDB"}).count());
+
+s.stop({noCleanData: true});
+
+const configDbPath = s.c0.dbpath;
+
+// Start the config server in standalone mode.
+let conn = MongoRunner.runMongod({noCleanData: true, dbpath: configDbPath});
+assert(conn);
+
+// Can't run the "_configsvrRunRestore" command without --restore.
+assert.commandFailedWithCode(conn.getDB("admin").runCommand({_configsvrRunRestore: 1}),
+ ErrorCodes.CommandFailed);
+
+MongoRunner.stopMongod(conn);
+
+// Start the config server in standalone restore mode.
+conn = MongoRunner.runMongod({noCleanData: true, dbpath: configDbPath, restore: ""});
+assert(conn);
+
+assert.commandWorked(conn.getDB("admin").runCommand({setParameter: 1, logLevel: 1}));
+
+// Can't run if the "local.system.collections_to_restore" collection is missing.
+assert.commandFailedWithCode(conn.getDB("admin").runCommand({_configsvrRunRestore: 1}),
+ ErrorCodes.NamespaceNotFound);
+
+// Create the "local.system.collections_to_restore" collection and insert "test.a".
+assert.commandWorked(conn.getDB("local").createCollection("system.collections_to_restore"));
+assert.commandWorked(conn.getDB("local").getCollection("system.collections_to_restore").insert({
+ ns: "test.a",
+ uuid: aCollUUID
+}));
+
+assert.commandWorked(conn.getDB("admin").runCommand({_configsvrRunRestore: 1}));
+
+assert.eq(4,
+ conn.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: aCollUUID, shard: primaryName})
+ .count());
+assert.eq(3,
+ conn.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: aCollUUID, shard: secondaryName})
+ .count());
+
+assert.eq(0,
+ conn.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: bCollUUID, shard: primaryName})
+ .count());
+assert.eq(0,
+ conn.getDB("config")
+ .getCollection("chunks")
+ .find({uuid: bCollUUID, shard: secondaryName})
+ .count());
+
+assert.eq(1, conn.getDB("config").getCollection("collections").find({_id: "test.a"}).count());
+assert.eq(0, conn.getDB("config").getCollection("collections").find({_id: "test.b"}).count());
+
+assert.eq(1, conn.getDB("config").getCollection("locks").find({_id: "test"}).count());
+assert.eq(1, conn.getDB("config").getCollection("locks").find({_id: "test.a"}).count());
+assert.eq(0, conn.getDB("config").getCollection("locks").find({_id: "test.b"}).count());
+assert.eq(0, conn.getDB("config").getCollection("locks").find({_id: "unusedDB"}).count());
+
+assert.eq(1, conn.getDB("config").getCollection("databases").find({_id: "test"}).count());
+assert.eq(0, conn.getDB("config").getCollection("databases").find({_id: "unusedDB"}).count());
+
+MongoRunner.stopMongod(conn);
+}());
diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp
index ec4d89e756f..ab4a511bc69 100644
--- a/src/mongo/db/namespace_string.cpp
+++ b/src/mongo/db/namespace_string.cpp
@@ -70,6 +70,9 @@ const NamespaceString NamespaceString::kSessionTransactionsTableNamespace(
const NamespaceString NamespaceString::kTransactionCoordinatorsNamespace(
NamespaceString::kConfigDb, "transaction_coordinators");
+const NamespaceString NamespaceString::kConfigsvrRestoreNamespace(NamespaceString::kLocalDb,
+ "system.collections_to_restore");
+
const NamespaceString NamespaceString::kMigrationCoordinatorsNamespace(NamespaceString::kConfigDb,
"migrationCoordinators");
@@ -186,6 +189,8 @@ bool NamespaceString::isLegalClientSystemNS(
return true;
if (coll() == "system.healthlog")
return true;
+ if (coll() == kConfigsvrRestoreNamespace.coll())
+ return true;
}
if (coll() == "system.users")
diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h
index c1d01550f90..505eb9d3130 100644
--- a/src/mongo/db/namespace_string.h
+++ b/src/mongo/db/namespace_string.h
@@ -210,6 +210,9 @@ public:
// Namespace for storing user write blocking critical section documents
static const NamespaceString kUserWritesCriticalSectionsNamespace;
+ // Namespace used during the recovery procedure for the config server.
+ static const NamespaceString kConfigsvrRestoreNamespace;
+
/**
* Constructs an empty NamespaceString.
*/
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 214f4b658fc..906836c0b7c 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -342,6 +342,7 @@ env.Library(
'config/configsvr_rename_collection_metadata_command.cpp',
'config/configsvr_reshard_collection_cmd.cpp',
'config/configsvr_repair_sharded_collection_chunks_history_command.cpp',
+ 'config/configsvr_run_restore_command.cpp',
'config/configsvr_set_allow_migrations_command.cpp',
'config/configsvr_split_chunk_command.cpp',
'config/configsvr_update_zone_key_range_command.cpp',
diff --git a/src/mongo/db/s/config/configsvr_run_restore_command.cpp b/src/mongo/db/s/config/configsvr_run_restore_command.cpp
new file mode 100644
index 00000000000..b7d11e7abd5
--- /dev/null
+++ b/src/mongo/db/s/config/configsvr_run_restore_command.cpp
@@ -0,0 +1,274 @@
+/**
+ * Copyright (C) 2022-present MongoDB, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the Server Side Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
+
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/catalog/create_collection.h"
+#include "mongo/db/catalog_raii.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/dbdirectclient.h"
+#include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl/storage_interface_impl.h"
+#include "mongo/logv2/log.h"
+#include "mongo/stdx/unordered_map.h"
+
+namespace mongo {
+namespace {
+
+enum class ShouldRestoreDocument { kYes, kMaybe, kNo };
+
+/**
+ * Given the nss and/or UUID of a config collection document, returns whether the document should be
+ * restored based on its presence in the local.system.collections_to_restore collection.
+ *
+ * If the given nss is referencing a database name only, returns maybe.
+ */
+ShouldRestoreDocument shouldRestoreDocument(OperationContext* opCtx,
+ boost::optional<NamespaceString> nss,
+ boost::optional<UUID> uuid) {
+ if (nss && nss->coll().empty()) {
+ return ShouldRestoreDocument::kMaybe;
+ }
+
+ auto findRequest = FindCommandRequest(NamespaceString::kConfigsvrRestoreNamespace);
+ if (nss && uuid) {
+ findRequest.setFilter(BSON("ns" << nss->toString() << "uuid" << *uuid));
+ } else if (nss) {
+ findRequest.setFilter(BSON("ns" << nss->toString()));
+ } else if (uuid) {
+ findRequest.setFilter(BSON("uuid" << *uuid));
+ }
+
+ findRequest.setLimit(1);
+
+ DBDirectClient client(opCtx);
+ return client.find(findRequest)->itcount() > 0 ? ShouldRestoreDocument::kYes
+ : ShouldRestoreDocument::kNo;
+}
+
+// { config collection namespace -> ( optional nss field name, optional UUID field name ) }
+const stdx::unordered_map<NamespaceString,
+ std::pair<boost::optional<std::string>, boost::optional<std::string>>>
+ kCollectionEntries = {
+ {NamespaceString("config.chunks"), std::make_pair(boost::none, std::string("uuid"))},
+ {NamespaceString("config.collections"),
+ std::make_pair(std::string("_id"), std::string("uuid"))},
+ {NamespaceString("config.locks"), std::make_pair(std::string("_id"), boost::none)},
+ {NamespaceString("config.migrationCoordinators"),
+ std::make_pair(std::string("nss"), std::string("collectionUuid"))},
+ {NamespaceString("config.tags"), std::make_pair(std::string("ns"), boost::none)},
+ {NamespaceString("config.rangeDeletions"),
+ std::make_pair(std::string("nss"), std::string("collectionUuid"))},
+ {NamespaceString("config.system.sharding_ddl_coordinators"),
+ std::make_pair(std::string("_id.namespace"), boost::none)}};
+
+class ConfigSvrRunResoreCommand : public BasicCommand {
+public:
+ ConfigSvrRunResoreCommand() : BasicCommand("_configsvrRunRestore") {}
+
+ bool skipApiVersionCheck() const override {
+ // Internal command used by the restore procedure.
+ return true;
+ }
+
+ AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+ return AllowedOnSecondary::kNever;
+ }
+
+ bool adminOnly() const {
+ return true;
+ }
+
+ bool supportsWriteConcern(const BSONObj& cmd) const override {
+ return false;
+ }
+
+ Status checkAuthForCommand(Client* client,
+ const std::string& dbname,
+ const BSONObj& cmdObj) const override {
+ if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource(
+ ResourcePattern::forClusterResource(), ActionType::internal)) {
+ return Status(ErrorCodes::Unauthorized, "Unauthorized");
+ }
+ return Status::OK();
+ }
+
+ bool run(OperationContext* opCtx,
+ const std::string& dbname_unused,
+ const BSONObj& cmdObj,
+ BSONObjBuilder& result) override {
+ uassert(ErrorCodes::CommandFailed,
+ "This command can only be used in standalone mode",
+ !repl::ReplicationCoordinator::get(opCtx)->getSettings().usingReplSets());
+
+ uassert(ErrorCodes::CommandFailed,
+ "This command can only be run during a restore procedure",
+ storageGlobalParams.restore);
+
+ {
+ // The "local.system.collections_to_restore" collection needs to exist prior to running
+ // this command.
+ CollectionPtr restoreColl = CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(
+ opCtx, NamespaceString::kConfigsvrRestoreNamespace);
+ uassert(ErrorCodes::NamespaceNotFound,
+ str::stream() << "Collection " << NamespaceString::kConfigsvrRestoreNamespace
+ << " is missing",
+ restoreColl);
+ }
+
+ // Keeps track of database names for collections restored. Databases with no collections
+ // restored will have their entries removed in the config collections.
+ std::set<std::string> databasesRestored;
+
+ for (const auto& collectionEntry : kCollectionEntries) {
+ const NamespaceString& nss = collectionEntry.first;
+
+ LOGV2(6261300, "1st Phase - Restoring collection entries", logAttrs(nss));
+ CollectionPtr coll =
+ CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, nss);
+ if (!coll) {
+ LOGV2(6261301, "Collection not found, skipping", logAttrs(nss));
+ continue;
+ }
+
+ DBDirectClient client(opCtx);
+ auto findRequest = FindCommandRequest(nss);
+ auto cursor = client.find(findRequest);
+
+ while (cursor->more()) {
+ auto doc = cursor->next();
+
+ boost::optional<std::string> nssFieldName = collectionEntry.second.first;
+ boost::optional<std::string> uuidFieldName = collectionEntry.second.second;
+
+ boost::optional<NamespaceString> docNss = boost::none;
+ boost::optional<UUID> docUUID = boost::none;
+
+ if (nssFieldName) {
+ const size_t dotPosition = nssFieldName->find(".");
+ if (dotPosition != std::string::npos) {
+ // Handles the "_id.namespace" case for collection
+ // "config.system.sharding_ddl_coordinators".
+ const auto obj = doc.getField(nssFieldName->substr(0, dotPosition)).Obj();
+ docNss = NamespaceString(
+ obj.getStringField(nssFieldName->substr(dotPosition + 1)));
+ } else {
+ docNss = NamespaceString(doc.getStringField(*nssFieldName));
+ }
+ }
+
+ if (uuidFieldName) {
+ auto swDocUUID = UUID::parse(doc.getField(*uuidFieldName));
+ uassertStatusOK(swDocUUID);
+
+ docUUID = swDocUUID.getValue();
+ }
+
+ ShouldRestoreDocument shouldRestore = shouldRestoreDocument(opCtx, docNss, docUUID);
+
+ LOGV2_DEBUG(6261302,
+ 1,
+ "Found document",
+ "doc"_attr = doc,
+ "shouldRestore"_attr = shouldRestore);
+
+ if (shouldRestore == ShouldRestoreDocument::kYes && docNss) {
+ databasesRestored.insert(docNss->db().toString());
+ }
+
+ if (shouldRestore == ShouldRestoreDocument::kYes ||
+ shouldRestore == ShouldRestoreDocument::kMaybe) {
+ continue;
+ }
+
+ // The collection for this document was not restored, delete it.
+ NamespaceStringOrUUID nssOrUUID(coll->ns().db().toString(), coll->uuid());
+ uassertStatusOK(repl::StorageInterface::get(opCtx)->deleteById(
+ opCtx, nssOrUUID, doc.getField("_id")));
+ }
+ }
+
+ {
+ const std::vector<NamespaceString> databasesEntries = {
+ NamespaceString("config.databases"), NamespaceString("config.locks")};
+
+ // Remove database entries from the config collections if no collection for the given
+ // database was restored.
+ for (const NamespaceString& nss : databasesEntries) {
+ LOGV2(6261303, "2nd Phase - Restoring database entries", logAttrs(nss));
+ CollectionPtr coll =
+ CollectionCatalog::get(opCtx)->lookupCollectionByNamespace(opCtx, nss);
+ if (!coll) {
+ LOGV2(6261304, "Collection not found, skipping", logAttrs(nss));
+ return true;
+ }
+
+ DBDirectClient client(opCtx);
+ auto findRequest = FindCommandRequest(nss);
+ auto cursor = client.find(findRequest);
+
+ while (cursor->more()) {
+ auto doc = cursor->next();
+
+ const NamespaceString dbNss = NamespaceString(doc.getStringField("_id"));
+ if (!dbNss.coll().empty()) {
+ // We want to handle database only namespaces.
+ continue;
+ }
+
+ bool shouldRestore =
+ databasesRestored.find(dbNss.db().toString()) != databasesRestored.end();
+
+ LOGV2_DEBUG(6261305,
+ 1,
+ "Found document",
+ "doc"_attr = doc,
+ "shouldRestore"_attr = shouldRestore);
+
+ if (shouldRestore) {
+ // This database had at least one collection restored.
+ continue;
+ }
+
+ // No collection for this database was restored, delete it.
+ NamespaceStringOrUUID nssOrUUID(coll->ns().db().toString(), coll->uuid());
+ uassertStatusOK(repl::StorageInterface::get(opCtx)->deleteById(
+ opCtx, nssOrUUID, doc.getField("_id")));
+ }
+ }
+ }
+
+ return true;
+ }
+
+} runRestoreCmd;
+
+} // namespace
+} // namespace mongo
View Lorry Controller Status