SERVER-25677 Clear cached clusterId if config.version document is rolled back.

10 files changed, 102 insertions, 10 deletions

diff --git a/src/mongo/s/catalog/replset/sharding_catalog_config_initialization_test.cpp b/src/mongo/s/catalog/replset/sharding_catalog_config_initialization_test.cpp
index 8d2b3595252..20d5bf39e5f 100644
--- a/src/mongo/s/catalog/replset/sharding_catalog_config_initialization_test.cpp
+++ b/src/mongo/s/catalog/replset/sharding_catalog_config_initialization_test.cpp
@@ -33,6 +33,8 @@
 
 #include "mongo/bson/json.h"
 #include "mongo/db/namespace_string.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/db/repl/replication_coordinator_mock.h"
 #include "mongo/s/catalog/config_server_version.h"
 #include "mongo/s/catalog/sharding_catalog_client.h"
 #include "mongo/s/catalog/sharding_catalog_manager.h"
@@ -44,6 +46,7 @@
 #include "mongo/s/catalog/type_tags.h"
 #include "mongo/s/client/shard.h"
 #include "mongo/s/config_server_test_fixture.h"
+#include "mongo/util/scopeguard.h"
 
 namespace mongo {
 namespace {
@@ -188,20 +191,55 @@ TEST_F(ConfigInitializationTest, OnlyRunsOnce) {
     ASSERT_EQUALS(CURRENT_CONFIG_VERSION, foundVersion.getCurrentVersion());
     ASSERT_EQUALS(MIN_COMPATIBLE_CONFIG_VERSION, foundVersion.getMinCompatibleVersion());
 
-    // Now remove the version document and re-run initializeConfigDatabaseIfNeeded().
-    ASSERT_OK(catalogClient()->removeConfigDocuments(operationContext(),
-                                                     VersionType::ConfigNS,
-                                                     BSONObj(),
-                                                     ShardingCatalogClient::kMajorityWriteConcern));
-
     ASSERT_EQUALS(ErrorCodes::AlreadyInitialized,
                   catalogManager()->initializeConfigDatabaseIfNeeded(operationContext()));
+}
+
+TEST_F(ConfigInitializationTest, ReRunsIfDocRolledBackThenReElected) {
+    ASSERT_OK(catalogManager()->initializeConfigDatabaseIfNeeded(operationContext()));
+
+    auto versionDoc = assertGet(findOneOnConfigCollection(
+        operationContext(), NamespaceString(VersionType::ConfigNS), BSONObj()));
+
+    VersionType foundVersion = assertGet(VersionType::fromBSON(versionDoc));
 
-    // Even though there was no version document, initializeConfigDatabaseIfNeeded() returned
-    // without making one because it has already run once successfully so didn't bother to check.
+    ASSERT_TRUE(foundVersion.getClusterId().isSet());
+    ASSERT_EQUALS(CURRENT_CONFIG_VERSION, foundVersion.getCurrentVersion());
+    ASSERT_EQUALS(MIN_COMPATIBLE_CONFIG_VERSION, foundVersion.getMinCompatibleVersion());
+
+    // Now remove the version document and re-run initializeConfigDatabaseIfNeeded().
+    {
+        // Mirror what happens if the config.version document is rolled back.
+        ON_BLOCK_EXIT([&] {
+            operationContext()->setReplicatedWrites(true);
+            getReplicationCoordinator()->setFollowerMode(repl::MemberState::RS_PRIMARY);
+        });
+        operationContext()->setReplicatedWrites(false);
+        getReplicationCoordinator()->setFollowerMode(repl::MemberState::RS_ROLLBACK);
+        ASSERT_OK(
+            catalogClient()->removeConfigDocuments(operationContext(),
+                                                   VersionType::ConfigNS,
+                                                   BSONObj(),
+                                                   ShardingCatalogClient::kMajorityWriteConcern));
+    }
+
+    // Verify the document was actually removed.
     ASSERT_EQUALS(ErrorCodes::NoMatchingDocument,
                   findOneOnConfigCollection(
                       operationContext(), NamespaceString(VersionType::ConfigNS), BSONObj()));
+
+    // Re-create the config.version document.
+    ASSERT_OK(catalogManager()->initializeConfigDatabaseIfNeeded(operationContext()));
+
+    auto newVersionDoc = assertGet(findOneOnConfigCollection(
+        operationContext(), NamespaceString(VersionType::ConfigNS), BSONObj()));
+
+    VersionType newFoundVersion = assertGet(VersionType::fromBSON(newVersionDoc));
+
+    ASSERT_TRUE(newFoundVersion.getClusterId().isSet());
+    ASSERT_NOT_EQUALS(newFoundVersion.getClusterId(), foundVersion.getClusterId());
+    ASSERT_EQUALS(CURRENT_CONFIG_VERSION, newFoundVersion.getCurrentVersion());
+    ASSERT_EQUALS(MIN_COMPATIBLE_CONFIG_VERSION, newFoundVersion.getMinCompatibleVersion());
 }
 
 TEST_F(ConfigInitializationTest, BuildsNecessaryIndexes) {
diff --git a/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.cpp b/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.cpp
index aac7bed4935..2b982106dc5 100644
--- a/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.cpp
+++ b/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.cpp
@@ -1408,12 +1408,15 @@ Status ShardingCatalogManagerImpl::initializeConfigDatabaseIfNeeded(OperationCon
         }
     }
 
-    Status status = _initConfigVersion(txn);
+    Status status = _initConfigIndexes(txn);
     if (!status.isOK()) {
         return status;
     }
 
-    status = _initConfigIndexes(txn);
+    // Make sure to write config.version last since we detect rollbacks of config.version and
+    // will re-run initializeConfigDatabaseIfNeeded if that happens, but we don't detect rollback
+    // of the index builds.
+    status = _initConfigVersion(txn);
     if (!status.isOK()) {
         return status;
     }
@@ -1424,6 +1427,11 @@ Status ShardingCatalogManagerImpl::initializeConfigDatabaseIfNeeded(OperationCon
     return Status::OK();
 }
 
+void ShardingCatalogManagerImpl::discardCachedConfigDatabaseInitializationState() {
+    stdx::lock_guard<stdx::mutex> lk(_mutex);
+    _configInitialized = false;
+}
+
 Status ShardingCatalogManagerImpl::_initConfigVersion(OperationContext* txn) {
     auto versionStatus =
         _catalogClient->getConfigVersion(txn, repl::ReadConcernLevel::kLocalReadConcern);
diff --git a/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.h b/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.h
index 516ab7aca52..aa4c437151b 100644
--- a/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.h
+++ b/src/mongo/s/catalog/replset/sharding_catalog_manager_impl.h
@@ -104,6 +104,8 @@ public:
 
     Status initializeConfigDatabaseIfNeeded(OperationContext* txn) override;
 
+    void discardCachedConfigDatabaseInitializationState() override;
+
     Status initializeShardingAwarenessOnUnawareShards(OperationContext* txn) override;
 
     Status upsertShardIdentityOnShard(OperationContext* txn, ShardType shardType) override;
diff --git a/src/mongo/s/catalog/sharding_catalog_manager.h b/src/mongo/s/catalog/sharding_catalog_manager.h
index 3a0c9c34a74..63f242d3598 100644
--- a/src/mongo/s/catalog/sharding_catalog_manager.h
+++ b/src/mongo/s/catalog/sharding_catalog_manager.h
@@ -170,6 +170,14 @@ public:
     virtual Status initializeConfigDatabaseIfNeeded(OperationContext* txn) = 0;
 
     /**
+     * Called if the config.version document is rolled back.  Indicates to the
+     * ShardingCatalogManager that on the next transition to primary
+     * initializeConfigDatabaseIfNeeded will need to re-run the work to initialize the config
+     * database.
+     */
+    virtual void discardCachedConfigDatabaseInitializationState() = 0;
+
+    /**
      * For upgrade from 3.2 to 3.4, for each shard in config.shards that is not marked as sharding
      * aware, schedules a task to upsert a shardIdentity doc into the shard and mark the shard as
      * sharding aware.
diff --git a/src/mongo/s/catalog/sharding_catalog_manager_mock.cpp b/src/mongo/s/catalog/sharding_catalog_manager_mock.cpp
index c9549bc32a4..cc53dfbada1 100644
--- a/src/mongo/s/catalog/sharding_catalog_manager_mock.cpp
+++ b/src/mongo/s/catalog/sharding_catalog_manager_mock.cpp
@@ -104,6 +104,8 @@ Status ShardingCatalogManagerMock::initializeConfigDatabaseIfNeeded(OperationCon
     return {ErrorCodes::InternalError, "Method not implemented"};
 }
 
+void ShardingCatalogManagerMock::discardCachedConfigDatabaseInitializationState() {}
+
 Status ShardingCatalogManagerMock::initializeShardingAwarenessOnUnawareShards(
     OperationContext* txn) {
     return {ErrorCodes::InternalError, "Method not implemented"};
diff --git a/src/mongo/s/catalog/sharding_catalog_manager_mock.h b/src/mongo/s/catalog/sharding_catalog_manager_mock.h
index 88d304385c7..23ab27831da 100644
--- a/src/mongo/s/catalog/sharding_catalog_manager_mock.h
+++ b/src/mongo/s/catalog/sharding_catalog_manager_mock.h
@@ -86,6 +86,8 @@ public:
 
     Status initializeConfigDatabaseIfNeeded(OperationContext* txn) override;
 
+    void discardCachedConfigDatabaseInitializationState() override;
+
     Status initializeShardingAwarenessOnUnawareShards(OperationContext* txn) override;
 
     Status upsertShardIdentityOnShard(OperationContext* txn, ShardType shardType) override;
diff --git a/src/mongo/s/cluster_identity_loader.cpp b/src/mongo/s/cluster_identity_loader.cpp
index dc2ae335f94..741a280ab4c 100644
--- a/src/mongo/s/cluster_identity_loader.cpp
+++ b/src/mongo/s/cluster_identity_loader.cpp
@@ -105,4 +105,16 @@ StatusWith<OID> ClusterIdentityLoader::_fetchClusterIdFromConfig(
     return loadResult.getValue().getClusterId();
 }
 
+void ClusterIdentityLoader::discardCachedClusterId() {
+    stdx::lock_guard<stdx::mutex> lk(_mutex);
+
+    if (_initializationState == InitializationState::kUninitialized) {
+        return;
+    }
+    invariant(_initializationState == InitializationState::kInitialized);
+    _lastLoadResult = {
+        Status{ErrorCodes::InternalError, "cluster ID never re-loaded after rollback"}};
+    _initializationState = InitializationState::kUninitialized;
+}
+
 }  // namespace mongo
diff --git a/src/mongo/s/cluster_identity_loader.h b/src/mongo/s/cluster_identity_loader.h
index 25645111ef1..d34f5368850 100644
--- a/src/mongo/s/cluster_identity_loader.h
+++ b/src/mongo/s/cluster_identity_loader.h
@@ -73,6 +73,12 @@ public:
      */
     Status loadClusterId(OperationContext* txn, const repl::ReadConcernLevel& readConcernLevel);
 
+    /**
+     * Called if the config.version document is rolled back.  Notifies the ClusterIdentityLoader
+     * that the cached cluster ID is invalid and needs to be reloaded.
+     */
+    void discardCachedClusterId();
+
 private:
     enum class InitializationState {
         kUninitialized,  // We have never successfully loaded the cluster ID
diff --git a/src/mongo/s/config_server_test_fixture.cpp b/src/mongo/s/config_server_test_fixture.cpp
index ed0f4810307..70b391e7077 100644
--- a/src/mongo/s/config_server_test_fixture.cpp
+++ b/src/mongo/s/config_server_test_fixture.cpp
@@ -109,6 +109,7 @@ void ConfigServerTestFixture::setUp() {
     repl::ReplSettings replSettings;
     replSettings.setReplSetString("mySet/node1:12345,node2:54321,node3:12543");
     auto replCoord = stdx::make_unique<repl::ReplicationCoordinatorMock>(replSettings);
+    _replCoord = replCoord.get();
 
     repl::ReplicaSetConfig config;
     config.initialize(BSON("_id"
@@ -307,6 +308,12 @@ OperationContext* ConfigServerTestFixture::operationContext() const {
     return _opCtx.get();
 }
 
+repl::ReplicationCoordinatorMock* ConfigServerTestFixture::getReplicationCoordinator() const {
+    invariant(_replCoord);
+
+    return _replCoord;
+}
+
 void ConfigServerTestFixture::onCommand(NetworkTestEnv::OnCommandFunction func) {
     _networkTestEnv->onCommand(func);
 }
diff --git a/src/mongo/s/config_server_test_fixture.h b/src/mongo/s/config_server_test_fixture.h
index fc929ca5a43..943b6d122b6 100644
--- a/src/mongo/s/config_server_test_fixture.h
+++ b/src/mongo/s/config_server_test_fixture.h
@@ -63,6 +63,10 @@ class NetworkInterfaceMock;
 class TaskExecutor;
 }  // namespace executor
 
+namespace repl {
+class ReplicationCoordinatorMock;
+}
+
 /**
  * Sets up the mocked out objects for testing the catalog manager and catalog client with the
  * remote interface backed by the NetworkTestEnv and config server as the local storage engine.
@@ -109,6 +113,8 @@ public:
 
     OperationContext* operationContext() const;
 
+    repl::ReplicationCoordinatorMock* getReplicationCoordinator() const;
+
     /**
      * Insert a document to this config server to the specified namespace.
      */
@@ -191,6 +197,7 @@ private:
     ReplSetDistLockManager* _distLockManager = nullptr;
     ShardingCatalogClientImpl* _catalogClient = nullptr;
     ShardingCatalogManagerImpl* _catalogManager = nullptr;
+    repl::ReplicationCoordinatorMock* _replCoord = nullptr;
 };
 
 }  // namespace mongo