SERVER-23802 Shut down shard if it rolls back the shardIdentity document.

5 files changed, 156 insertions, 5 deletions

diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 072bf231adf..0a5d9ed80e5 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -57,6 +57,7 @@ env.Library(
         'migration_source_manager.cpp',
         'move_timing_helper.cpp',
         'operation_sharding_state.cpp',
+        'shard_identity_rollback_notifier.cpp',
         'sharded_connection_info.cpp',
         'sharding_initialization_mongod.cpp',
         'sharding_state.cpp',
diff --git a/src/mongo/db/s/collection_sharding_state.cpp b/src/mongo/db/s/collection_sharding_state.cpp
index 01fc23bd9b1..a6d1fa3c47a 100644
--- a/src/mongo/db/s/collection_sharding_state.cpp
+++ b/src/mongo/db/s/collection_sharding_state.cpp
@@ -40,6 +40,7 @@
 #include "mongo/db/s/migration_chunk_cloner_source.h"
 #include "mongo/db/s/migration_source_manager.h"
 #include "mongo/db/s/operation_sharding_state.h"
+#include "mongo/db/s/shard_identity_rollback_notifier.h"
 #include "mongo/db/s/sharded_connection_info.h"
 #include "mongo/db/s/sharding_state.h"
 #include "mongo/db/s/type_shard_identity.h"
@@ -50,6 +51,7 @@
 #include "mongo/s/chunk_version.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/stale_exception.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 
@@ -273,12 +275,23 @@ void CollectionShardingState::onDeleteOp(OperationContext* txn,
                                          const CollectionShardingState::DeleteState& deleteState) {
     dassert(txn->lockState()->isCollectionLockedForMode(_nss.ns(), MODE_IX));
 
-    if (txn->writesAreReplicated() && serverGlobalParams.clusterRole == ClusterRole::ShardServer &&
+    if (serverGlobalParams.clusterRole == ClusterRole::ShardServer &&
         _nss == NamespaceString::kConfigCollectionNamespace) {
+
         if (auto idElem = deleteState.idDoc["_id"]) {
-            uassert(40070,
-                    "cannot delete shardIdentity document while in --shardsvr mode",
-                    idElem.str() != ShardIdentityType::IdName);
+            auto idStr = idElem.str();
+            if (idStr == ShardIdentityType::IdName) {
+                if (txn->writesAreReplicated()) {
+                    uasserted(40070,
+                              "cannot delete shardIdentity document while in --shardsvr mode");
+                } else {
+                    if (repl::ReplicationCoordinator::get(txn)->getMemberState().rollback()) {
+                        warning() << "Shard identity document rolled back.  Will shut down after "
+                                     "finishing rollback.";
+                        ShardIdentityRollbackNotifier::get(txn)->recordThatRollbackHappened();
+                    }
+                }
+            }
         }
     }
 
diff --git a/src/mongo/db/s/operation_sharding_state.cpp b/src/mongo/db/s/operation_sharding_state.cpp
index 5d0cad65b52..b1ba9e0e16f 100644
--- a/src/mongo/db/s/operation_sharding_state.cpp
+++ b/src/mongo/db/s/operation_sharding_state.cpp
@@ -42,7 +42,7 @@ const OperationContext::Decoration<OperationShardingState> shardingMetadataDecor
 // Max time to wait for the migration critical section to complete
 const Microseconds kMaxWaitForMigrationCriticalSection = Minutes(5);
 
-}  // namespace mongo
+}  // namespace
 
 OperationShardingState::OperationShardingState() = default;
 
diff --git a/src/mongo/db/s/shard_identity_rollback_notifier.cpp b/src/mongo/db/s/shard_identity_rollback_notifier.cpp
new file mode 100644
index 00000000000..b78efaa6fef
--- /dev/null
+++ b/src/mongo/db/s/shard_identity_rollback_notifier.cpp
@@ -0,0 +1,52 @@
+/**
+ *    Copyright (C) 2016 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/shard_identity_rollback_notifier.h"
+
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+
+namespace {
+const auto getRollbackNotifier = ServiceContext::declareDecoration<ShardIdentityRollbackNotifier>();
+}  // namespace
+
+ShardIdentityRollbackNotifier::ShardIdentityRollbackNotifier() = default;
+
+ShardIdentityRollbackNotifier* ShardIdentityRollbackNotifier::get(OperationContext* txn) {
+    return get(txn->getServiceContext());
+}
+
+ShardIdentityRollbackNotifier* ShardIdentityRollbackNotifier::get(ServiceContext* txn) {
+    return &getRollbackNotifier(txn);
+}
+
+
+}  // namespace mongo
diff --git a/src/mongo/db/s/shard_identity_rollback_notifier.h b/src/mongo/db/s/shard_identity_rollback_notifier.h
new file mode 100644
index 00000000000..a8bb6592350
--- /dev/null
+++ b/src/mongo/db/s/shard_identity_rollback_notifier.h
@@ -0,0 +1,85 @@
+/*
+*    Copyright (C) 2016 MongoDB Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*    As a special exception, the copyright holders give permission to link the
+*    code of portions of this program with the OpenSSL library under certain
+*    conditions as described in each individual source file and distribute
+*    linked combinations including the program with the OpenSSL library. You
+*    must comply with the GNU Affero General Public License in all respects for
+*    all of the code used other than as permitted herein. If you modify file(s)
+*    with this exception, you may extend this exception to your version of the
+*    file(s), but you are not obligated to do so. If you do not wish to do so,
+*    delete this exception statement from your version. If you delete this
+*    exception statement from all source files in the program, then also delete
+*    it in the license file.
+*/
+
+#pragma once
+
+#include "mongo/base/disallow_copying.h"
+
+namespace mongo {
+
+class OperationContext;
+class ServiceContext;
+
+/**
+ * Used to detect when a shard's shardIdentity document has been rolled back.  Since rolling back
+ * the shardIdentity document is illegal (as we can't clear the in-memory state associated with it),
+ * we force the mongod to shut down if it happens.  We detect shardIdentity rollback by checking in
+ * the OpObserver for that document to be deleted.  We can't shut down right at that moment,
+ * however, as doing so would interrupt the rollback process and leave the document in place for
+ * when the server is restarted.  Instead, when we detect the shardIdentity document being deleted,
+ * we call recordThatRollbackHappened() on this class, which records the fact that the shardIdentity
+ * document was deleted as part of the rollback, and then when we exit rollback we check
+ * didRollbackHappen() and shut down if so.
+ *
+ * No concurrency control is needed in this class as recordThatRollbackHappened can only happen in
+ * an OpObserver detecting that the document was deleted, and didRollbackHappen is only called on
+ * exiting rollback, and there can only be one of those things happening at any given time.
+ */
+class ShardIdentityRollbackNotifier {
+    MONGO_DISALLOW_COPYING(ShardIdentityRollbackNotifier);
+
+public:
+    ShardIdentityRollbackNotifier();
+
+    /**
+     * Retrieves the ShardIdentityRollbackNotifier associated with the specified service context.
+     */
+    static ShardIdentityRollbackNotifier* get(OperationContext* txn);
+    static ShardIdentityRollbackNotifier* get(ServiceContext* txn);
+
+    /**
+     * Records the fact that the shardIdentity document was rolled back.
+     */
+    void recordThatRollbackHappened() {
+        _rollbackHappened = true;
+    }
+
+    /**
+     * Checks whether the shardIdentity document was rolled back.
+     */
+    bool didRollbackHappen() {
+        return _rollbackHappened;
+    }
+
+private:
+    // Whether or not a rollback of the shardIdentity document has been detected.
+    // No concurrency control necessary since this is only consulted during
+    bool _rollbackHappened = false;
+};
+
+}  // namespace mongo