SERVER-46995 convert ReplIndexBuildState to class and un-inline. add methods for resumable index builds

4 files changed, 172 insertions, 78 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 826cafe408a..957a3f9e982 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -926,6 +926,7 @@ env.Library(
     target="index_builds_coordinator_interface",
     source=[
         "index_builds_coordinator.cpp",
+        "repl_index_build_state.cpp",
     ],
     LIBDEPS=[
         "$BUILD_DIR/mongo/base",
diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp
index 74d1f2ddcab..1e0e2afe127 100644
--- a/src/mongo/db/index_builds_coordinator.cpp
+++ b/src/mongo/db/index_builds_coordinator.cpp
@@ -469,9 +469,8 @@ bool isIndexBuildResumable(OperationContext* opCtx,
  */
 RecoveryUnit::ReadSource getReadSourceForDrainBeforeCommitQuorum(
     const ReplIndexBuildState& replState) {
-    return replState.lastOpTimeBeforeInterceptors.isNull()
-        ? RecoveryUnit::ReadSource::kNoTimestamp
-        : RecoveryUnit::ReadSource::kMajorityCommitted;
+    return replState.isResumable() ? RecoveryUnit::ReadSource::kMajorityCommitted
+                                   : RecoveryUnit::ReadSource::kNoTimestamp;
 }
 
 }  // namespace
@@ -1342,9 +1341,8 @@ bool IndexBuildsCoordinator::abortIndexBuildByBuildUUID(OperationContext* opCtx,
             // No locks are required when aborting due to rollback. This performs no storage engine
             // writes, only cleans up the remaining in-memory state.
             CollectionWriter coll(opCtx, replState->collectionUUID);
-            auto isResumable = !replState->lastOpTimeBeforeInterceptors.isNull();
             _indexBuildsManager.abortIndexBuildWithoutCleanup(
-                opCtx, coll.get(), replState->buildUUID, isResumable);
+                opCtx, coll.get(), replState->buildUUID, replState->isResumable());
         }
 
         {
@@ -1466,9 +1464,8 @@ void IndexBuildsCoordinator::_completeAbortForShutdown(
     std::shared_ptr<ReplIndexBuildState> replState,
     const CollectionPtr& collection) {
     // Leave it as-if kill -9 happened. Startup recovery will restart the index build.
-    auto isResumable = !replState->lastOpTimeBeforeInterceptors.isNull();
     _indexBuildsManager.abortIndexBuildWithoutCleanup(
-        opCtx, collection, replState->buildUUID, isResumable);
+        opCtx, collection, replState->buildUUID, replState->isResumable());
 
     {
         // Promise should be set at least once before it's getting destroyed.
@@ -2218,7 +2215,7 @@ IndexBuildsCoordinator::PostSetupAction IndexBuildsCoordinator::_setUpIndexBuild
         // After the interceptors are set, get the latest optime in the oplog that could have
         // contained a write to this collection. We need to be holding the collection lock in X mode
         // so that we ensure that there are not any uncommitted transactions on this collection.
-        replState->lastOpTimeBeforeInterceptors = getLatestOplogOpTime(opCtx);
+        replState->setLastOpTimeBeforeInterceptors(getLatestOplogOpTime(opCtx));
     }
 
     return PostSetupAction::kContinueIndexBuild;
@@ -2522,15 +2519,15 @@ void IndexBuildsCoordinator::_awaitLastOpTimeBeforeInterceptorsMajorityCommitted
     OperationContext* opCtx, std::shared_ptr<ReplIndexBuildState> replState) {
     auto replCoord = repl::ReplicationCoordinator::get(opCtx);
 
-    // The last optime could be null if the node is in initial sync while building the index.
-    if (replState->lastOpTimeBeforeInterceptors.isNull()) {
+    // The index build is not resumable if the node is in initial sync while building the index.
+    if (!replState->isResumable()) {
         return;
     }
 
     auto timeoutMillis = gResumableIndexBuildMajorityOpTimeTimeoutMillis;
     if (timeoutMillis == 0) {
         // Disable resumable index build.
-        replState->lastOpTimeBeforeInterceptors = {};
+        replState->clearLastOpTimeBeforeInterceptors();
         return;
     }
 
@@ -2549,12 +2546,13 @@ void IndexBuildsCoordinator::_awaitLastOpTimeBeforeInterceptorsMajorityCommitted
         deadline = Date_t::max();
     }
 
+    auto lastOpTimeBeforeInterceptors = replState->getLastOpTimeBeforeInterceptors();
     LOGV2(4847600,
           "Index build: waiting for last optime before interceptors to be majority committed",
           "buildUUID"_attr = replState->buildUUID,
           "deadline"_attr = deadline,
           "timeout"_attr = timeout,
-          "lastOpTime"_attr = replState->lastOpTimeBeforeInterceptors);
+          "lastOpTime"_attr = lastOpTimeBeforeInterceptors);
 
     hangIndexBuildBeforeWaitingUntilMajorityOpTime.executeIf(
         [opCtx, buildUUID = replState->buildUUID](const BSONObj& data) {
@@ -2574,17 +2572,16 @@ void IndexBuildsCoordinator::_awaitLastOpTimeBeforeInterceptorsMajorityCommitted
             });
         });
 
-    auto status = replCoord->waitUntilMajorityOpTime(
-        opCtx, replState->lastOpTimeBeforeInterceptors, deadline);
+    auto status = replCoord->waitUntilMajorityOpTime(opCtx, lastOpTimeBeforeInterceptors, deadline);
     if (!status.isOK()) {
-        replState->lastOpTimeBeforeInterceptors = {};
+        replState->clearLastOpTimeBeforeInterceptors();
         LOGV2(5053900,
               "Index build: timed out waiting for the last optime before interceptors to be "
               "majority committed. Continuing as a non-resumable index build.",
               "buildUUID"_attr = replState->buildUUID,
               "deadline"_attr = deadline,
               "timeout"_attr = timeout,
-              "lastOpTime"_attr = replState->lastOpTimeBeforeInterceptors,
+              "lastOpTime"_attr = lastOpTimeBeforeInterceptors,
               "waitStatus"_attr = status);
         return;
     }
diff --git a/src/mongo/db/repl_index_build_state.cpp b/src/mongo/db/repl_index_build_state.cpp
new file mode 100644
index 00000000000..a7617172312
--- /dev/null
+++ b/src/mongo/db/repl_index_build_state.cpp
@@ -0,0 +1,125 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/repl_index_build_state.h"
+
+namespace mongo {
+
+namespace {
+
+/**
+ * Parses index specs to generate list of index names for ReplIndexBuildState initialization.
+ */
+std::vector<std::string> extractIndexNames(const std::vector<BSONObj>& specs) {
+    std::vector<std::string> indexNames;
+    for (const auto& spec : specs) {
+        std::string name = spec.getStringField(IndexDescriptor::kIndexNameFieldName);
+        invariant(!name.empty(),
+                  str::stream() << "Bad spec passed into ReplIndexBuildState constructor, missing '"
+                                << IndexDescriptor::kIndexNameFieldName << "' field: " << spec);
+        indexNames.push_back(name);
+    }
+    return indexNames;
+}
+
+/**
+ * Returns true if the requested IndexBuildState transition is allowed.
+ */
+bool checkIfValidTransition(IndexBuildState::StateFlag currentState,
+                            IndexBuildState::StateFlag newState) {
+    if ((currentState == IndexBuildState::StateFlag::kSetup &&
+         newState == IndexBuildState::StateFlag::kInProgress) ||
+        (currentState == IndexBuildState::StateFlag::kInProgress &&
+         newState != IndexBuildState::StateFlag::kSetup) ||
+        (currentState == IndexBuildState::StateFlag::kPrepareCommit &&
+         newState == IndexBuildState::StateFlag::kCommitted)) {
+        return true;
+    }
+    return false;
+}
+
+}  // namespace
+
+void IndexBuildState::setState(StateFlag state,
+                               bool skipCheck,
+                               boost::optional<Timestamp> timestamp,
+                               boost::optional<std::string> abortReason) {
+    if (!skipCheck) {
+        invariant(checkIfValidTransition(_state, state),
+                  str::stream() << "current state :" << toString(_state)
+                                << ", new state: " << toString(state));
+    }
+    _state = state;
+    if (timestamp)
+        _timestamp = timestamp;
+    if (abortReason) {
+        invariant(_state == kAborted);
+        _abortReason = abortReason;
+    }
+}
+
+ReplIndexBuildState::ReplIndexBuildState(const UUID& indexBuildUUID,
+                                         const UUID& collUUID,
+                                         const std::string& dbName,
+                                         const std::vector<BSONObj>& specs,
+                                         IndexBuildProtocol protocol)
+    : buildUUID(indexBuildUUID),
+      collectionUUID(collUUID),
+      dbName(dbName),
+      indexNames(extractIndexNames(specs)),
+      indexSpecs(specs),
+      protocol(protocol) {
+    waitForNextAction = std::make_unique<SharedPromise<IndexBuildAction>>();
+    if (protocol == IndexBuildProtocol::kTwoPhase)
+        commitQuorumLock.emplace(indexBuildUUID.toString());
+}
+
+bool ReplIndexBuildState::isResumable() const {
+    stdx::unique_lock<Latch> lk(mutex);
+    return !_lastOpTimeBeforeInterceptors.isNull();
+}
+
+repl::OpTime ReplIndexBuildState::getLastOpTimeBeforeInterceptors() const {
+    stdx::unique_lock<Latch> lk(mutex);
+    return _lastOpTimeBeforeInterceptors;
+}
+
+void ReplIndexBuildState::setLastOpTimeBeforeInterceptors(repl::OpTime opTime) {
+    stdx::unique_lock<Latch> lk(mutex);
+    _lastOpTimeBeforeInterceptors = std::move(opTime);
+}
+
+void ReplIndexBuildState::clearLastOpTimeBeforeInterceptors() {
+    stdx::unique_lock<Latch> lk(mutex);
+    _lastOpTimeBeforeInterceptors = {};
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/repl_index_build_state.h b/src/mongo/db/repl_index_build_state.h
index bc27a80adce..fde82963b1f 100644
--- a/src/mongo/db/repl_index_build_state.h
+++ b/src/mongo/db/repl_index_build_state.h
@@ -140,37 +140,16 @@ public:
         kAborted = 1 << 4,
     };
 
-    using StateSet = int;
-    bool isSet(StateSet stateSet) const {
-        return _state & stateSet;
-    }
-
-    bool checkIfValidTransition(StateFlag newState) {
-        if ((_state == kSetup && newState == kInProgress) ||
-            (_state == kInProgress && newState != kSetup) ||
-            (_state == kPrepareCommit && newState == kCommitted)) {
-            return true;
-        }
-        return false;
-    }
-
+    /**
+     * Transitions this index build to new 'state'.
+     * Invariants if the requested transition is not valid and 'skipCheck' is true.
+     * 'timestamp' and 'abortReason' may be provided for certain states such as 'commit' and
+     * 'abort'.
+     */
     void setState(StateFlag state,
                   bool skipCheck,
                   boost::optional<Timestamp> timestamp = boost::none,
-                  boost::optional<std::string> abortReason = boost::none) {
-        if (!skipCheck) {
-            invariant(checkIfValidTransition(state),
-                      str::stream() << "current state :" << toString(_state)
-                                    << ", new state: " << toString(state));
-        }
-        _state = state;
-        if (timestamp)
-            _timestamp = timestamp;
-        if (abortReason) {
-            invariant(_state == kAborted);
-            _abortReason = abortReason;
-        }
-    }
+                  boost::optional<std::string> abortReason = boost::none);
 
     bool isCommitPrepared() const {
         return _state == kPrepareCommit;
@@ -234,22 +213,26 @@ private:
  *
  * TODO: pass in commit quorum setting.
  */
-struct ReplIndexBuildState {
+class ReplIndexBuildState {
+    ReplIndexBuildState(const ReplIndexBuildState&) = delete;
+    ReplIndexBuildState& operator=(const ReplIndexBuildState&) = delete;
+
+public:
     ReplIndexBuildState(const UUID& indexBuildUUID,
                         const UUID& collUUID,
                         const std::string& dbName,
                         const std::vector<BSONObj>& specs,
-                        IndexBuildProtocol protocol)
-        : buildUUID(indexBuildUUID),
-          collectionUUID(collUUID),
-          dbName(dbName),
-          indexNames(extractIndexNames(specs)),
-          indexSpecs(specs),
-          protocol(protocol) {
-        waitForNextAction = std::make_unique<SharedPromise<IndexBuildAction>>();
-        if (protocol == IndexBuildProtocol::kTwoPhase)
-            commitQuorumLock.emplace(indexBuildUUID.toString());
-    }
+                        IndexBuildProtocol protocol);
+
+    /**
+     * Accessor and mutator for last optime in the oplog before the interceptors were installed.
+     * This supports resumable index builds.
+     */
+    bool isResumable() const;
+    repl::OpTime getLastOpTimeBeforeInterceptors() const;
+    void setLastOpTimeBeforeInterceptors(repl::OpTime opTime);
+    void clearLastOpTimeBeforeInterceptors();
+
 
     // Uniquely identifies this index build across replica set members.
     const UUID buildUUID;
@@ -273,13 +256,6 @@ struct ReplIndexBuildState {
     // at the start of the index build will determine this setting.
     const IndexBuildProtocol protocol;
 
-    // Protects the state below.
-    mutable Mutex mutex = MONGO_MAKE_LATCH("ReplIndexBuildState::mutex");
-
-    // The OperationId of the index build. This allows external callers to interrupt the index build
-    // thread.
-    OperationId opId = 0;
-
     /*
      * Readers who read the commit quorum value from "config.system.indexBuilds" collection
      * to decide if the commit quorum got satisfied for an index build, should take this lock in
@@ -300,12 +276,16 @@ struct ReplIndexBuildState {
     };
 
     // Tracks the index build stats that are returned to the caller upon success.
+    // Used only by the thread pool task for the index build. No synchronization necessary.
     IndexCatalogStats stats;
 
     // Communicates the final outcome of the index build to any callers waiting upon the associated
     // SharedSemiFuture(s).
     SharedPromise<IndexCatalogStats> sharedPromise;
 
+    // Protects the state below.
+    mutable Mutex mutex = MONGO_MAKE_LATCH("ReplIndexBuildState::mutex");
+
     // Primary and secondaries gets their commit or abort signal via this promise future pair.
     std::unique_ptr<SharedPromise<IndexBuildAction>> waitForNextAction;
 
@@ -315,24 +295,15 @@ struct ReplIndexBuildState {
     // Represents the callback handle for scheduled remote command "voteCommitIndexBuild".
     executor::TaskExecutor::CallbackHandle voteCmdCbkHandle;
 
+    // The OperationId of the index build. This allows external callers to interrupt the index build
+    // thread.
+    OperationId opId = 0;
+
+private:
     // The last optime in the oplog before the interceptors were installed. If this is a single
     // phase index build, isn't running a hybrid index build, or isn't running during oplog
     // application, this will be null.
-    repl::OpTime lastOpTimeBeforeInterceptors;
-
-private:
-    std::vector<std::string> extractIndexNames(const std::vector<BSONObj>& specs) {
-        std::vector<std::string> indexNames;
-        for (const auto& spec : specs) {
-            std::string name = spec.getStringField(IndexDescriptor::kIndexNameFieldName);
-            invariant(!name.empty(),
-                      str::stream()
-                          << "Bad spec passed into ReplIndexBuildState constructor, missing '"
-                          << IndexDescriptor::kIndexNameFieldName << "' field: " << spec);
-            indexNames.push_back(name);
-        }
-        return indexNames;
-    }
+    repl::OpTime _lastOpTimeBeforeInterceptors;
 };
 
 }  // namespace mongo