SERVER-40882 Restore the --noIndexBuildRetry server parameter flag and the storage.indexBuildRetry config file option

6 files changed, 64 insertions, 3 deletions

diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp
index 62d28f73b50..cdbff5a8d20 100644
--- a/src/mongo/db/mongod_options.cpp
+++ b/src/mongo/db/mongod_options.cpp
@@ -243,6 +243,20 @@ Status canonicalizeMongodOptions(moe::Environment* params) {
         }
     }
 
+    // "storage.indexBuildRetry" comes from the config file, so override it if "noIndexBuildRetry"
+    // is set since that comes from the command line.
+    if (params->count("noIndexBuildRetry")) {
+        Status ret = params->set("storage.indexBuildRetry",
+                                 moe::Value(!(*params)["noIndexBuildRetry"].as<bool>()));
+        if (!ret.isOK()) {
+            return ret;
+        }
+        ret = params->remove("noIndexBuildRetry");
+        if (!ret.isOK()) {
+            return ret;
+        }
+    }
+
     // "sharding.clusterRole" comes from the config file, so override it if "configsvr" or
     // "shardsvr" are set since those come from the command line.
     if (params->count("configsvr")) {
@@ -422,6 +436,10 @@ Status storeMongodOptions(const moe::Environment& params) {
         serverGlobalParams.cpu = params["cpu"].as<bool>();
     }
 
+    if (params.count("storage.indexBuildRetry")) {
+        serverGlobalParams.indexBuildRetry = params["storage.indexBuildRetry"].as<bool>();
+    }
+
     if (params.count("storage.journal.enabled")) {
         storageGlobalParams.dur = params["storage.journal.enabled"].as<bool>();
     }
diff --git a/src/mongo/db/mongod_options_storage.idl b/src/mongo/db/mongod_options_storage.idl
index 222eb750516..d784ef931e9 100644
--- a/src/mongo/db/mongod_options_storage.idl
+++ b/src/mongo/db/mongod_options_storage.idl
@@ -54,11 +54,13 @@ configs:
         description: 'Each database will be stored in a separate directory'
         short_name: directoryperdb
         arg_vartype: Switch
+
     'storage.queryableBackupMode':
         description: 'Enable read-only mode - if true the server will not accept writes'
         short_name: queryableBackupMode
         arg_vartype: Switch
         hidden: true
+
     'storage.groupCollections':
         description: >-
             Group collections - if true the storage engine may group
@@ -66,6 +68,7 @@ configs:
         short_name: groupCollections
         arg_vartype: Switch
         hidden: true
+
     'storage.syncPeriodSecs':
         description: 'Seconds between disk syncs (0=never, but not recommended)'
         short_name: syncdelay
@@ -74,19 +77,36 @@ configs:
         validator:
             gte: 0.0
             lte: { expr: 'StorageGlobalParams::kMaxSyncdelaySecs' }
+
+    'storage.indexBuildRetry':
+        description: "Do not retry any index builds that were interrupted by shutdown"
+        arg_vartype: Switch
+        conflicts: ['replication.replSet', 'replication.replSetName']
+        source: yaml
+
+    noIndexBuildRetry:
+        description: "Do not retry any index builds that were interrupted by shutdown"
+        short_name: noIndexBuildRetry
+        arg_vartype: Switch
+        conflicts: ['replication.replSet', 'replication.replSetName']
+        source: [ cli, ini ]
+
     upgrade:
         description: 'Upgrade db if needed'
         arg_vartype: Switch
         source: [ cli, ini ]
+
     repair:
         description: 'Run repair on all dbs'
         arg_vartype: Switch
         source: [ cli, ini ]
+
     'storage.journal.enabled':
         description: 'Enable journaling'
         short_name: journal
         deprecated_short_name: dur
         arg_vartype: Switch
+
     nojournal:
         description: 'Disable journaling (journaling is on by default for 64 bit)'
         deprecated_short_name: nodur
diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp
index bc83a173eaf..447656921ab 100644
--- a/src/mongo/db/repair_database_and_check_version.cpp
+++ b/src/mongo/db/repair_database_and_check_version.cpp
@@ -260,6 +260,16 @@ void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) {
     std::vector<StorageEngine::CollectionIndexNamePair> indexesToRebuild =
         fassert(40593, storageEngine->reconcileCatalogAndIdents(opCtx));
 
+    if (!indexesToRebuild.empty() && serverGlobalParams.indexBuildRetry) {
+        log() << "note: restart the server with --noIndexBuildRetry "
+              << "to skip index rebuilds";
+    }
+
+    if (!serverGlobalParams.indexBuildRetry) {
+        log() << "  not rebuilding interrupted indexes";
+        return;
+    }
+
     // Determine which indexes need to be rebuilt. rebuildIndexesOnCollection() requires that all
     // indexes on that collection are done at once, so we use a map to group them together.
     StringMap<IndexNameObjs> nsToIndexNameObjMap;
diff --git a/src/mongo/db/server_options.h b/src/mongo/db/server_options.h
index 82ebf186362..c8cc07b444e 100644
--- a/src/mongo/db/server_options.h
+++ b/src/mongo/db/server_options.h
@@ -59,6 +59,8 @@ struct ServerGlobalParams {
 
     int listenBacklog = 0;  // --listenBacklog, real default is SOMAXCONN
 
+    bool indexBuildRetry = true;  // --noIndexBuildRetry
+
     AtomicWord<bool> quiet{false};  // --quiet
 
     ClusterRole clusterRole = ClusterRole::None;  // --configsvr/--shardsvr
diff --git a/src/mongo/db/storage/kv/SConscript b/src/mongo/db/storage/kv/SConscript
index 09c485f28c3..45d1e34d760 100644
--- a/src/mongo/db/storage/kv/SConscript
+++ b/src/mongo/db/storage/kv/SConscript
@@ -53,6 +53,7 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/catalog/catalog_control',
+        '$BUILD_DIR/mongo/db/server_options_core',
         '$BUILD_DIR/mongo/db/storage/kv/kv_engine_core',
         '$BUILD_DIR/mongo/db/storage/storage_options',
         'kv_drop_pending_ident_reaper',
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index 4b55155450c..d9471cc6d2c 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -42,6 +42,7 @@
 #include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/logical_clock.h"
 #include "mongo/db/operation_context_noop.h"
+#include "mongo/db/server_options.h"
 #include "mongo/db/storage/kv/kv_catalog_feature_tracker.h"
 #include "mongo/db/storage/kv/kv_engine.h"
 #include "mongo/db/storage/kv/temporary_kv_record_store.h"
@@ -431,9 +432,18 @@ KVStorageEngine::reconcileCatalogAndIdents(OperationContext* opCtx) {
             // table is not found, or the index build did not successfully complete, this code
             // will return the index to be rebuilt.
             if (indexMetaData.isBackgroundSecondaryBuild && (!foundIdent || !indexMetaData.ready)) {
-                log()
-                    << "Expected background index build did not complete, rebuilding. Collection: "
-                    << coll << " Index: " << indexName;
+                if (!serverGlobalParams.indexBuildRetry) {
+                    log() << "Dropping an unfinished index because --noIndexBuildRetry is set. "
+                             "Collection: "
+                          << coll << " Index: " << indexName;
+                    fassert(51197, _engine->dropIdent(opCtx, indexIdent));
+                    indexesToDrop.push_back(indexName);
+                    continue;
+                }
+
+                log() << "Expected background index build did not complete, rebuilding. "
+                         "Collection: "
+                      << coll << " Index: " << indexName;
                 ret.emplace_back(coll.ns(), indexName);
                 continue;
             }