SERVER-7019 Return initial sync status information by default in replSetGetStatus and remove it after successful initial sync attempt

(cherry picked from commit fffb564422ebf4d0569052ebe61bd96e6494e25f)

14 files changed, 73 insertions, 22 deletions

diff --git a/jstests/disk/libs/wt_file_helper.js b/jstests/disk/libs/wt_file_helper.js
index 54e7781608d..75bd3688301 100644
--- a/jstests/disk/libs/wt_file_helper.js
+++ b/jstests/disk/libs/wt_file_helper.js
@@ -128,10 +128,15 @@ let assertStartInReplSet = function(replSet, originalNode, cleanData, expectResy
     let node = replSet.start(
         originalNode, {dbpath: originalNode.dbpath, port: originalNode.port, restart: !cleanData});
 
+    // Skip clearing initial sync progress after a successful initial sync attempt so that we
+    // can check initialSyncStatus fields after initial sync is complete.
+    assert.commandWorked(
+        node.adminCommand({configureFailPoint: 'skipClearInitialSyncState', mode: 'alwaysOn'}));
+
     replSet.awaitSecondaryNodes();
 
     // Ensure that an initial sync attempt was made and succeeded if the data directory was cleaned.
-    let res = assert.commandWorked(node.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    let res = assert.commandWorked(node.adminCommand({replSetGetStatus: 1}));
     if (expectResync) {
         assert.eq(1, res.initialSyncStatus.initialSyncAttempts.length);
         assert.eq(0, res.initialSyncStatus.failedInitialSyncAttempts);
@@ -139,6 +144,9 @@ let assertStartInReplSet = function(replSet, originalNode, cleanData, expectResy
         assert.eq(undefined, res.initialSyncStatus);
     }
 
+    assert.commandWorked(
+        node.adminCommand({configureFailPoint: 'skipClearInitialSyncState', mode: 'off'}));
+
     testFunc(node);
     return node;
 };
diff --git a/jstests/replsets/initial_sync_drop_collection.js b/jstests/replsets/initial_sync_drop_collection.js
index 63229527ee1..c68e9288a1f 100644
--- a/jstests/replsets/initial_sync_drop_collection.js
+++ b/jstests/replsets/initial_sync_drop_collection.js
@@ -39,6 +39,9 @@ function setupTest({failPoint, secondaryStartupParams}) {
 
     jsTestLog("Restarting secondary with failPoint " + failPoint + " set for " + nss);
     secondaryStartupParams = secondaryStartupParams || {};
+    // Skip clearing initial sync progress after a successful initial sync attempt so that we can
+    // check initialSyncStatus fields after initial sync is complete.
+    secondaryStartupParams['failpoint.skipClearInitialSyncState'] = tojson({mode: 'alwaysOn'});
     secondaryStartupParams['failpoint.' + failPoint] = tojson({mode: 'alwaysOn', data: {nss: nss}});
     secondaryStartupParams['numInitialSyncAttempts'] = 1;
     replTest.restart(secondary, {startClean: true, setParameter: secondaryStartupParams});
@@ -80,7 +83,7 @@ function finishTest({failPoint, secondaryStartupParams, expectedLog, waitForDrop
     jsTestLog("Waiting for initial sync to complete.");
     replTest.waitForState(secondary, ReplSetTest.State.SECONDARY);
 
-    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
     assert.eq(0, res.initialSyncStatus.failedInitialSyncAttempts);
 
     if (createNew) {
diff --git a/jstests/replsets/initial_sync_during_stepdown.js b/jstests/replsets/initial_sync_during_stepdown.js
index d10575f6f2e..a1e0033d709 100644
--- a/jstests/replsets/initial_sync_during_stepdown.js
+++ b/jstests/replsets/initial_sync_during_stepdown.js
@@ -48,6 +48,9 @@ function setupTest({
 
     jsTestLog("Starting secondary.");
     secondaryStartupParams['numInitialSyncAttempts'] = 1;
+    // Skip clearing initial sync progress after a successful initial sync attempt so that we can
+    // check initialSyncStatus fields after initial sync is complete.
+    secondaryStartupParams['failpoint.skipClearInitialSyncState'] = tojson({mode: 'alwaysOn'});
     rst.start(secondary, {startClean: true, setParameter: secondaryStartupParams});
 
     // Wait until secondary reaches RS_STARTUP2 state.
@@ -78,7 +81,7 @@ function finishTest(
     rst.waitForState(primary, ReplSetTest.State.SECONDARY);
 
     jsTestLog("Validating initial sync data.");
-    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
     assert.eq(0, res.initialSyncStatus.failedInitialSyncAttempts);
     assert.eq(2 + DocsCopiedByOplogFetcher, secondaryColl.find().itcount());
 
diff --git a/jstests/replsets/initial_sync_fcv.js b/jstests/replsets/initial_sync_fcv.js
index 5c8d37a4bdf..af0a466848c 100644
--- a/jstests/replsets/initial_sync_fcv.js
+++ b/jstests/replsets/initial_sync_fcv.js
@@ -36,6 +36,7 @@ function runInitialSync(cmd, initialFCV) {
         startClean: true,
         setParameter: {
             'failpoint.initialSyncHangBeforeListCollections': failPointOptions,
+            'failpoint.skipClearInitialSyncState': tojson({mode: 'alwaysOn'}),
             numInitialSyncAttempts: 2
         }
     });
@@ -61,7 +62,7 @@ function runInitialSync(cmd, initialFCV) {
     rst.awaitSecondaryNodes();
     rst.awaitReplication();
 
-    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    let res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
     assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 1);
 
     // We check oplogs and data hashes before we restart the second node.
diff --git a/jstests/replsets/initial_sync_replSetGetStatus.js b/jstests/replsets/initial_sync_replSetGetStatus.js
index fb79ae2e6d7..415f4b5cddb 100644
--- a/jstests/replsets/initial_sync_replSetGetStatus.js
+++ b/jstests/replsets/initial_sync_replSetGetStatus.js
@@ -37,11 +37,12 @@ checkLog.contains(secondary,
 
 // Test that replSetGetStatus returns the correct results while initial sync is in progress.
 var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
-assert(!res.initialSyncStatus,
-       "Response should not have an 'initialSyncStatus' field: " + tojson(res));
+assert(res.initialSyncStatus,
+       () => "Response should have an 'initialSyncStatus' field: " + tojson(res));
 
-res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
-assert(res.initialSyncStatus, "Response should have an 'initialSyncStatus' field: " + tojson(res));
+res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 0}));
+assert(!res.initialSyncStatus,
+       () => "Response should not have an 'initialSyncStatus' field: " + tojson(res));
 
 assert.commandFailedWithCode(secondary.adminCommand({replSetGetStatus: 1, initialSync: "t"}),
                              ErrorCodes.TypeMismatch);
@@ -56,9 +57,9 @@ assert.commandWorked(secondary.getDB('admin').runCommand(
 // Wait for initial sync to pause right before it finishes.
 checkLog.contains(secondary, 'initial sync - initialSyncHangBeforeFinish fail point enabled');
 
-// Test that replSetGetStatus returns the correct results when initial sync is at the very end.
-res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
-assert(res.initialSyncStatus, "Response should have an 'initialSyncStatus' field.");
+res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
+assert(res.initialSyncStatus,
+       () => "Response should have an 'initialSyncStatus' field: " + tojson(res));
 assert.eq(res.initialSyncStatus.fetchedMissingDocs, 0);
 assert.eq(res.initialSyncStatus.appliedOps, 3);
 assert.eq(res.initialSyncStatus.failedInitialSyncAttempts, 0);
@@ -79,7 +80,7 @@ replSet.awaitSecondaryNodes(60 * 1000);
 // Test that replSetGetStatus returns the correct results after initial sync is finished.
 res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
 assert(!res.initialSyncStatus,
-       "Response should not have an 'initialSyncStatus' field: " + tojson(res));
+       () => "Response should not have an 'initialSyncStatus' field: " + tojson(res));
 
 assert.commandFailedWithCode(secondary.adminCommand({replSetGetStatus: 1, initialSync: "m"}),
                              ErrorCodes.TypeMismatch);
diff --git a/jstests/replsets/initial_sync_update_missing_doc1.js b/jstests/replsets/initial_sync_update_missing_doc1.js
index 93eda8b7702..02576960290 100644
--- a/jstests/replsets/initial_sync_update_missing_doc1.js
+++ b/jstests/replsets/initial_sync_update_missing_doc1.js
@@ -41,7 +41,7 @@ updateRemove(coll, {_id: 0});
 
 turnOffHangBeforeCopyingDatabasesFailPoint(secondary);
 
-var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
 assert.eq(res.initialSyncStatus.fetchedMissingDocs, 0);
 var firstOplogEnd = res.initialSyncStatus.initialSyncOplogEnd;
 
diff --git a/jstests/replsets/initial_sync_update_missing_doc2.js b/jstests/replsets/initial_sync_update_missing_doc2.js
index 420aaee8adc..bd3b2d8957a 100644
--- a/jstests/replsets/initial_sync_update_missing_doc2.js
+++ b/jstests/replsets/initial_sync_update_missing_doc2.js
@@ -47,7 +47,7 @@ turnOffHangBeforeCopyingDatabasesFailPoint(secondary);
 // insert this document after failing to apply the udpate.
 assert.commandWorked(coll.insert({_id: 0, x: 3}));
 
-var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
 assert.eq(res.initialSyncStatus.fetchedMissingDocs, 0);
 var firstOplogEnd = res.initialSyncStatus.initialSyncOplogEnd;
 
diff --git a/jstests/replsets/initial_sync_update_missing_doc3.js b/jstests/replsets/initial_sync_update_missing_doc3.js
index 67e44b5cd6c..dadc0f32d9b 100644
--- a/jstests/replsets/initial_sync_update_missing_doc3.js
+++ b/jstests/replsets/initial_sync_update_missing_doc3.js
@@ -59,7 +59,7 @@ assert.commandWorked(coll.insert({_id: 0, x: 3}));
 // Mark the collection as drop pending so it gets renamed, but retains the UUID.
 assert.commandWorked(primary.getDB('test').runCommand({"drop": name}));
 
-var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+var res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
 assert.eq(res.initialSyncStatus.fetchedMissingDocs, 0);
 var firstOplogEnd = res.initialSyncStatus.initialSyncOplogEnd;
 
diff --git a/jstests/replsets/libs/initial_sync_test.js b/jstests/replsets/libs/initial_sync_test.js
index 9e38a4edd54..7ec45729173 100644
--- a/jstests/replsets/libs/initial_sync_test.js
+++ b/jstests/replsets/libs/initial_sync_test.js
@@ -99,10 +99,13 @@ function InitialSyncTest(name = "InitialSyncTest", replSet, timeout) {
      * Calls replSetGetStatus and checks if the node is in the provided state.
      */
     function isNodeInState(node, state) {
+        // We suppress the initialSync field here, because initial sync is paused while holding the
+        // mutex needed to report initial sync progress.
         return state ===
             assert
-                .commandWorkedOrFailedWithCode(node.adminCommand({replSetGetStatus: 1}),
-                                               ErrorCodes.NotYetInitialized)
+                .commandWorkedOrFailedWithCode(
+                    node.adminCommand({replSetGetStatus: 1, initialSync: 0}),
+                    ErrorCodes.NotYetInitialized)
                 .myState;
     }
 
diff --git a/jstests/replsets/libs/initial_sync_update_missing_doc.js b/jstests/replsets/libs/initial_sync_update_missing_doc.js
index 7a8e6823a7d..58330eb8236 100644
--- a/jstests/replsets/libs/initial_sync_update_missing_doc.js
+++ b/jstests/replsets/libs/initial_sync_update_missing_doc.js
@@ -24,6 +24,11 @@ var reInitiateSetWithSecondary = function(replSet, secondaryConfig) {
     assert.commandWorked(secondary.getDB('admin').runCommand(
         {configureFailPoint: 'initialSyncHangBeforeGettingMissingDocument', mode: 'alwaysOn'}));
 
+    // Skip clearing initial sync progress after a successful initial sync attempt so that we
+    // can check initialSyncStatus fields after initial sync is complete.
+    assert.commandWorked(secondary.getDB('admin').runCommand(
+        {configureFailPoint: 'skipClearInitialSyncState', mode: 'alwaysOn'}));
+
     replSet.reInitiate();
 
     // Wait for fail point message to be logged.
@@ -85,7 +90,7 @@ var finishAndValidate = function(replSet, name, firstOplogEnd, numInserted, numD
               secondary.getDB(dbName).getCollection(name).find().itcount(),
               'documents successfully synced to secondary');
 
-    const res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1, initialSync: 1}));
+    const res = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
 
     // If we haven't re-inserted any documents after deleting them, the fetch count is 0 because we
     // are unable to get the document from the sync source.
diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp
index 8ab94935a86..3ed877a9acc 100644
--- a/src/mongo/db/repl/initial_syncer.cpp
+++ b/src/mongo/db/repl/initial_syncer.cpp
@@ -110,6 +110,9 @@ MONGO_FAIL_POINT_DEFINE(failInitialSyncBeforeApplyingBatch);
 // Failpoint which fasserts if applying a batch fails.
 MONGO_FAIL_POINT_DEFINE(initialSyncFassertIfApplyingBatchFails);
 
+// Failpoint which skips clearing _initialSyncState after a successful initial sync attempt.
+MONGO_FAIL_POINT_DEFINE(skipClearInitialSyncState);
+
 namespace {
 using namespace executor;
 using CallbackArgs = executor::TaskExecutor::CallbackArgs;
@@ -363,6 +366,16 @@ std::string InitialSyncer::getDiagnosticString() const {
 
 BSONObj InitialSyncer::getInitialSyncProgress() const {
     LockGuard lk(_mutex);
+
+    // We return an empty BSON object after an initial sync attempt has been successfully
+    // completed. When an initial sync attempt completes successfully, initialSyncCompletes is
+    // incremented and then _initialSyncState is cleared. We check that _initialSyncState has been
+    // cleared because an initial sync attempt can fail even after initialSyncCompletes is
+    // incremented, and we also check that initialSyncCompletes is positive because an initial sync
+    // attempt can also fail before _initialSyncState is initialized.
+    if (!_initialSyncState && initialSyncCompletes.get() > 0) {
+        return BSONObj();
+    }
     return _getInitialSyncProgress_inlock();
 }
 
@@ -1517,6 +1530,12 @@ void InitialSyncer::_finishCallback(StatusWith<OpTimeAndWallTime> lastApplied) {
     invariant(_state != State::kComplete);
     _state = State::kComplete;
     _stateCondition.notify_all();
+
+    // Clear the initial sync progress after an initial sync attempt has been successfully
+    // completed.
+    if (lastApplied.isOK() && !MONGO_FAIL_POINT(skipClearInitialSyncState)) {
+        _initialSyncState.reset();
+    }
 }
 
 Status InitialSyncer::_scheduleLastOplogEntryFetcher_inlock(Fetcher::CallbackFn callback) {
diff --git a/src/mongo/db/repl/initial_syncer.h b/src/mongo/db/repl/initial_syncer.h
index 5aea265e90e..4b994f9ea88 100644
--- a/src/mongo/db/repl/initial_syncer.h
+++ b/src/mongo/db/repl/initial_syncer.h
@@ -212,7 +212,7 @@ public:
 
     /**
      * Returns stats about the progress of initial sync. If initial sync is not in progress it
-     * returns summary statistics for what occurred during initial sync.
+     * returns an empty BSON object.
      */
     BSONObj getInitialSyncProgress() const;
 
diff --git a/src/mongo/db/repl/initial_syncer_test.cpp b/src/mongo/db/repl/initial_syncer_test.cpp
index f9b94d7193d..003ec073d1d 100644
--- a/src/mongo/db/repl/initial_syncer_test.cpp
+++ b/src/mongo/db/repl/initial_syncer_test.cpp
@@ -3986,6 +3986,10 @@ TEST_F(
     // when reconstructPreparedTransactions uses DBDirectClient to call into ServiceEntryPoint.
     FailPointEnableBlock skipReconstructPreparedTransactions("skipReconstructPreparedTransactions");
 
+    // Skip clearing initial sync progress so that we can check if missing documents have been
+    // fetched after the initial sync attempt.
+    FailPointEnableBlock skipClearInitialSyncState("skipClearInitialSyncState");
+
     auto initialSyncer = &getInitialSyncer();
     auto opCtx = makeOpCtx();
 
@@ -4082,7 +4086,7 @@ TEST_F(
     ASSERT_TRUE(fetchCountIncremented);
 
     auto progress = initialSyncer->getInitialSyncProgress();
-    log() << "Progress after failed initial sync attempt: " << progress;
+    log() << "Progress after initial sync attempt: " << progress;
     ASSERT_EQUALS(1, progress.getIntField("fetchedMissingDocs")) << progress;
 }
 
@@ -4167,6 +4171,10 @@ TEST_F(InitialSyncerTest, GetInitialSyncProgressReturnsCorrectProgress) {
     // when reconstructPreparedTransactions uses DBDirectClient to call into ServiceEntryPoint.
     FailPointEnableBlock skipReconstructPreparedTransactions("skipReconstructPreparedTransactions");
 
+    // Skip clearing initial sync progress so that we can check initialSyncStatus fields after
+    // initial sync is complete.
+    FailPointEnableBlock skipClearInitialSyncState("skipClearInitialSyncState");
+
     auto initialSyncer = &getInitialSyncer();
     auto opCtx = makeOpCtx();
     ASSERT_OK(ServerParameterSet::getGlobal()
diff --git a/src/mongo/db/repl/repl_set_get_status_cmd.cpp b/src/mongo/db/repl/repl_set_get_status_cmd.cpp
index cd8c4caec19..4fcd45073f4 100644
--- a/src/mongo/db/repl/repl_set_get_status_cmd.cpp
+++ b/src/mongo/db/repl/repl_set_get_status_cmd.cpp
@@ -55,9 +55,9 @@ public:
         Status status = ReplicationCoordinator::get(opCtx)->checkReplEnabledForCommand(&result);
         uassertStatusOK(status);
 
-        bool includeInitialSync = false;
+        bool includeInitialSync = true;
         Status initialSyncStatus =
-            bsonExtractBooleanFieldWithDefault(cmdObj, "initialSync", false, &includeInitialSync);
+            bsonExtractBooleanFieldWithDefault(cmdObj, "initialSync", true, &includeInitialSync);
         uassertStatusOK(initialSyncStatus);
 
         auto responseStyle = ReplicationCoordinator::ReplSetGetStatusResponseStyle::kBasic;