SERVER-68338 sync_source_changes.js needs to wait for heartbeat

(cherry picked from commit 5f83f7ee5c4756c31dcf7e249aa200bf7718362e)

1 files changed, 34 insertions, 23 deletions

diff --git a/jstests/replsets/sync_source_changes.js b/jstests/replsets/sync_source_changes.js
index 80655c06667..38e55dac5e7 100644
--- a/jstests/replsets/sync_source_changes.js
+++ b/jstests/replsets/sync_source_changes.js
@@ -10,6 +10,34 @@
 load("jstests/replsets/rslib.js");             // reconfig
 load("jstests/replsets/libs/sync_source.js");  // assertSyncSourceMatchesSoon
 
+// We need to wait for a heartbeat from the secondary to the sync source, then run sync
+// source selection, because:
+// 1) The sync source changes only after retrieving a batch and
+// 2) The sync source won't change if the secondary isn't behind the expected sync source, as
+//    determined by heartbeats.
+function assertSyncSourceChangesTo(rst, secondary, expectedSyncSource) {
+    // Insert a document while 'secondary' is not replicating to force it to run
+    // shouldChangeSyncSource.
+    stopServerReplication(secondary);
+    assert.commandWorked(
+        rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, {
+            writeConcern: {w: 1}
+        }));
+    const sourceId = rst.getNodeId(expectedSyncSource);
+    // Waits for the secondary to see the expected sync source advance beyond it.
+    assert.soon(function() {
+        const status = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1}));
+        const appliedTimestamp = status.optimes.appliedOpTime.ts;
+        const sourceMember = status.members.find((x) => x._id == sourceId);
+        return timestampCmp(sourceMember.optime.ts, appliedTimestamp) > 0;
+    });
+    restartServerReplication(secondary);
+    assertSyncSourceMatchesSoon(secondary, expectedSyncSource.host);
+}
+
+// Replication verbosity 2 includes the sync source change debug logs.
+TestData["setParameters"]["logComponentVerbosity"]["replication"]["verbosity"] = 2;
+
 // Start RST with only one voting node, node 0 -- this will be the only valid voting node and sync
 // source
 const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]});
@@ -21,6 +49,7 @@ const primary = rst.getPrimary();
 assert.eq(primary, rst.nodes[0]);
 
 // Add a new voting node, node 2 -- voting nodes will choose voting nodes as sync sources.
+jsTestLog("Adding node 2");
 const newNode = rst.add({});
 rst.reInitiate();
 rst.waitForState(newNode, ReplSetTest.State.SECONDARY);
@@ -28,7 +57,7 @@ rst.awaitReplication();
 rst.awaitSecondaryNodes();
 
 // Assure that node 2 will set node 0 as its sync source, since it is the best option.
-assertSyncSourceMatchesSoon(newNode, rst.nodes[0].host);
+assertSyncSourceChangesTo(rst, newNode, rst.nodes[0]);
 
 // Make node 1 a voter so that it will be a valid option for sync source
 let cfg = rst.getReplSetConfigFromNode();
@@ -38,36 +67,18 @@ reconfig(rst, cfg);
 
 // Force a stepup of node 1 -- we need to step node 0 down so that we can set it as a non-voter
 // without causing errors.
+jsTestLog("Stepping up node 1");
 rst.stepUp(rst.nodes[1]);
 
+jsTestLog("Reconfiguring node 0 as nonvoter");
 // Make node 0 a nonvoter so that it will be an invalid option for sync source
 cfg = rst.getReplSetConfigFromNode();
 cfg.members[0].priority = 0;
 cfg.members[0].votes = 0;
 reconfig(rst, cfg);
+jsTestLog("Reconfig complete");
 
-// Run this repeatedly, as sometimes the stop, insert, restart won't cause the sync source to be
-// switched correctly due to transient issues with the sync source we want to switch to.
-assert.soon(() => {
-    // Insert a document while newNode is not replicating to force it to run shouldChangeSyncSource
-    stopServerReplication(newNode);
-    assert.commandWorked(
-        rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, {
-            writeConcern: {w: 1}
-        }));
-    restartServerReplication(newNode);
-    try {
-        assertSyncSourceMatchesSoon(newNode,
-                                    cfg.members[1].host,
-                                    undefined /* msg */,
-                                    5 * 1000 /* timeout */,
-                                    undefined /* interval */,
-                                    {runHangAnalyzer: false});
-        return true;
-    } catch (e) {
-        return false;
-    }
-});
+assertSyncSourceChangesTo(rst, newNode, rst.nodes[1]);
 
 rst.stopSet();
 })();