diff options
author | Matthew Russotto <matthew.russotto@mongodb.com> | 2022-07-26 16:45:30 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-12-02 19:07:21 +0000 |
commit | 69cc757bea9f7d62cfea7309cebaaa55a95a3a3a (patch) | |
tree | d4fef4821fb1b26ab62f68302fc24acaf7641b2d | |
parent | d3e569ea57fab877c9e2b9036c13100828c1e4b3 (diff) | |
download | mongo-69cc757bea9f7d62cfea7309cebaaa55a95a3a3a.tar.gz |
SERVER-68338 sync_source_changes.js needs to wait for heartbeat
(cherry picked from commit 5f83f7ee5c4756c31dcf7e249aa200bf7718362e)
-rw-r--r-- | jstests/replsets/sync_source_changes.js | 57 |
1 files changed, 34 insertions, 23 deletions
diff --git a/jstests/replsets/sync_source_changes.js b/jstests/replsets/sync_source_changes.js index 80655c06667..38e55dac5e7 100644 --- a/jstests/replsets/sync_source_changes.js +++ b/jstests/replsets/sync_source_changes.js @@ -10,6 +10,34 @@ load("jstests/replsets/rslib.js"); // reconfig load("jstests/replsets/libs/sync_source.js"); // assertSyncSourceMatchesSoon +// We need to wait for a heartbeat from the secondary to the sync source, then run sync +// source selection, because: +// 1) The sync source changes only after retrieving a batch and +// 2) The sync source won't change if the secondary isn't behind the expected sync source, as +// determined by heartbeats. +function assertSyncSourceChangesTo(rst, secondary, expectedSyncSource) { + // Insert a document while 'secondary' is not replicating to force it to run + // shouldChangeSyncSource. + stopServerReplication(secondary); + assert.commandWorked( + rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, { + writeConcern: {w: 1} + })); + const sourceId = rst.getNodeId(expectedSyncSource); + // Waits for the secondary to see the expected sync source advance beyond it. + assert.soon(function() { + const status = assert.commandWorked(secondary.adminCommand({replSetGetStatus: 1})); + const appliedTimestamp = status.optimes.appliedOpTime.ts; + const sourceMember = status.members.find((x) => x._id == sourceId); + return timestampCmp(sourceMember.optime.ts, appliedTimestamp) > 0; + }); + restartServerReplication(secondary); + assertSyncSourceMatchesSoon(secondary, expectedSyncSource.host); +} + +// Replication verbosity 2 includes the sync source change debug logs. +TestData["setParameters"]["logComponentVerbosity"]["replication"]["verbosity"] = 2; + // Start RST with only one voting node, node 0 -- this will be the only valid voting node and sync // source const rst = new ReplSetTest({nodes: [{}, {rsConfig: {priority: 0, votes: 0}}]}); @@ -21,6 +49,7 @@ const primary = rst.getPrimary(); assert.eq(primary, rst.nodes[0]); // Add a new voting node, node 2 -- voting nodes will choose voting nodes as sync sources. +jsTestLog("Adding node 2"); const newNode = rst.add({}); rst.reInitiate(); rst.waitForState(newNode, ReplSetTest.State.SECONDARY); @@ -28,7 +57,7 @@ rst.awaitReplication(); rst.awaitSecondaryNodes(); // Assure that node 2 will set node 0 as its sync source, since it is the best option. -assertSyncSourceMatchesSoon(newNode, rst.nodes[0].host); +assertSyncSourceChangesTo(rst, newNode, rst.nodes[0]); // Make node 1 a voter so that it will be a valid option for sync source let cfg = rst.getReplSetConfigFromNode(); @@ -38,36 +67,18 @@ reconfig(rst, cfg); // Force a stepup of node 1 -- we need to step node 0 down so that we can set it as a non-voter // without causing errors. +jsTestLog("Stepping up node 1"); rst.stepUp(rst.nodes[1]); +jsTestLog("Reconfiguring node 0 as nonvoter"); // Make node 0 a nonvoter so that it will be an invalid option for sync source cfg = rst.getReplSetConfigFromNode(); cfg.members[0].priority = 0; cfg.members[0].votes = 0; reconfig(rst, cfg); +jsTestLog("Reconfig complete"); -// Run this repeatedly, as sometimes the stop, insert, restart won't cause the sync source to be -// switched correctly due to transient issues with the sync source we want to switch to. -assert.soon(() => { - // Insert a document while newNode is not replicating to force it to run shouldChangeSyncSource - stopServerReplication(newNode); - assert.commandWorked( - rst.getPrimary().getDB("testSyncSourceChangesDb").getCollection("coll").insert({a: 1}, { - writeConcern: {w: 1} - })); - restartServerReplication(newNode); - try { - assertSyncSourceMatchesSoon(newNode, - cfg.members[1].host, - undefined /* msg */, - 5 * 1000 /* timeout */, - undefined /* interval */, - {runHangAnalyzer: false}); - return true; - } catch (e) { - return false; - } -}); +assertSyncSourceChangesTo(rst, newNode, rst.nodes[1]); rst.stopSet(); })(); |