1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
/**
* Tests that a migration will retry if the oplog fetcher discoveres that its sync source is too
* stale. We test this with a donor replica set that has two secondaries, 'donorSecondary' and
* 'delayedSecondary'. We force the recipient to sync from 'donorSecondary'. Then, after the
* recipient has set its 'startFetchingDonorOpTime', we stop replication on 'delayedSecondary' and
* advance the OpTime on 'donorSecondary'. Next, we wait until the recipient is about to report that
* it has reached a consistent state. At this point, it should have advanced its 'lastFetched' to be
* ahead of 'startFetchingDonorOpTime'. After forcing the recipient to restart and sync from
* 'delayedSecondary', it should see that it is too stale. As a result, it should retry sync source
* selection until it finds a sync source that is no longer too stale.
*
* @tags: [requires_fcv_49, requires_majority_read_concern, incompatible_with_eft,
* incompatible_with_windows_tls, incompatible_with_macos, requires_persistence]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/uuid_util.js");
load("jstests/libs/write_concern_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load('jstests/replsets/rslib.js');
const donorRst = new ReplSetTest({
name: `${jsTestName()}_donor`,
nodes: 3,
settings: {chainingAllowed: false},
nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, {
setParameter: {
// Allow non-timestamped reads on donor after migration completes for testing.
'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}),
}
}),
});
donorRst.startSet();
donorRst.initiateWithHighElectionTimeout();
if (!TenantMigrationUtil.isFeatureFlagEnabled(donorRst.getPrimary())) {
jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
donorRst.stopSet();
return;
}
const tenantMigrationTest = new TenantMigrationTest({
name: jsTestName(),
donorRst,
// Set a low value for excluding donor hosts so that the test doesn't take long to retry a sync
// source.
sharedOptions: {setParameter: {tenantMigrationExcludeDonorHostTimeoutMS: 1000}}
});
const tenantId = "testTenantId";
const tenantDB = tenantMigrationTest.tenantDB(tenantId, "testDB");
const collName = "testColl";
const delayedSecondary = donorRst.getSecondaries()[0];
const donorSecondary = donorRst.getSecondaries()[1];
const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
const hangRecipientPrimaryAfterCreatingRSM =
configureFailPoint(recipientPrimary, 'hangAfterCreatingRSM');
const hangRecipientPrimaryAfterRetrievingStartOpTimes = configureFailPoint(
recipientPrimary, 'fpAfterRetrievingStartOpTimesMigrationRecipientInstance', {action: "hang"});
const hangRecipientPrimaryBeforeConsistentState = configureFailPoint(
recipientPrimary, 'fpBeforeFulfillingDataConsistentPromise', {action: "hang"});
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId,
// Configure the recipient primary to only choose a secondary as sync source.
readPreference: {mode: 'secondary'}
};
jsTestLog("Starting the tenant migration");
assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
hangRecipientPrimaryAfterCreatingRSM.wait();
awaitRSClientHosts(recipientPrimary, donorSecondary, {ok: true, secondary: true});
awaitRSClientHosts(recipientPrimary, delayedSecondary, {ok: true, secondary: true});
// Turn on the 'waitInHello' failpoint. This will cause the delayed secondary to cease sending
// hello responses and the RSM should mark the node as down. This is necessary so that the
// delayed secondary is not chosen as the sync source here.
jsTestLog(
"Turning on waitInHello failpoint. Delayed donor secondary should stop sending hello responses.");
const helloFailpoint = configureFailPoint(delayedSecondary, "waitInHello");
awaitRSClientHosts(recipientPrimary, delayedSecondary, {ok: false});
hangRecipientPrimaryAfterCreatingRSM.off();
hangRecipientPrimaryAfterRetrievingStartOpTimes.wait();
let res = recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"});
let currOp = res.inprog[0];
// The migration should not be complete.
assert.eq(currOp.migrationCompleted, false, tojson(res));
assert.eq(currOp.dataSyncCompleted, false, tojson(res));
// The sync source can only be 'donorSecondary'.
assert.eq(donorSecondary.host, currOp.donorSyncSource, tojson(res));
helloFailpoint.off();
// Stop replicating on one of the secondaries and advance the OpTime on the other nodes in the
// donor replica set.
jsTestLog("Stopping replication on delayedSecondary");
stopServerReplication(delayedSecondary);
tenantMigrationTest.insertDonorDB(tenantDB, collName);
// Wait for 'lastFetched' to be advanced on the recipient.
hangRecipientPrimaryAfterRetrievingStartOpTimes.off();
hangRecipientPrimaryBeforeConsistentState.wait();
const hangAfterRetrievingOpTimesAfterRestart = configureFailPoint(
recipientPrimary, 'fpAfterRetrievingStartOpTimesMigrationRecipientInstance', {action: "hang"});
jsTestLog("Stopping donorSecondary");
donorRst.stop(donorSecondary);
awaitRSClientHosts(recipientPrimary, delayedSecondary, {ok: true, secondary: true});
awaitRSClientHosts(recipientPrimary, donorSecondary, {ok: false});
hangRecipientPrimaryBeforeConsistentState.off();
const configRecipientNs = recipientPrimary.getCollection(TenantMigrationTest.kConfigRecipientsNS);
assert.soon(() => {
// Wait for the recipient to realize that the donor sync source has been shut down and retry
// sync source selection.
const recipientDoc = configRecipientNs.find({"_id": migrationId}).toArray();
return recipientDoc[0].numRestartsDueToDonorConnectionFailure == 1;
});
hangAfterRetrievingOpTimesAfterRestart.wait();
res = recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"});
currOp = res.inprog[0];
// The migration should not be complete.
assert.eq(currOp.migrationCompleted, false, tojson(res));
assert.eq(currOp.dataSyncCompleted, false, tojson(res));
// Since 'donorSecondary' was shut down, the sync source can only be 'delayedSecondary'.
assert.eq(delayedSecondary.host, currOp.donorSyncSource, tojson(res));
const hangAfterPersistingTenantMigrationRecipientInstanceStateDoc =
configureFailPoint(recipientPrimary,
"fpAfterPersistingTenantMigrationRecipientInstanceStateDoc",
{action: "hang"});
hangAfterRetrievingOpTimesAfterRestart.off();
assert.soon(() => {
// Verify that the recipient eventually has to restart again, since its lastFetched is ahead of
// the last OpTime on 'delayedSecondary'.
const recipientDoc = configRecipientNs.find({"_id": migrationId}).toArray();
return recipientDoc[0].numRestartsDueToDonorConnectionFailure >= 2;
});
hangAfterPersistingTenantMigrationRecipientInstanceStateDoc.wait();
// Let 'delayedSecondary' catch back up to the recipient's lastFetched OpTime.
donorRst.remove(donorSecondary);
restartServerReplication(delayedSecondary);
donorRst.awaitReplication();
hangAfterPersistingTenantMigrationRecipientInstanceStateDoc.off();
// Verify that the migration eventually commits successfully.
const migrationRes =
assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
assert.eq(migrationRes.state, TenantMigrationTest.DonorState.kCommitted);
donorRst.stopSet();
tenantMigrationTest.stop();
})();
|
