1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
/**
* Tests that tenant migrations resume successfully on recipient stepup and restart.
*
* @tags: [requires_fcv_47, requires_majority_read_concern, requires_persistence,
* incompatible_with_eft, incompatible_with_windows_tls, incompatible_with_macos]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/parallelTester.js");
load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
const kMaxSleepTimeMS = 100;
const kTenantId = "testTenantId";
// Set the delay before a state doc is garbage collected to be short to speed up the test but long
// enough for the state doc to still be around after stepup or restart.
const kGarbageCollectionDelayMS = 30 * 1000;
// Set the TTL monitor to run at a smaller interval to speed up the test.
const kTTLMonitorSleepSecs = 1;
const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
/**
* Runs the donorStartMigration command to start a migration, and interrupts the migration on the
* recipient using the 'interruptFunc' after the migration starts on the recipient side, and
* asserts that migration eventually commits.
* @param {recipientRestarted} bool is needed to properly assert the tenant migrations stat count.
*/
function testRecipientSyncDataInterrupt(interruptFunc, recipientRestarted) {
const recipientRst = new ReplSetTest(
{nodes: 3, name: "recipientRst", nodeOptions: migrationX509Options.recipient});
recipientRst.startSet();
recipientRst.initiate();
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), recipientRst});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
recipientRst.stopSet();
return;
}
const donorRst = tenantMigrationTest.getDonorRst();
const donorPrimary = tenantMigrationTest.getDonorPrimary();
let recipientPrimary = tenantMigrationTest.getRecipientPrimary();
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId: kTenantId,
recipientConnString: tenantMigrationTest.getRecipientConnString(),
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
const runMigrationThread = new Thread(TenantMigrationUtil.runMigrationAsync,
migrationOpts,
donorRstArgs,
false /* retryOnRetryableErrors */);
runMigrationThread.start();
// Wait for recipientSyncData command to start.
assert.soon(
() => recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"})
.inprog.length > 0);
sleep(Math.random() * kMaxSleepTimeMS);
interruptFunc(recipientRst);
TenantMigrationTest.assertCommitted(runMigrationThread.returnData());
tenantMigrationTest.waitForDonorNodesToReachState(donorRst.nodes,
migrationId,
migrationOpts.tenantId,
TenantMigrationTest.DonorState.kCommitted);
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
tenantMigrationTest.awaitTenantMigrationStatsCounts(donorPrimary,
{totalSuccessfulMigrationsDonated: 1});
recipientPrimary = tenantMigrationTest.getRecipientPrimary(); // Could change after interrupt.
if (!recipientRestarted) {
tenantMigrationTest.awaitTenantMigrationStatsCounts(recipientPrimary,
{totalSuccessfulMigrationsReceived: 1});
} else {
// In full restart the count could be lost completely.
const stats = tenantMigrationTest.getTenantMigrationStats(recipientPrimary);
assert(1 == stats.totalSuccessfulMigrationsReceived ||
0 == stats.totalSuccessfulMigrationsReceived);
}
tenantMigrationTest.stop();
recipientRst.stopSet();
}
/**
* Starts a migration and waits for it to commit, then runs the donorForgetMigration, and interrupts
* the recipient using the 'interruptFunc', and asserts that the migration state is eventually
* garbage collected.
*/
function testRecipientForgetMigrationInterrupt(interruptFunc) {
const donorRst = new ReplSetTest({
nodes: 1,
name: "donorRst",
nodeOptions: Object.assign(migrationX509Options.donor, {
setParameter: {
tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS,
ttlMonitorSleepSecs: kTTLMonitorSleepSecs,
}
})
});
const recipientRst = new ReplSetTest({
nodes: 3,
name: "recipientRst",
nodeOptions: Object.assign(migrationX509Options.recipient, {
setParameter: {
tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS,
ttlMonitorSleepSecs: kTTLMonitorSleepSecs,
}
})
});
donorRst.startSet();
donorRst.initiate();
recipientRst.startSet();
recipientRst.initiate();
const tenantMigrationTest =
new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
donorRst.stopSet();
recipientRst.stopSet();
return;
}
const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId: kTenantId,
recipientConnString: recipientRst.getURL(),
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(
migrationOpts, false /* retryOnRetryableErrors */, false /* automaticForgetMigration */));
const forgetMigrationThread = new Thread(TenantMigrationUtil.forgetMigrationAsync,
migrationOpts.migrationIdString,
donorRstArgs,
false /* retryOnRetryableErrors */);
forgetMigrationThread.start();
// Wait for recipientForgetMigration command to start.
assert.soon(() => {
const res = assert.commandWorked(
recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"}));
return res.inprog[0].expireAt != null;
});
sleep(Math.random() * kMaxSleepTimeMS);
interruptFunc(recipientRst);
assert.commandWorkedOrFailedWithCode(
tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString),
ErrorCodes.NoSuchTenantMigration);
assert.commandWorked(forgetMigrationThread.returnData());
tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, migrationOpts.tenantId);
tenantMigrationTest.stop();
donorRst.stopSet();
recipientRst.stopSet();
}
(() => {
jsTest.log("Test that the migration resumes on stepup");
testRecipientSyncDataInterrupt((recipientRst) => {
// Force the primary to step down but make it likely to step back up.
const recipientPrimary = recipientRst.getPrimary();
assert.commandWorked(recipientPrimary.adminCommand(
{replSetStepDown: ReplSetTest.kForeverSecs, force: true}));
assert.commandWorked(recipientPrimary.adminCommand({replSetFreeze: 0}));
}, false);
})();
(() => {
jsTest.log("Test that the migration resumes after restart");
testRecipientSyncDataInterrupt((recipientRst) => {
recipientRst.stopSet(null /* signal */, true /*forRestart */);
recipientRst.startSet({restart: true});
}, true);
})();
(() => {
jsTest.log("Test that the recipientForgetMigration command can be retried on stepup");
testRecipientForgetMigrationInterrupt((recipientRst) => {
// Force the primary to step down but make it likely to step back up.
const recipientPrimary = recipientRst.getPrimary();
assert.commandWorked(recipientPrimary.adminCommand(
{replSetStepDown: ReplSetTest.kForeverSecs, force: true}));
assert.commandWorked(recipientPrimary.adminCommand({replSetFreeze: 0}));
});
})();
(() => {
jsTest.log("Test that the recipientForgetMigration command can be retried after restart");
testRecipientForgetMigrationInterrupt((recipientRst) => {
recipientRst.stopSet(null /* signal */, true /*forRestart */);
recipientRst.startSet({restart: true});
});
})();
})();
|
