1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
/**
* Tests a back-to-back migration scenario where we migrate immediately from replica sets A->B->C.
* Specifically, this tests that when replica set B has both a recipient and donor access blocker,
* old reads will continue to be blocked by the recipient access blocker even while it acts as a
* donor for a newly initiated migration.
*
* @tags: [requires_fcv_49, requires_majority_read_concern, incompatible_with_eft,
* incompatible_with_windows_tls, incompatible_with_macos, requires_persistence]
*/
(function() {
"use strict";
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
load("jstests/libs/fail_point_util.js");
load("jstests/libs/parallelTester.js"); // for 'Thread'
load("jstests/libs/uuid_util.js");
load("jstests/replsets/rslib.js"); // for 'getLastOpTime'
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
return;
}
const kTenantId = "testTenantId";
const kDbName = tenantMigrationTest.tenantDB(kTenantId, "testDb");
const kCollName = "testColl";
const donorPrimary = tenantMigrationTest.getDonorPrimary();
const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
const recipientRst = tenantMigrationTest.getRecipientRst();
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
recipientConnString: tenantMigrationTest.getRecipientConnString(),
tenantId: kTenantId,
};
// Select a read timestamp < blockTimestamp.
const preMigrationTimestamp = getLastOpTime(donorPrimary).ts;
let waitForRejectReadsBeforeTsFp = configureFailPoint(
recipientPrimary, "fpAfterWaitForRejectReadsBeforeTimestamp", {action: "hang"});
const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst());
const migrationThread =
new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs);
migrationThread.start();
waitForRejectReadsBeforeTsFp.wait();
const donorDoc =
donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).findOne({tenantId: kTenantId});
assert.lt(preMigrationTimestamp, donorDoc.blockTimestamp);
waitForRejectReadsBeforeTsFp.off();
// Wait for the migration to complete.
jsTest.log("Waiting for migration to complete");
TenantMigrationTest.assertCommitted(migrationThread.returnData());
tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString);
recipientRst.nodes.forEach(node => {
const db = node.getDB(kDbName);
const res = db.runCommand({
find: kCollName,
readConcern: {
level: "snapshot",
atClusterTime: preMigrationTimestamp,
}
});
assert.commandFailedWithCode(res, ErrorCodes.SnapshotTooOld, tojson(res));
assert.eq(res.errmsg, "Tenant read is not allowed before migration completes");
});
jsTestLog("Running a back-to-back migration");
const tenantMigrationTest2 = new TenantMigrationTest(
{name: jsTestName() + "2", donorRst: tenantMigrationTest.getRecipientRst()});
const donor2Primary = tenantMigrationTest2.getDonorPrimary();
const donor2RstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest2.getDonorRst());
const migration2Id = UUID();
const migrationOpts2 = {
migrationIdString: extractUUIDFromObject(migration2Id),
recipientConnString: tenantMigrationTest2.getRecipientConnString(),
tenantId: kTenantId,
};
const newDonorRst = recipientRst;
let waitAfterCreatingMtab =
configureFailPoint(donor2Primary, "pauseTenantMigrationBeforeLeavingBlockingState");
const migration2Thread =
new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts2, donor2RstArgs);
migration2Thread.start();
// At this point, 'donor2Primary' should have both a recipient and donor access blocker. The donor
// access blocker has entered the blocking state, and the recipient access blocker should
// still be blocking reads with timestamps < blocktimestamp from the previous migration.
waitAfterCreatingMtab.wait();
// Check that the current serverStatus reflects the recipient access blocker.
const mtabStatus = tenantMigrationTest.getTenantMigrationAccessBlocker(donor2Primary, kTenantId);
assert.eq(mtabStatus.state, TenantMigrationTest.RecipientAccessState.kRejectBefore, mtabStatus);
assert(mtabStatus.hasOwnProperty("rejectBeforeTimestamp"), mtabStatus);
assert.eq(mtabStatus["rejectBeforeTimestamp"], donorDoc.blockTimestamp, mtabStatus);
// The server value representation of the donor blocking state.
// TODO (SERVER-55913): Check that serverStatus returns both recipient and donor states.
const kBlocking = 3;
const res = assert.commandWorked(
donor2Primary.adminCommand({currentOp: true, desc: "tenant donor migration"}));
assert.eq(bsonWoCompare(res.inprog[0].instanceID.uuid, migration2Id), 0, tojson(res.inprog));
assert.eq(res.inprog[0].lastDurableState, kBlocking, tojson(res.inprog));
// Get the block timestamp for this new migration.
const donorDoc2 =
donor2Primary.getCollection(TenantMigrationTest.kConfigDonorsNS).findOne({tenantId: kTenantId});
const blockTimestamp2 = donorDoc2.blockTimestamp;
// The donor access blocker should block reads after the blockTimestamp of the new migration.
newDonorRst.nodes.forEach(node => {
jsTestLog("Test that read times out on node: " + node);
const db = node.getDB(kDbName);
assert.commandFailedWithCode(db.runCommand({
find: kCollName,
readConcern: {
afterClusterTime: blockTimestamp2,
},
maxTimeMS: 2 * 1000,
}),
ErrorCodes.MaxTimeMSExpired);
});
// The recipient access blocker should fail reads before the blockTimestamp of the old migration.
newDonorRst.nodes.forEach(node => {
jsTestLog("Test that read fails on node: " + node);
const db = node.getDB(kDbName);
const res = db.runCommand({
find: kCollName,
readConcern: {
level: "snapshot",
atClusterTime: preMigrationTimestamp,
}
});
assert.commandFailedWithCode(res, ErrorCodes.SnapshotTooOld, tojson(res));
assert.eq(res.errmsg, "Tenant read is not allowed before migration completes");
});
waitAfterCreatingMtab.off();
TenantMigrationTest.assertCommitted(migration2Thread.returnData());
tenantMigrationTest2.stop();
tenantMigrationTest.stop();
})();
|
