1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
/**
* Tests that the migration still proceeds successfully after a state transition write aborts.
*
* @tags: [requires_fcv_47, incompatible_with_eft, requires_majority_read_concern,
* incompatible_with_windows_tls, incompatible_with_macos, requires_persistence]
*/
(function() {
"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/uuid_util.js");
load("jstests/libs/parallelTester.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
const kTenantIdPrefix = "testTenantId";
const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
if (!tenantMigrationTest.isFeatureFlagEnabled()) {
jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
return;
}
/**
* Starts a migration and forces the write to insert the donor's state doc to abort on the first few
* tries. Asserts that the migration still completes successfully.
*/
function testAbortInitialState(donorRst) {
const donorPrimary = donorRst.getPrimary();
// Force the storage transaction for the insert to abort prior to inserting the WiredTiger
// record.
let writeConflictFp = configureFailPoint(donorPrimary, "WTWriteConflictException");
const tenantId = `${kTenantIdPrefix}-initial`;
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId,
recipientConnString: tenantMigrationTest.getRecipientConnString(),
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
// Run the migration in its own thread, since the initial 'donorStartMigration' command will
// hang due to the failpoint.
let migrationThread =
new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs);
migrationThread.start();
writeConflictFp.wait();
// Next, force the storage transaction for the insert to abort after inserting the WiredTiger
// record and initializing the in-memory migration state.
let opObserverFp = configureFailPoint(donorPrimary, "donorOpObserverFailAfterOnInsert");
writeConflictFp.off();
opObserverFp.wait();
opObserverFp.off();
// Verify that the migration completes successfully.
assert.commandWorked(migrationThread.returnData());
tenantMigrationTest.waitForDonorNodesToReachState(
donorRst.nodes, migrationId, tenantId, TenantMigrationTest.DonorState.kCommitted);
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
}
/**
* Starts a migration after enabling 'pauseFailPoint' (must pause the migration) and
* 'setUpFailPoints' on the donor's primary. Forces the write to transition to 'nextState' after
* reaching 'pauseFailPoint' to abort on the first few tries. Asserts that the migration still
* completes successfully.
*/
function testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState) {
jsTest.log(`Test aborting the write to transition to state "${
nextState}" after reaching failpoint "${pauseFailPoint}"`);
const donorPrimary = donorRst.getPrimary();
const tenantId = `${kTenantIdPrefix}-${nextState}`;
const migrationId = UUID();
const migrationOpts = {
migrationIdString: extractUUIDFromObject(migrationId),
tenantId,
};
let failPointsToClear = [];
setUpFailPoints.forEach(failPoint => {
failPointsToClear.push(configureFailPoint(donorPrimary, failPoint));
});
let pauseFp = configureFailPoint(donorPrimary, pauseFailPoint);
assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
pauseFp.wait();
// Force the storage transaction for the write to transition to the next state to abort prior to
// updating the WiredTiger record.
let writeConflictFp = configureFailPoint(donorPrimary, "WTWriteConflictException");
pauseFp.off();
writeConflictFp.wait();
// Next, force the storage transaction for the write to abort after updating the WiredTiger
// record and registering the change.
let opObserverFp = configureFailPoint(donorPrimary, "donorOpObserverFailAfterOnUpdate");
writeConflictFp.off();
opObserverFp.wait();
opObserverFp.off();
// Verify that the migration completes successfully.
assert.commandWorked(tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
if (nextState === TenantMigrationTest.DonorState.kAborted) {
tenantMigrationTest.waitForDonorNodesToReachState(
donorRst.nodes, migrationId, tenantId, TenantMigrationTest.DonorState.kAborted);
} else {
tenantMigrationTest.waitForDonorNodesToReachState(
donorRst.nodes, migrationId, tenantId, TenantMigrationTest.DonorState.kCommitted);
}
failPointsToClear.forEach(failPoint => {
failPoint.off();
});
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
}
const donorRst = tenantMigrationTest.getDonorRst();
jsTest.log("Test aborting donor's state doc insert");
testAbortInitialState(donorRst);
jsTest.log("Test aborting donor's state doc update");
[{
pauseFailPoint: "pauseTenantMigrationBeforeLeavingDataSyncState",
nextState: TenantMigrationTest.DonorState.kBlocking
},
{
pauseFailPoint: "pauseTenantMigrationBeforeLeavingBlockingState",
nextState: TenantMigrationTest.DonorState.kCommitted
},
{
pauseFailPoint: "pauseTenantMigrationBeforeLeavingBlockingState",
setUpFailPoints: ["abortTenantMigrationBeforeLeavingBlockingState"],
nextState: TenantMigrationTest.DonorState.kAborted
}].forEach(({pauseFailPoint, setUpFailPoints = [], nextState}) => {
testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState);
});
tenantMigrationTest.stop();
}());
|
