SERVER-26016 Add more injected-state tests of oplog replay at startup

1 files changed, 382 insertions, 37 deletions

diff --git a/jstests/replsets/oplog_replay_on_startup.js b/jstests/replsets/oplog_replay_on_startup.js
index 8f6291be9cd..4b1affa61dd 100644
--- a/jstests/replsets/oplog_replay_on_startup.js
+++ b/jstests/replsets/oplog_replay_on_startup.js
@@ -5,8 +5,9 @@
 (function() {
     "use strict";
 
+    var ns = "test.coll";
+
     var rst = new ReplSetTest({
-        name: "server_7200",
         nodes: 1,
     });
 
@@ -14,56 +15,400 @@
     rst.initiate();
 
     var conn = rst.getPrimary();  // Waits for PRIMARY state.
-    var ns = "test.coll";
+    var term = conn.getCollection('local.oplog.rs').find().sort({$natural: -1}).limit(1).next().t;
+    if (typeof(term) == 'undefined') {
+        term = -1;  // Use a dummy term for PV0.
+    }
+
+    function runTest({
+        oplogEntries,
+        collectionContents,
+        deletePoint,
+        begin,
+        minValid,
+        expectedState,
+        expectedApplied,
+    }) {
+        if (term != -1) {
+            term++;  // Each test gets a new term on PV1 to ensure OpTimes always move forward.
+        }
+
+        conn = rst.restart(0, {noReplSet: true});  // Restart as a standalone node.
+        assert.neq(null, conn, "failed to restart");
+        var oplog = conn.getCollection('local.oplog.rs');
+        var minValidColl = conn.getCollection('local.replset.minvalid');
+        var coll = conn.getCollection(ns);
+
+        // Reset state to empty.
+        assert.commandWorked(oplog.runCommand('emptycapped'));
+        coll.drop();
+        assert.commandWorked(coll.runCommand('create'));
+
+        var ts = (num) => num === null ? Timestamp() : Timestamp(1000, num);
+
+        oplogEntries.forEach((num) => {
+            assert.writeOK(oplog.insert({
+                ts: ts(num),
+                t: term,
+                h: 1,
+                op: 'i',
+                ns: ns,
+                o: {_id: num},
+            }));
+        });
+
+        collectionContents.forEach((num) => {
+            assert.writeOK(coll.insert({_id: num}));
+        });
+
+        var injectedMinValidDoc = {
+            _id: ObjectId(),
+
+            // minvalid:
+            ts: ts(minValid),
+            t: term,
+
+            // appliedThrough
+            begin: {
+                ts: ts(begin),
+                t: term,
+            },
+
+            oplogDeleteFromPoint: ts(deletePoint),
+        };
+
+        // This weird mechanism is the only way to bypass mongod's attempt to fill in null
+        // Timestamps.
+        assert.writeOK(minValidColl.remove({}));
+        assert.writeOK(minValidColl.update({}, {$set: injectedMinValidDoc}, {upsert: true}));
+        assert.eq(minValidColl.findOne(),
+                  injectedMinValidDoc,
+                  "If the Timestamps differ, the server may be filling in the null timestamps");
+
+        try {
+            conn = rst.restart(0);  // Restart in replSet mode again.
+        } catch (e) {
+            assert.eq(expectedState, 'FATAL', 'node failed to restart: ' + e);
+            return;
+        }
+
+        // Wait for the node to go to SECONDARY if it is able.
+        assert.soon(
+            () =>
+                conn.adminCommand('serverStatus').metrics.repl.apply.attemptsToBecomeSecondary > 0,
+            () => conn.adminCommand('serverStatus').metrics.repl.apply.attemptsToBecomeSecondary);
 
-    conn.getCollection(ns).insert({_id: 1});
+        var isMaster = conn.adminCommand('ismaster');
+        switch (expectedState) {
+            case 'SECONDARY':
+                // Primary is also acceptable since once a node becomes secondary, it will try to
+                // become primary if it is eligible and has enough votes (which this node does).
+                // This is supposed to test that we reach secondary, not that we stay there.
+                assert(isMaster.ismaster || isMaster.secondary,
+                       'not PRIMARY or SECONDARY: ' + tojson(isMaster));
+                break;
 
-    conn = rst.restart(0, {noReplSet: true});  // Restart as a standalone node.
-    assert.neq(null, conn, "failed to restart");
+            case 'RECOVERING':
+                assert(!isMaster.ismaster && !isMaster.secondary,
+                       'not in RECOVERING: ' + tojson(isMaster));
 
-    var originalOp = conn.getCollection('local.oplog.rs').find().sort({$natural: -1}).limit(1)[0];
-    if (!('t' in originalOp)) {
-        originalOp.t = -1;  // Add the dummy term for PV0.
+                // Restart as a standalone node again so we can read from the collection.
+                conn = rst.restart(0, {noReplSet: true});
+                break;
+
+            case 'FATAL':
+                doassert("server startup didn't fail when it should have");
+                break;
+
+            default:
+                doassert(`expectedState ${expectedState} is not supported`);
+        }
+
+        // Ensure the oplog has the entries it should have and none that it shouldn't.
+        assert.eq(conn.getCollection('local.oplog.rs')
+                      .find({ns: ns, op: 'i'})
+                      .sort({$natural: 1})
+                      .map((op) => op.o._id),
+                  expectedApplied);
+
+        // Ensure that all ops that should have been applied were.
+        conn.setSlaveOk(true);
+        assert.eq(conn.getCollection(ns).find().sort({_id: 1}).map((obj) => obj._id),
+                  expectedApplied);
     }
-    var tsPlus = (inc) => Timestamp(originalOp.ts.t, originalOp.ts.i + inc);
 
-    var injectedOps = [
-        // These will be applied.
-        {ts: tsPlus(1), t: originalOp.t, h: 1, op: 'i', ns: ns, o: {_id: 2}},
-        {ts: tsPlus(2), t: originalOp.t, h: 1, op: 'i', ns: ns, o: {_id: 3}},
+    //
+    // Normal 3.4 cases
+    //
 
-        // These will be deleted.
-        {ts: tsPlus(3), t: originalOp.t, h: 1, op: 'i', ns: ns, o: {_id: 4}},
-        {ts: tsPlus(4), t: originalOp.t, h: 1, op: 'i', ns: ns, o: {_id: 5}},
-    ];
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: null,
+        minValid: null,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-    var injectedMinValidDoc = {
-        // minvalid:
-        ts: injectedOps[1].ts,
-        t: injectedOps[1].t,
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: null,
+        minValid: 2,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-        // appliedThrough
-        begin: {
-            ts: originalOp.ts,
-            t: originalOp.t,
-        },
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: null,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-        oplogDeleteFromPoint: injectedOps[2].ts,
-    };
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-    conn.getCollection("local.oplog.rs").insert(injectedOps);
-    conn.getCollection("local.replset.minvalid").update({}, injectedMinValidDoc, {upsert: true});
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-    rst.restart(0);  // Restart in replSet mode again.
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
-    var conn = rst.getPrimary();  // Waits for PRIMARY state.
-    assert.neq(null, conn, "failed to restart");
+    runTest({
+        oplogEntries: [1, 2, 3, /*4,*/ 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
+
+    //
+    // 3.2 -> 3.4 upgrade cases
+    //
 
-    // Ensure the oplog has the entries it should have and none that it shouldn't.
-    assert.eq(conn.getCollection('local.oplog.rs').distinct('o._id', {ns: ns, op: 'i'}), [1, 2, 3]);
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5],
+        collectionContents: [1, 2, 3],
+        deletePoint: null,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3, 4, 5],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5],
+        collectionContents: [1, 2, 3, 4, 5],
+        deletePoint: null,
+        begin: null,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3, 4, 5],
+    });
+
+    //
+    // 3.4 -> 3.2 -> 3.4 downgrade/reupgrade cases
+    //
+
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, /*4,*/ 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: 4,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: 2,
+        begin: null,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3],
+        deletePoint: 2,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5],
+        collectionContents: [1, 2, 3],
+        deletePoint: 2,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'RECOVERING',
+        expectedApplied: [1, 2, 3, 4, 5],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: 2,
+        begin: 3,
+        minValid: 6,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
 
-    // Ensure that all ops that should have been applied were.
-    assert.eq(conn.getCollection(ns).distinct('_id'), [1, 2, 3]);
+    //
+    // These states should be impossible to get into.
+    //
+
+    runTest({
+        oplogEntries: [1, 2, 3],
+        collectionContents: [1, 2, 3, 4],
+        deletePoint: null,
+        begin: 4,
+        minValid: null,  // doesn't matter.
+        expectedState: 'FATAL',
+    });
+
+    runTest({
+        oplogEntries: [4, 5, 6],
+        collectionContents: [1, 2],
+        deletePoint: 2,
+        begin: 3,
+        minValid: null,  // doesn't matter.
+        expectedState: 'FATAL',
+    });
+
+    runTest({
+        oplogEntries: [4, 5, 6],
+        collectionContents: [1, 2],
+        deletePoint: null,
+        begin: 3,
+        minValid: null,  // doesn't matter.
+        expectedState: 'FATAL',
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3],
+        deletePoint: 2,
+        begin: 3,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3, 4, 5],
+        deletePoint: null,
+        begin: 5,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5, 6],
+        collectionContents: [1, 2, 3, 4, 5],
+        deletePoint: null,
+        begin: 5,
+        minValid: null,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3, 4, 5, 6],
+    });
+
+    runTest({
+        oplogEntries: [1, 2, 3, 4, 5],
+        collectionContents: [1],
+        deletePoint: 4,
+        begin: 1,
+        minValid: 3,
+        expectedState: 'SECONDARY',
+        expectedApplied: [1, 2, 3],
+    });
 
     rst.stopSet();
 })();