SERVER-27722 write tests for upgrade/downgrade scenario for logical clock

3 files changed, 336 insertions, 0 deletions

diff --git a/jstests/multiVersion/causal_consistency_downgrade_cluster.js b/jstests/multiVersion/causal_consistency_downgrade_cluster.js
new file mode 100644
index 00000000000..f173a30cf7a
--- /dev/null
+++ b/jstests/multiVersion/causal_consistency_downgrade_cluster.js
@@ -0,0 +1,104 @@
+/**
+ * Test the downgrade of a sharded cluster from latest to last-stable version succeeds, verifying
+ * behavior related to causal consistency at each stage.
+ */
+(function() {
+    "use strict";
+
+    load("jstests/multiVersion/libs/multi_rs.js");
+    load("jstests/multiVersion/libs/multi_cluster.js");
+    load("jstests/multiVersion/libs/causal_consistency_helpers.js");
+
+    if (!supportsMajorityReadConcern()) {
+        jsTestLog("Skipping test since storage engine doesn't support majority read concern.");
+        return;
+    }
+
+    // Start a cluster at the latest version, with majority read concern enabled.
+    var st = new ShardingTest({
+        shards: 2,
+        mongos: 1,
+        other: {
+            mongosOptions: {binVersion: "latest"},
+            configOptions: {binVersion: "latest"},
+            // Set catchUpTimeoutMillis for compatibility with v3.4.
+            configReplSetTestOptions: {settings: {catchUpTimeoutMillis: 2000}},
+            rsOptions: {
+                binVersion: "latest",
+                settings: {catchUpTimeoutMillis: 2000},
+                enableMajorityReadConcern: ""
+            },
+            rs: true
+        }
+    });
+    st.configRS.awaitReplication();
+
+    st.s.getDB("test").runCommand({insert: "foo", documents: [{_id: 1, x: 1}]});
+
+    // Both logical and operation times are returned, and logical times are signed by mongos. Mongos
+    // doesn't wait for keys at startup, so retry.
+    assert.soonNoExcept(function() {
+        assertContainsLogicalAndOperationTime(st.s.getDB("test").runCommand({isMaster: 1}),
+                                              {initialized: true, signed: true});
+        return true;
+    });
+
+    // Mongos and shards can accept afterClusterTime reads.
+    assertAfterClusterTimeReadSucceeds(st.s.getDB("test"), "foo");
+    assertAfterClusterTimeReadSucceeds(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadSucceeds(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Change featureCompatibilityVersion to 3.4.
+    assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "3.4"}));
+
+    // Mongos still signs logical times, because they are held in memory.
+    assertContainsLogicalAndOperationTime(st.s.getDB("test").runCommand({isMaster: 1}),
+                                          {initialized: true, signed: true});
+
+    // afterClusterTime reads are no longer accepted.
+    assertAfterClusterTimeReadFails(st.s.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Shards and the config servers should no longer send logical or operation times.
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // Downgrade mongos first.
+    jsTest.log("Downgrading mongos servers.");
+    st.upgradeCluster("last-stable", {upgradeConfigs: false, upgradeShards: false});
+    st.restartMongoses();
+
+    // Mongos should no longer return operation or logical times.
+    assertDoesNotContainLogicalOrOperationTime(st.s.getDB("test").runCommand({isMaster: 1}));
+
+    // Downgrade shards next.
+    jsTest.log("Downgrading shard servers.");
+    st.upgradeCluster("last-stable", {upgradeConfigs: false, upgradeMongos: false});
+    st.restartMongoses();
+
+    // Finally, downgrade config servers.
+    jsTest.log("Downgrading config servers.");
+    st.upgradeCluster("last-stable", {upgradeMongos: false, upgradeShards: false});
+    st.restartMongoses();
+
+    // No servers return logical or operation time.
+    assertDoesNotContainLogicalOrOperationTime(st.s.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // afterClusterTime reads are still not accepted.
+    assertAfterClusterTimeReadFails(st.s.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs1.getPrimary().getDB("test"), "foo");
+
+    st.stop();
+})();
diff --git a/jstests/multiVersion/causal_consistency_upgrade_cluster.js b/jstests/multiVersion/causal_consistency_upgrade_cluster.js
new file mode 100644
index 00000000000..916684ea29a
--- /dev/null
+++ b/jstests/multiVersion/causal_consistency_upgrade_cluster.js
@@ -0,0 +1,154 @@
+/**
+ * Tests upgrading a cluster with two shards and two mongos servers from last stable to current
+ * version, verifying the behavior of $logicalTime metadata and afterClusterTime commands throughout
+ * the process.
+ */
+(function() {
+    "use strict";
+
+    load("jstests/multiVersion/libs/multi_rs.js");
+    load("jstests/multiVersion/libs/multi_cluster.js");
+    load("jstests/multiVersion/libs/causal_consistency_helpers.js");
+
+    if (!supportsMajorityReadConcern()) {
+        jsTestLog("Skipping test since storage engine doesn't support majority read concern.");
+        return;
+    }
+
+    // Start a cluster at the last stable version, with majority read concern enabled.
+    var st = new ShardingTest({
+        shards: 2,
+        mongos: 2,
+        other: {
+            mongosOptions: {binVersion: "last-stable"},
+            configOptions: {binVersion: "last-stable"},
+            rsOptions: {binVersion: "last-stable", enableMajorityReadConcern: ""},
+            rs: true
+        }
+    });
+    st.configRS.awaitReplication();
+
+    st.s.getDB("test").runCommand({insert: "foo", documents: [{_id: 1, x: 1}]});
+
+    // No servers return logical or operation time.
+    assertDoesNotContainLogicalOrOperationTime(st.s0.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(st.s1.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // Upgrade the config servers.
+    jsTest.log("Upgrading config servers.");
+    st.upgradeCluster("latest", {upgradeMongos: false, upgradeShards: false});
+    st.restartMongoses();
+
+    // Mongod and mongos cannot accept afterClusterTime reads.
+    assertAfterClusterTimeReadFails(st.s0.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.s1.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Config servers still don't return logical or operation times.
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    assertDoesNotContainLogicalOrOperationTime(st.s0.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(st.s1.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // Then upgrade the shard servers.
+    jsTest.log("Upgrading shard servers.");
+    st.upgradeCluster("latest", {upgradeConfigs: false, upgradeMongos: false});
+    st.restartMongoses();
+
+    // Mongod and mongos still cannot accept afterClusterTime reads.
+    assertAfterClusterTimeReadFails(st.s0.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.s1.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Shards still don't return logical or operation times.
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // Neither do config servers.
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    assertDoesNotContainLogicalOrOperationTime(st.s0.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(st.s1.getDB("test").runCommand({isMaster: 1}));
+
+    // Finally, upgrade mongos servers.
+    jsTest.log("Upgrading mongos servers.");
+    st.upgradeCluster("latest", {upgradeConfigs: false, upgradeShards: false});
+    st.restartMongoses();
+
+    // afterClusterTime reads are still not accepted.
+    assertAfterClusterTimeReadFails(st.s.getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadFails(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Neither mongos returns logical time or operation time, because there are no keys in the
+    // config server, since feature compatibility version is still 3.4.
+    assertDoesNotContainLogicalOrOperationTime(st.s0.getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(st.s1.getDB("test").runCommand({isMaster: 1}));
+
+    // All shards and the config servers still don't return logical or operation time.
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assertDoesNotContainLogicalOrOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}));
+
+    // Set feature compatibility version to 3.6 on one mongos.
+    assert.commandWorked(st.s0.getDB("admin").runCommand({setFeatureCompatibilityVersion: "3.6"}));
+
+    // Now shards and config servers return dummy signed logical times and operation times.
+    assertContainsLogicalAndOperationTime(
+        st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}),
+        {initialized: true, signed: false});
+    assertContainsLogicalAndOperationTime(
+        st.rs1.getPrimary().getDB("test").runCommand({isMaster: 1}),
+        {initialized: true, signed: false});
+    assertContainsLogicalAndOperationTime(
+        st.configRS.getPrimary().getDB("test").runCommand({isMaster: 1}),
+        {initialized: true, signed: false});
+
+    // Once the config primary creates keys, both mongos servers discover them and start returning
+    // signed logical times.
+    assert.soonNoExcept(function() {
+        assertContainsLogicalAndOperationTime(st.s0.getDB("test").runCommand({isMaster: 1}),
+                                              {initialized: true, signed: true});
+        assertContainsLogicalAndOperationTime(st.s1.getDB("test").runCommand({isMaster: 1}),
+                                              {initialized: true, signed: true});
+        return true;
+    });
+
+    // Now shards and mongos can accept afterClusterTime reads.
+    assertAfterClusterTimeReadSucceeds(st.s0.getDB("test"), "foo");
+    assertAfterClusterTimeReadSucceeds(st.s1.getDB("test"), "foo");
+    assertAfterClusterTimeReadSucceeds(st.rs0.getPrimary().getDB("test"), "foo");
+    assertAfterClusterTimeReadSucceeds(st.rs1.getPrimary().getDB("test"), "foo");
+
+    // Causally consistent requests are correctly processed.
+    let res = assert.commandWorked(
+        st.s.getDB("test").runCommand({insert: "foo", documents: [{_id: 2, x: 2}]}));
+    res = assert.commandWorked(
+        st.s.getDB("test").runCommand({delete: "foo", deletes: [{q: {_id: 1}, limit: 1}]}));
+
+    let operationTime = res.operationTime;
+    res = assert.commandWorked(st.s.getDB("test").runCommand(
+        {find: "foo", readConcern: {level: "majority", afterClusterTime: operationTime}}));
+    assert.eq(res.cursor.firstBatch, [{_id: 2, x: 2}]);
+
+    st.stop();
+})();
diff --git a/jstests/multiVersion/libs/causal_consistency_helpers.js b/jstests/multiVersion/libs/causal_consistency_helpers.js
new file mode 100644
index 00000000000..2f3734be0ee
--- /dev/null
+++ b/jstests/multiVersion/libs/causal_consistency_helpers.js
@@ -0,0 +1,78 @@
+/**
+ * Helper functions for testing causal consistency.
+ */
+
+load('jstests/replsets/rslib.js');  // For startSetIfSupportsReadMajority.
+
+function assertAfterClusterTimeReadFails(db, collName) {
+    assert.commandFailed(db.runCommand(
+        {find: collName, readConcern: {level: "majority", afterClusterTime: Timestamp(1, 1)}}));
+}
+
+function assertAfterClusterTimeReadSucceeds(db, collName) {
+    assert.commandWorked(db.runCommand(
+        {find: collName, readConcern: {level: "majority", afterClusterTime: Timestamp(1, 1)}}));
+}
+
+function assertDoesNotContainLogicalOrOperationTime(res) {
+    assertDoesNotContainLogicalTime(res);
+    assertDoesNotContainOperationTime(res);
+}
+
+function assertDoesNotContainLogicalTime(res) {
+    assert.eq(res.$logicalTime, undefined);
+}
+
+function assertDoesNotContainOperationTime(res) {
+    assert.eq(res.operationTime, undefined);
+}
+
+function assertContainsLogicalAndOperationTime(res, opts) {
+    assertContainsLogicalTime(res, opts);
+    assertContainsOperationTime(res, opts);
+}
+
+function assertContainsLogicalTime(res, opts) {
+    assert.hasFields(res, ["$logicalTime"]);
+    assert.hasFields(res.$logicalTime, ["clusterTime", "signature"]);
+    assert.hasFields(res.$logicalTime.signature, ["hash", "keyId"]);
+
+    if (opts.signed !== undefined) {
+        // Signed logical times have a keyId greater than 0.
+        if (opts.signed) {
+            assert(res.$logicalTime.signature.keyId > NumberLong(0));
+        } else {
+            assert.eq(res.$logicalTime.signature.keyId, NumberLong(0));
+        }
+    }
+
+    if (opts.initialized !== undefined) {
+        // Initialized operation times are greater than a null timestamp.
+        if (opts.initialized) {
+            assert.eq(bsonWoCompare(res.$logicalTime.clusterTime, Timestamp(0, 0)), 1);
+        } else {
+            assert.eq(bsonWoCompare(res.$logicalTime.clusterTime, Timestamp(0, 0)), 0);
+        }
+    }
+}
+
+function assertContainsOperationTime(res, opts) {
+    assert.hasFields(res, ["operationTime"]);
+
+    if (opts.initialized !== undefined) {
+        // Initialized operation times are greater than a null timestamp.
+        if (opts.initialized) {
+            assert.eq(bsonWoCompare(res.operationTime, Timestamp(0, 0)), 1);
+        } else {
+            assert.eq(bsonWoCompare(res.operationTime, Timestamp(0, 0)), 0);
+        }
+    }
+}
+
+function supportsMajorityReadConcern() {
+    const rst = new ReplSetTest({nodes: 1, nodeOptions: {enableMajorityReadConcern: ""}});
+    if (!startSetIfSupportsReadMajority(rst)) {
+        return false;
+    }
+    return true;
+}