SERVER-28453 Key rotation integration tests

5 files changed, 200 insertions, 0 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
index d35dd4c0751..9ddd6b19111 100644
--- a/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml
@@ -84,6 +84,7 @@ selector:
   - jstests/sharding/drop_configdb.js
   - jstests/sharding/shard_identity_config_update.js
   - jstests/sharding/mongos_does_not_gossip_logical_time_without_keys.js
+  - jstests/sharding/key_rotation.js
   # Runs setShardVersion/getShardVersion against the config server and we don't support retries
   # for this command
   - jstests/sharding/major_version_check.js
diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index ff32a36ebe4..1bd0907f6b1 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -30,6 +30,8 @@ selector:
   - jstests/sharding/after_cluster_time.js
   - jstests/sharding/auth_drop_user_while_logged_in.js
   - jstests/sharding/mongos_does_not_gossip_logical_time_without_keys.js
+  - jstests/sharding/key_rotation.js
+  - jstests/sharding/advance_logical_time_with_valid_signature.js
   # New feature in v3.6 mongo shell.
   - jstests/sharding/causal_consistency_shell_support.js
   # Changes to currentOp format in 3.6.
diff --git a/jstests/noPassthrough/auth_reject_mismatching_logical_times.js b/jstests/noPassthrough/auth_reject_mismatching_logical_times.js
new file mode 100644
index 00000000000..3de5f3f440c
--- /dev/null
+++ b/jstests/noPassthrough/auth_reject_mismatching_logical_times.js
@@ -0,0 +1,71 @@
+/**
+ * Verifies mismatching logical time objects are rejected by a sharded cluster when auth is on. In
+ * noPassthrough because auth is manually set.
+ */
+(function() {
+    "use strict";
+
+    // Given a valid logical time object, returns one with the same signature, but a mismatching
+    // cluster time.
+    function mismatchingLogicalTime(lt) {
+        return Object.merge(lt, {clusterTime: Timestamp(lt.clusterTime.getTime() + 100, 0)});
+    }
+
+    function assertRejectsMismatchingLogicalTime(db) {
+        let validTime = db.runCommand({isMaster: 1}).$logicalTime;
+        let mismatchingTime = mismatchingLogicalTime(validTime);
+
+        assert.commandFailedWithCode(
+            db.runCommand({isMaster: 1, $logicalTime: mismatchingTime}),
+            ErrorCodes.TimeProofMismatch,
+            "expected command with mismatching logical time and signature to be rejected");
+    }
+
+    function assertAcceptsValidLogicalTime(db) {
+        let validTime = db.runCommand({isMaster: 1}).$logicalTime;
+        assert.commandWorked(
+            testDB.runCommand({isMaster: 1, $logicalTime: validTime}),
+            "expected command with valid logical time and signature to be accepted");
+    }
+
+    // Start the sharding test with auth on.
+    const st = new ShardingTest({
+        mongos: 1,
+        manualAddShard: true,
+        mongosWaitsForKeys: true,
+        other: {keyFile: "jstests/libs/key1"}
+    });
+
+    // Create admin user and authenticate as them.
+    st.s.getDB("admin").createUser({user: "foo", pwd: "bar", roles: jsTest.adminUserRoles});
+    st.s.getDB("admin").auth("foo", "bar");
+
+    // Add shard with auth enabled.
+    const rst = new ReplSetTest({nodes: 2});
+    rst.startSet({keyFile: "jstests/libs/key1", shardsvr: ""});
+    rst.initiate();
+    assert.commandWorked(st.s.adminCommand({addShard: rst.getURL()}));
+
+    const testDB = st.s.getDB("test");
+
+    // Unsharded collections reject mismatching logical times and accept valid ones.
+    assertRejectsMismatchingLogicalTime(testDB);
+    assertAcceptsValidLogicalTime(testDB);
+
+    // Initialize sharding.
+    assert.commandWorked(testDB.adminCommand({enableSharding: "test"}));
+    assert.commandWorked(
+        testDB.adminCommand({shardCollection: testDB.foo.getFullName(), key: {_id: 1}}));
+
+    // Sharded collections reject mismatching logical times and accept valid ones.
+    assertRejectsMismatchingLogicalTime(testDB);
+    assertAcceptsValidLogicalTime(testDB);
+
+    // Shards and config servers also reject mismatching times and accept valid ones.
+    assertRejectsMismatchingLogicalTime(rst.getPrimary().getDB("test"));
+    assertAcceptsValidLogicalTime(rst.getPrimary().getDB("test"));
+    assertRejectsMismatchingLogicalTime(st.configRS.getPrimary().getDB("admin"));
+    assertAcceptsValidLogicalTime(st.configRS.getPrimary().getDB("admin"));
+
+    st.stop();
+})();
diff --git a/jstests/sharding/advance_logical_time_with_valid_signature.js b/jstests/sharding/advance_logical_time_with_valid_signature.js
new file mode 100644
index 00000000000..e2a379d981f
--- /dev/null
+++ b/jstests/sharding/advance_logical_time_with_valid_signature.js
@@ -0,0 +1,41 @@
+/**
+ * Tests that the mongo shell can use a logical time with a valid signature to advance a server's
+ * cluster time.
+ */
+(function() {
+    "use strict";
+
+    // Setup 2 mongos processes with mongobridge.
+    let st = new ShardingTest({shards: 1, mongos: 2, useBridge: true, mongosWaitsForKeys: true});
+
+    // Sever outgoing communications from the second mongos.
+    st.s0.disconnect(st.s1);
+    st._configServers.forEach(function(configSvr) {
+        configSvr.disconnect(st.s1);
+    });
+    st._connections.forEach(function(conn) {
+        conn.disconnect(st.s1);
+    });
+
+    let connectedDB = st.s0.getDB("test");
+    let disconnectedDB = st.s1.getDB("test");
+
+    // Send an insert to the connected mongos to advance its cluster time.
+    let res = assert.commandWorked(connectedDB.runCommand({insert: "foo", documents: [{x: 1}]}));
+
+    // Get logicalTime metadata from the connected mongos's response and send it in an isMaster
+    // command to the disconnected mongos. isMaster does not require mongos to contact any other
+    // servers, so the command should succeed.
+    let lt = res.$logicalTime;
+    res = assert.commandWorked(
+        disconnectedDB.runCommand({isMaster: 1, $logicalTime: lt}),
+        "expected the disconnected mongos to accept logical time: " + tojson(lt));
+
+    // Verify logical time response from the disconnected mongos matches what was passed.
+    assert.eq(lt,
+              res.$logicalTime,
+              "expected the disconnected mongos to send logical time: " + tojson(lt) +
+                  ", received: " + tojson(res.$logicalTime));
+
+    st.stop();
+})();
diff --git a/jstests/sharding/key_rotation.js b/jstests/sharding/key_rotation.js
new file mode 100644
index 00000000000..56b3c26c85c
--- /dev/null
+++ b/jstests/sharding/key_rotation.js
@@ -0,0 +1,85 @@
+/**
+ * Tests for causal consistency key rotation. In particular, tests:
+ * - that a sharded cluster with no keys inserts new keys after startup.
+ * - responses from servers in a sharded cluster contain a logical time object with a signature.
+ * - manual key rotation is possible by deleting existing keys and restarting the cluster.
+ */
+(function() {
+    "use strict";
+
+    let st = new ShardingTest({shards: {rs0: {nodes: 2}}, mongosWaitsForKeys: true});
+
+    // Verify after startup there is a new key in admin.system.keys.
+    jsTestLog("Verify the admin.system.keys collection after startup.");
+
+    let startupKeys = st.s.getDB("admin").system.keys.find();
+    assert(startupKeys.count() >= 2);  // Should be at least two generations of keys available.
+    startupKeys.toArray().forEach(function(key, i) {
+        assert.hasFields(
+            key,
+            ["purpose", "key", "expiresAt"],
+            "key document " + i + ": " + tojson(key) + ", did not have all of the expected fields");
+    });
+
+    // Verify there is a $logicalTime with a signature in the response.
+    jsTestLog("Verify a signature is included in the logical time in a response.");
+
+    let res = assert.commandWorked(st.s.getDB("test").runCommand({isMaster: 1}));
+    assert.hasFields(res, ["$logicalTime"]);
+    assert.hasFields(res.$logicalTime, ["signature"]);
+    assert.hasFields(res.$logicalTime.signature, ["hash", "keyId"]);
+
+    // Verify manual key rotation.
+    jsTestLog("Verify manual key rotation.");
+
+    // Pause key generation on the config server primary.
+    st.configRS.getPrimary().getDB("admin").runCommand(
+        {configureFailPoint: "disableKeyGeneration", mode: "alwaysOn"});
+
+    // Delete all existing keys.
+    res =
+        st.configRS.getPrimary().getDB("admin").system.keys.remove({purpose: "SigningClusterTime"});
+    assert(res.nRemoved >= 2);
+    assert(st.s.getDB("admin").system.keys.find().count() == 0);
+
+    // Restart the config servers, so they will create new keys once the failpoint is disabled.
+    st.configRS.stopSet(null /* signal */, true /* forRestart */);
+    st.configRS.startSet(
+        {restart: true, setParameter: {"failpoint.disableKeyGeneration": "{'mode':'alwaysOn'}"}});
+
+    // Limit the max time between refreshes on the config server, so new keys are created quickly.
+    st.configRS.getPrimary().adminCommand({
+        "configureFailPoint": "maxKeyRefreshWaitTimeOverrideMS",
+        "mode": "alwaysOn",
+        "data": {"overrideMS": 1000}
+    });
+
+    // Kill and restart all shards and mongos processes so they have no keys in memory.
+    st.rs0.stopSet(null /* signal */, true /* forRestart */);
+    st.rs0.startSet({restart: true});
+    st.restartMongos(0);
+
+    // The shard primary should return a dummy signed logical time, because there are no keys.
+    res = assert.commandWorked(st.rs0.getPrimary().getDB("test").runCommand({isMaster: 1}));
+    assert.hasFields(res, ["$logicalTime", "operationTime"]);
+    assert.eq(res.$logicalTime.signature.keyId, NumberLong(0));
+
+    // Mongos shouldn't return a logical time at all.
+    res = assert.commandWorked(st.s.getDB("test").runCommand({isMaster: 1}));
+    assert.throws(function() {
+        assert.hasFields(res, ["$logicalTime", "operationTime"]);
+    }, [], "expected the mongos not to return logical time or operation time");
+
+    // Resume key generation.
+    st.configRS.getPrimary().getDB("admin").runCommand(
+        {configureFailPoint: "disableKeyGeneration", mode: "off"});
+
+    // Wait for config server primary to create new keys.
+    assert.soonNoExcept(function() {
+        let keys = st.s.getDB("admin").system.keys.find();
+        assert(keys.count() >= 2);
+        return true;
+    }, "expected the config server primary to create new keys");
+
+    st.stop();
+})();