SERVER-49749 Test rotate functionality in various cases

6 files changed, 415 insertions, 0 deletions

diff --git a/jstests/ssl/client_x509_rotate.js b/jstests/ssl/client_x509_rotate.js
new file mode 100644
index 00000000000..95a170768ee
--- /dev/null
+++ b/jstests/ssl/client_x509_rotate.js
@@ -0,0 +1,124 @@
+// Check that rotation works for the cluster certificate in a sharded cluster
+
+(function() {
+"use strict";
+
+load('jstests/ssl/libs/ssl_helpers.js');
+
+if (determineSSLProvider() === "openssl") {
+    return;
+}
+
+let mongos;
+function getConnPoolHosts() {
+    const ret = mongos.adminCommand({connPoolStats: 1});
+    assert.commandWorked(ret);
+    jsTestLog("Connection pool stats by host: " + tojson(ret.hosts));
+    return ret.hosts;
+}
+
+const dbPath = MongoRunner.toRealDir("$dataDir/cluster_x509_rotate_test/");
+mkdir(dbPath);
+
+copyCertificateFile("jstests/libs/ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/client.pem", dbPath + "/client-test.pem");
+copyCertificateFile("jstests/libs/server.pem", dbPath + "/server-test.pem");
+
+// server certificate is held constant so that shell can still connect
+// we start a cluster using the old certificates, then rotate one shard to use new certificates.
+// Make sure that mongos can communicate with every connected host EXCEPT that shard before a
+// rotate, and make sure it can communicate with ONLY that shard after a rotate.
+const mongosOptions = {
+    sslMode: "requireSSL",
+    sslPEMKeyFile: "jstests/libs/server.pem",
+    sslCAFile: dbPath + "/ca-test.pem",
+    sslClusterFile: dbPath + "/client-test.pem",
+    sslAllowInvalidHostnames: "",
+};
+
+const configOptions = {
+    sslMode: "requireSSL",
+    sslPEMKeyFile: dbPath + "/server-test.pem",
+    sslCAFile: dbPath + "/ca-test.pem",
+    sslAllowInvalidHostnames: "",
+};
+
+const sharding_config = {
+    config: 1,
+    mongos: 1,
+    shards: 3,
+    other: {
+        configOptions: configOptions,
+        mongosOptions: mongosOptions,
+        rsOptions: configOptions,
+        shardOptions: configOptions,
+    }
+};
+
+const st = new ShardingTest(sharding_config);
+
+mongos = st.s0;
+
+// Keep track of the hosts we hit in the initial ping multicast to compare against later multicasts
+let output = mongos.adminCommand({multicast: {ping: 0}});
+assert.eq(output.ok, 1);
+
+let keys = [];
+for (let key in output.hosts) {
+    keys.push(key);
+}
+
+const rst = st.rs0;
+const primary = rst.getPrimary();
+
+// Swap out the certificate files and rotate the primary shard.
+copyCertificateFile("jstests/libs/trusted-ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/trusted-client.pem", dbPath + "/client-test.pem");
+copyCertificateFile("jstests/libs/trusted-server.pem", dbPath + "/server-test.pem");
+
+assert.commandWorked(primary.adminCommand({rotateCertificates: 1}));
+
+// Make sure the primary is initially present
+assert(primary.host in getConnPoolHosts());
+
+// Drop connection to all hosts to see what we can reconnect to
+assert.commandWorked(mongos.adminCommand({dropConnections: 1, hostAndPort: keys}));
+assert(!(primary.host in getConnPoolHosts()));
+
+output = mongos.adminCommand({multicast: {ping: 0}});
+jsTestLog("Multicast 1 output: " + tojson(output));
+// multicast should fail, because the primary shard isn't hit
+assert.eq(output.ok, 0);
+for (let host in output.hosts) {
+    if (host === primary.host) {
+        assert.eq(output.hosts[host].ok, 0);
+    } else {
+        assert.eq(output.hosts[host].ok, 1);
+    }
+}
+for (let key of keys) {
+    assert(key in output.hosts);
+}
+
+// rotate, drop all connections, re-multicast and see what we now hit
+assert.commandWorked(mongos.adminCommand({rotateCertificates: 1}));
+
+mongos.adminCommand({dropConnections: 1, hostAndPort: keys});
+assert(!(primary.host in getConnPoolHosts()));
+
+output = mongos.adminCommand({multicast: {ping: 0}});
+jsTestLog("Multicast 2 output: " + tojson(output));
+assert.eq(output.ok, 0);
+for (let host in output.hosts) {
+    if (host === primary.host) {
+        assert.eq(output.hosts[host].ok, 1);
+    } else {
+        assert.eq(output.hosts[host].ok, 0);
+    }
+}
+for (let key of keys) {
+    assert(key in output.hosts);
+}
+// Don't call st.stop() -- breaks because cluster is only partially rotated (this is hard to fix)
+return;
+}());
diff --git a/jstests/ssl/cluster_x509_rotate.js b/jstests/ssl/cluster_x509_rotate.js
new file mode 100644
index 00000000000..4637e16b31c
--- /dev/null
+++ b/jstests/ssl/cluster_x509_rotate.js
@@ -0,0 +1,92 @@
+// Check that rotation works for the cluster certificate
+
+(function() {
+"use strict";
+
+load('jstests/ssl/libs/ssl_helpers.js');
+
+if (determineSSLProvider() === "openssl") {
+    return;
+}
+
+const dbPath = MongoRunner.toRealDir("$dataDir/cluster_x509_rotate_test/");
+mkdir(dbPath);
+
+copyCertificateFile("jstests/libs/ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/client.pem", dbPath + "/client-test.pem");
+copyCertificateFile("jstests/libs/server.pem", dbPath + "/server-test.pem");
+
+// Make replset with old certificates, rotate to new certificates, and try to add
+// a node with new certificates.
+const rst = new ReplSetTest({nodes: 2});
+rst.startSet({
+    sslMode: "requireSSL",
+    sslPEMKeyFile: dbPath + "/server-test.pem",
+    sslCAFile: dbPath + "/ca-test.pem",
+    sslClusterFile: dbPath + "/client-test.pem",
+    sslAllowInvalidHostnames: "",
+});
+
+rst.initiate();
+rst.awaitReplication();
+
+copyCertificateFile("jstests/libs/trusted-ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/trusted-client.pem", dbPath + "/client-test.pem");
+copyCertificateFile("jstests/libs/trusted-server.pem", dbPath + "/server-test.pem");
+
+for (let node of rst.nodes) {
+    assert.commandWorked(node.adminCommand({rotateCertificates: 1}));
+}
+
+const newnode = rst.add({
+    sslMode: "requireSSL",
+    sslPEMKeyFile: "jstests/libs/trusted-server.pem",
+    sslCAFile: "jstests/libs/trusted-ca.pem",
+    sslClusterFile: "jstests/libs/trusted-client.pem",
+    sslAllowInvalidHostnames: "",
+    waitForConnect: false,
+});
+
+// Emulate waitForConnect so we wait for new node to come up before killing rst
+const host = "localhost:" + newnode.port;
+assert.soon(() => {
+    return 0 ===
+        runMongoProgram("mongo",
+                        "--ssl",
+                        "--sslAllowInvalidHostnames",
+                        "--host",
+                        host,
+                        "--sslPEMKeyFile",
+                        "jstests/libs/trusted-client.pem",
+                        "--sslCAFile",
+                        "jstests/libs/trusted-ca.pem",
+                        "--eval",
+                        ";");
+});
+
+rst.reInitiate();
+
+// Make sure each node can connect to each other node
+for (let node of rst.nodeList()) {
+    for (let target of rst.nodeList()) {
+        if (node !== target) {
+            assert.eq(0,
+                      runMongoProgram(
+                          "mongo",
+                          "--ssl",
+                          "--sslAllowInvalidHostnames",
+                          "--host",
+                          node,
+                          "--sslPEMKeyFile",
+                          "jstests/libs/trusted-client.pem",
+                          "--sslCAFile",
+                          "jstests/libs/trusted-ca.pem",
+                          "--eval",
+                          `assert.commandWorked(db.adminCommand({replSetTestEgress: 1, target: "${
+                              target}"}));`));
+        }
+    }
+}
+
+rst.stopSet();
+}());
diff --git a/jstests/ssl/crl_x509_rotate.js b/jstests/ssl/crl_x509_rotate.js
new file mode 100644
index 00000000000..8781c5c75c1
--- /dev/null
+++ b/jstests/ssl/crl_x509_rotate.js
@@ -0,0 +1,71 @@
+// Check that rotation works for the CRL
+
+(function() {
+"use strict";
+
+load('jstests/ssl/libs/ssl_helpers.js');
+
+if (determineSSLProvider() === "openssl" || determineSSLProvider() === "apple") {
+    return;
+}
+
+const dbPath = MongoRunner.toRealDir("$dataDir/cluster_x509_rotate_test/");
+mkdir(dbPath);
+
+copyCertificateFile("jstests/libs/crl.pem", dbPath + "/crl-test.pem");
+
+const mongod = MongoRunner.runMongod({
+    sslMode: "requireSSL",
+    sslPEMKeyFile: "jstests/libs/server.pem",
+    sslCAFile: "jstests/libs/ca.pem",
+    sslCRLFile: dbPath + "/crl-test.pem"
+});
+
+const host = "localhost:" + mongod.port;
+
+// Make sure that client-revoked can connect at first
+let out = runMongoProgram("mongo",
+                          "--host",
+                          host,
+                          "--ssl",
+                          "--sslPEMKeyFile",
+                          "jstests/libs/client_revoked.pem",
+                          "--sslCAFile",
+                          "jstests/libs/ca.pem",
+                          "--eval",
+                          ";");
+assert.eq(out, 0, "Initial mongo invocation failed");
+
+// Rotate in new CRL
+copyCertificateFile("jstests/libs/crl_client_revoked.pem", dbPath + "/crl-test.pem");
+
+assert.commandWorked(mongod.adminCommand({rotateCertificates: 1}));
+
+// Make sure client-revoked can't connect
+out = runMongoProgram("mongo",
+                      "--host",
+                      host,
+                      "--ssl",
+                      "--sslPEMKeyFile",
+                      "jstests/libs/client_revoked.pem",
+                      "--sslCAFile",
+                      "jstests/libs/ca.pem",
+                      "--eval",
+                      ";");
+assert.neq(out, 0, "Mongo invocation did not fail");
+
+// Make sure client can still connect
+out = runMongoProgram("mongo",
+                      "--host",
+                      host,
+                      "--ssl",
+                      "--sslPEMKeyFile",
+                      "jstests/libs/client.pem",
+                      "--sslCAFile",
+                      "jstests/libs/ca.pem",
+                      "--eval",
+                      ";");
+assert.eq(out, 0, "Mongo invocation failed");
+
+MongoRunner.stopMongod(mongod);
+}());
\ No newline at end of file
diff --git a/jstests/ssl/libs/ssl_helpers.js b/jstests/ssl/libs/ssl_helpers.js
index 2045cc464b9..2c76f4be797 100644
--- a/jstests/ssl/libs/ssl_helpers.js
+++ b/jstests/ssl/libs/ssl_helpers.js
@@ -367,3 +367,14 @@ function opensslVersionAsInt() {
 function supportsStapling() {
     return opensslVersionAsInt() >= 0x01000200;
 }
+
+function copyCertificateFile(a, b) {
+    if (_isWindows()) {
+        // correctly replace forward slashes for Windows
+        a = a.replace(/\//g, "\\");
+        b = b.replace(/\//g, "\\");
+        assert.eq(0, runProgram("cmd.exe", "/c", "copy", a, b));
+        return;
+    }
+    assert.eq(0, runProgram("cp", a, b));
+}
diff --git a/jstests/ssl/server_x509_rotate.js b/jstests/ssl/server_x509_rotate.js
new file mode 100644
index 00000000000..892561c830d
--- /dev/null
+++ b/jstests/ssl/server_x509_rotate.js
@@ -0,0 +1,61 @@
+// Check that rotation works for the server certificate
+
+(function() {
+"use strict";
+
+load('jstests/ssl/libs/ssl_helpers.js');
+
+if (determineSSLProvider() === "openssl") {
+    return;
+}
+
+const dbPath = MongoRunner.toRealDir("$dataDir/cluster_x509_rotate_test/");
+mkdir(dbPath);
+
+copyCertificateFile("jstests/libs/ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/server.pem", dbPath + "/server-test.pem");
+
+const mongod = MongoRunner.runMongod({
+    sslMode: "requireSSL",
+    sslPEMKeyFile: dbPath + "/server-test.pem",
+    sslCAFile: dbPath + "/ca-test.pem"
+});
+
+// Rotate in new certificates
+copyCertificateFile("jstests/libs/trusted-ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/trusted-server.pem", dbPath + "/server-test.pem");
+
+assert.commandWorked(mongod.adminCommand({rotateCertificates: 1}));
+// make sure that mongo is still connected after rotation
+assert.commandWorked(mongod.adminCommand({connectionStatus: 1}));
+
+const host = "localhost:" + mongod.port;
+
+// Start shell with old certificates and make sure it can't connect
+let out = runMongoProgram("mongo",
+                          "--host",
+                          host,
+                          "--ssl",
+                          "--sslPEMKeyFile",
+                          "jstests/libs/client.pem",
+                          "--sslCAFile",
+                          "jstests/libs/ca.pem",
+                          "--eval",
+                          ";");
+assert.neq(out, 0, "Mongo invocation did not fail");
+
+// Start shell with new certificates and make sure it can connect
+out = runMongoProgram("mongo",
+                      "--host",
+                      host,
+                      "--ssl",
+                      "--sslPEMKeyFile",
+                      "jstests/libs/trusted-client.pem",
+                      "--sslCAFile",
+                      "jstests/libs/trusted-ca.pem",
+                      "--eval",
+                      ";");
+assert.eq(out, 0, "Mongo invocation failed");
+
+MongoRunner.stopMongod(mongod);
+}());
\ No newline at end of file
diff --git a/jstests/ssl/x509_rotate_missing_files.js b/jstests/ssl/x509_rotate_missing_files.js
new file mode 100644
index 00000000000..68fea8fa683
--- /dev/null
+++ b/jstests/ssl/x509_rotate_missing_files.js
@@ -0,0 +1,56 @@
+// Check that rotation will fail if a certificate file is missing
+
+(function() {
+"use strict";
+
+load('jstests/ssl/libs/ssl_helpers.js');
+
+if (determineSSLProvider() === "openssl") {
+    return;
+}
+
+function deleteFile(file) {
+    if (_isWindows()) {
+        // correctly replace forward slashes for Windows
+        file = file.replace(/\//g, "\\");
+        assert.eq(0, runProgram("cmd.exe", "/c", "del", file));
+        return;
+    }
+    assert.eq(0, runProgram("rm", file));
+}
+
+const dbPath = MongoRunner.toRealDir("$dataDir/cluster_x509_rotate_test/");
+mkdir(dbPath);
+
+copyCertificateFile("jstests/libs/ca.pem", dbPath + "/ca-test.pem");
+copyCertificateFile("jstests/libs/client.pem", dbPath + "/client-test.pem");
+copyCertificateFile("jstests/libs/server.pem", dbPath + "/server-test.pem");
+copyCertificateFile("jstests/libs/crl.pem", dbPath + "/crl-test.pem");
+
+const mongod = MongoRunner.runMongod({
+    sslMode: "requireSSL",
+    sslPEMKeyFile: dbPath + "/server-test.pem",
+    sslCAFile: dbPath + "/ca-test.pem",
+    sslClusterFile: dbPath + "/client-test.pem",
+    sslCRLFile: dbPath + "/crl-test.pem",
+});
+
+// if we are on apple, don't do delete test on CRL -- it will succeed.
+let certTypes = ["server", "ca", "client"];
+if (determineSSLProvider() !== "apple") {
+    certTypes.push("crl");
+}
+
+for (let certType of certTypes) {
+    copyCertificateFile("jstests/libs/ca.pem", dbPath + "/ca-test.pem");
+    copyCertificateFile("jstests/libs/client.pem", dbPath + "/client-test.pem");
+    copyCertificateFile("jstests/libs/server.pem", dbPath + "/server-test.pem");
+    copyCertificateFile("jstests/libs/crl.pem", dbPath + "/crl-test.pem");
+    assert.commandWorked(mongod.adminCommand({rotateCertificates: 1}));
+
+    deleteFile(`${dbPath}/${certType}-test.pem`);
+    assert.commandFailed(mongod.adminCommand({rotateCertificates: 1}));
+}
+
+MongoRunner.stopMongod(mongod);
+})();
\ No newline at end of file