SERVER-54136 Make the authenticate command respect enforceUserClusterSeparation

13 files changed, 466 insertions, 104 deletions

diff --git a/jstests/libs/server_SAN.pem b/jstests/libs/server_SAN.pem
new file mode 100644
index 00000000000..1a72b2bb229
--- /dev/null
+++ b/jstests/libs/server_SAN.pem
@@ -0,0 +1,54 @@
+# Autogenerated file, do not edit.
+# Generate using jstests/ssl/x509/mkcert.py --config jstests/ssl/x509/certs.yml server_SAN.pem
+#
+# General purpose server certificate with good SANs.
+-----BEGIN CERTIFICATE-----
+MIIDmjCCAoICBATC72IwDQYJKoZIhvcNAQELBQAwdDELMAkGA1UEBhMCVVMxETAP
+BgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MRAwDgYDVQQK
+DAdNb25nb0RCMQ8wDQYDVQQLDAZLZXJuZWwxFzAVBgNVBAMMDktlcm5lbCBUZXN0
+IENBMB4XDTE5MDkyNTIzMjc0MVoXDTM5MDkyNzIzMjc0MVowfTELMAkGA1UEBhMC
+VVMxETAPBgNVBAgMCE5ldyBZb3JrMRYwFAYDVQQHDA1OZXcgWW9yayBDaXR5MRAw
+DgYDVQQKDAdNb25nb0RCMQ8wDQYDVQQLDAZLZXJuZWwxIDAeBgNVBAMMF0tlcm5l
+bCBDbGllbnQgUGVlciBSb2xlMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAxwFNtDYxGxUePcHurZkya1xSEK/N+wk/ugHouvLEUv3tA/igQ4NIVs3b+q/e
+RsWLQxzkJZh0H78rDhvIv4HtViFmc79Zwekr36Pv/3p0RxwVmPZnKi1FnuX6PrWb
+r0g0ANlKLup5AjAM23RBFufMSV9lVIOFf6w1axQX95AKOnUQvRtDFGPazES5KJ7X
+fbm907rdo5RSlbQUAOrn24n2So4QZrBppk2U5NE/sfL6cgQUR9kaelkDjj27jp2u
+N/NdXAcAx1yXXGP66ShEfodSbEWJ2aaQYvKUwEju2xoGRJVV0eTSKeH7g2km3+Vd
+ng7J1SwsQcKw8IGTma4HIhMvbQIDAQABozAwLjAsBgNVHREEJTAjgglsb2NhbGhv
+c3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQADggEBAHFB
+8fiC5UljqekYY39U09NAMtrw/frphewMxvzAdkMvwaOwVWhcxoFM1w67QAT+VvjO
+Av/5aOREl9IxdLEALKTG6lATbPcc9chPlCyUPqJFELVybmMLaHXZd5iLpp32ylr8
+oldpFKDYYRhgz94RPYH6dpZDfToyD33YCnP46sWt3nqtQdyESyW3ccXYWINxGEWJ
+EBaOB7mZYXTjRJJW86EhhCG2CmRT+L7PSi0yXYiLRdNLHkMguNxArlklp+6ha0Kg
+uTl/o5ZGgo+PphK+vr1vh+YgDJb1neVOyXx8gDLa/jjJOdDHnwtIxiHMgAy7qpKU
++iw8BrYQWUC7PUddwfM=
+-----END CERTIFICATE-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDHAU20NjEbFR49
+we6tmTJrXFIQr837CT+6Aei68sRS/e0D+KBDg0hWzdv6r95GxYtDHOQlmHQfvysO
+G8i/ge1WIWZzv1nB6Svfo+//enRHHBWY9mcqLUWe5fo+tZuvSDQA2Uou6nkCMAzb
+dEEW58xJX2VUg4V/rDVrFBf3kAo6dRC9G0MUY9rMRLkontd9ub3Tut2jlFKVtBQA
+6ufbifZKjhBmsGmmTZTk0T+x8vpyBBRH2Rp6WQOOPbuOna43811cBwDHXJdcY/rp
+KER+h1JsRYnZppBi8pTASO7bGgZElVXR5NIp4fuDaSbf5V2eDsnVLCxBwrDwgZOZ
+rgciEy9tAgMBAAECggEBALLpkRp7vhpDGjUbraGVIac0EPO/msgewJRSMxIgQDD8
+6aGwXSVVWivwV91BT+0AnPtxdgv2zC/ehq7ge9Lw1Jei36EgX2neX/toP9TIGtHz
+hddHb+iTWwxIZnsKezVTDK3WrMMROkmlOWnzUrFtJFSy1z7sH5FxGXVzUzjW3ts9
+3JPMYowjn+4OFBEVG7/BCKn2+k5vjeI0+NL2gKJ4GTDtrOp6vGhyClyL+W+bZUta
+AHAkljYuwIlwxm3ACfrOBAgzW98uZBDSiaBQ6RyLBTlWFY+h53A2nlODuGTg6r7k
+JjTeY/XqxcTHz1GOGmn2obaCvrPIiLkrrBK5VQRC6GECgYEA9xTJds9D6WS6WdBB
+VmkPjeUuCj/xlNOW0Pr+9Q//4nTPC6bUoqmDaN4hIiUCeFLgBOH23MZ/ZHKihjvq
+xcEzhrvZfH0oHhnAMGIumLj+QwHNStRG1zJq4Auyc+Crr/TOW0ePDTHkRYSR4ZQu
+wdm9m2ioEeqT7Jbn/uclkkpEnIMCgYEAzjBB/WHX+WRIJbfpoandR/jXHzW2A4MM
+kHdpp/RpqoulGQNlkRjhufQHk+yTQ2v+/ZeROZbFHoiR7Z8j+vF7wCT1tfvz4Jor
+RWNNMM4sJEmA+biyqZLWRH5+cedOz/gM5EFkBFm0P8g11QStWGdVDvHVCLB7WzJW
+ggntrP0+IU8CgYBpwVD19IL52+5TUOsWZEOPFYAeTP5HGEWeT0BffiilMiAIubCb
+qvifIyyFY9fpzncqP9mc561RPivITs8PM8GtMqxFN4C7WYREcVHGdrAF2DiZhcCI
+jYDnQrpwFMMHvj2xHjDhkfBFvRd/uSBGQr+ATExb1s7X3kOQfDk9o2I4pwKBgDdw
+KD4NkblfajqiAXJTLskMMiMVGPRGxxkM0l6zS7nsRB6+mA90RjTuVR9Dy7CkF/Jt
+T0HRtPxpsfrXACCXzzMxfNNKhB8OtLW6ZmXJ+XpkPQZnz7niXjQaYrsTe1BVnHm+
+zzqQN/OyTC8TS9xqJCeZhDMK6I/zr+jU7I/Ho23nAoGBAIGbEzmTADdbnfSJdcNU
+mhIOIjPU/qrf2lHpjHhyAaEPPLVnCtIbdzq5WiIt+UKMkcOMDbJanH1HbGEEL3eR
+u5xaEIEhOPplh9Dr46RetPcr8qEFxV8eX5H+RKCaBv1V+qnXg5h7hUQy0yKPREO7
+HE6kiNLIRtN9XY0PRxoeM+Zq
+-----END PRIVATE KEY-----
diff --git a/jstests/ssl/x509_client.js b/jstests/ssl/x509_client.js
index ed458937425..1b53f9a6405 100644
--- a/jstests/ssl/x509_client.js
+++ b/jstests/ssl/x509_client.js
@@ -16,15 +16,13 @@ conn.getDB('admin').dropAllUsers();
 conn.getDB('admin').logout();
 MongoRunner.stopMongod(conn);
 
-var SERVER_CERT = "jstests/libs/server.pem";
-var CA_CERT = "jstests/libs/ca.pem";
+const SERVER_CERT = "jstests/libs/server.pem";
+const CA_CERT = "jstests/libs/ca.pem";
 
-var SERVER_USER = 'CN=server,OU=Kernel,O=MongoDB,L=New York City,ST=New York,C=US';
-var INTERNAL_USER = 'CN=internal,OU=Kernel,O=MongoDB,L=New York City,ST=New York,C=US';
-var CLIENT_USER = 'CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US';
-var INVALID_CLIENT_USER = 'CN=invalid,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US';
+const CLIENT_USER = "CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US";
+const INVALID_CLIENT_USER = "C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=invalid";
 
-function authAndTest(mongo, clusterUserSeparationOveride = false) {
+function authAndTest(mongo) {
     external = mongo.getDB("$external");
     test = mongo.getDB("test");
 
@@ -62,21 +60,23 @@ function authAndTest(mongo, clusterUserSeparationOveride = false) {
 
     assert(log.some((line) => successRegex.test(line)));
 
-    const overrideDependentTester =
-        clusterUserSeparationOveride ? assert.doesNotThrow : assert.throws;
-    // It should be impossible to create users with the same name as the server's subject,
-    // unless guardrails are explicitly overridden
-    overrideDependentTester(function() {
+    let createServerUser = function() {
+        // It should be impossible to create users with the same name as the server's subject,
+        // unless guardrails are explicitly overridden
         external.createUser(
             {user: SERVER_USER, roles: [{'role': 'userAdminAnyDatabase', 'db': 'admin'}]});
-    }, [], "Created user with same name as the server's x.509 subject");
+    };
+    assert.throws(
+        createServerUser, [], "Created user with same name as the server's x.509 subject");
 
-    // It should be impossible to create users with names recognized as cluster members,
-    // unless guardrails are explicitly overridden
-    overrideDependentTester(function() {
+    let createInternalUser = function() {
+        // It should be impossible to create users with names recognized as cluster members,
+        // unless guardrails are explicitly overridden
         external.createUser(
             {user: INTERNAL_USER, roles: [{'role': 'userAdminAnyDatabase', 'db': 'admin'}]});
-    }, [], "Created user which would be recognized as a cluster member");
+    };
+    assert.throws(
+        createInternalUser, [], "Created user which would be recognized as a cluster member");
 
     // Check that we can add a user and read data
     test.createUser(
@@ -102,14 +102,6 @@ const x509_options = {
     authAndTest(mongo);
     MongoRunner.stopMongod(mongo);
 }
-{
-    print("1.5. Testing x.509 auth to mongod with cluster/user separation disabled");
-    const mongo = MongoRunner.runMongod(Object.merge(
-        x509_options, {auth: "", setParameter: {enforceUserClusterSeparation: false}}));
-
-    authAndTest(mongo, true);
-    MongoRunner.stopMongod(mongo);
-}
 
 {
     print("2. Testing x.509 auth to mongos");
@@ -128,21 +120,3 @@ const x509_options = {
     authAndTest(new Mongo("localhost:" + st.s0.port));
     st.stop();
 }
-{
-    print("2.5 Testing x.509 auth to mongos with cluster/user separation disabled");
-    var st = new ShardingTest({
-        shards: 1,
-        mongos: 1,
-        other: {
-            keyFile: 'jstests/libs/key1',
-            configOptions:
-                Object.merge(x509_options, {setParameter: {enforceUserClusterSeparation: false}}),
-            mongosOptions: x509_options,
-            shardOptions: x509_options,
-            useHostname: false
-        }
-    });
-
-    authAndTest(new Mongo("localhost:" + st.s0.port), true);
-    st.stop();
-}
diff --git a/jstests/ssl/x509_enforce_user_cluster_separation.js b/jstests/ssl/x509_enforce_user_cluster_separation.js
new file mode 100644
index 00000000000..3f98b6e787b
--- /dev/null
+++ b/jstests/ssl/x509_enforce_user_cluster_separation.js
@@ -0,0 +1,205 @@
+// Check if this build supports the authenticationMechanisms startup parameter.
+
+const SERVER_CERT = "jstests/libs/server.pem";
+const SERVER_SAN_CERT = "jstests/libs/server_SAN.pem";
+const CLIENT_CERT = "jstests/libs/client.pem";
+const CA_CERT = "jstests/libs/ca.pem";
+
+const SERVER_USER = "CN=server,OU=Kernel,O=MongoDB,L=New York City,ST=New York,C=US";
+const SERVER_SAN_USER =
+    "CN=Kernel Client Peer Role,OU=Kernel,O=MongoDB,L=New York City,ST=New York,C=US";
+const CLIENT_USER = "CN=client,OU=KernelUser,O=MongoDB,L=New York City,ST=New York,C=US";
+
+function authAndTest(cert, user) {
+    const INVALID_USER = "C=US,ST=New York,L=New York City,O=MongoDB,OU=KernelUser,CN=invalid";
+
+    let external = db.getSiblingDB("$external");
+    let test = db.getSiblingDB("test");
+
+    assert(!external.auth({user: INVALID_USER, mechanism: 'MONGODB-X509'}),
+           "authentication with invalid user should fail");
+    assert(external.auth({user: user, mechanism: 'MONGODB-X509'}),
+           "authentication with valid user failed");
+    assert(external.auth({mechanism: 'MONGODB-X509'}),
+           "authentication with valid cert and no user field failed");
+    assert(external.runCommand({authenticate: 1, mechanism: 'MONGODB-X509', user: user}).ok,
+           "runCommand authentication with valid cert and user field failed");
+    assert(external.runCommand({authenticate: 1, mechanism: 'MONGODB-X509'}).ok,
+           "runCommand authentication with valid cert and no user field failed");
+    // Smoke our current user with a find.
+    test.foo.findOne();
+
+    // Check that we can add a user and read data.
+    test.createUser(
+        {user: "test", pwd: "test", roles: [{'role': 'readWriteAnyDatabase', 'db': 'admin'}]});
+    test.foo.findOne();
+
+    // Reads are not allowed after logout.
+    external.logout();
+    assert.throws(function() {
+        test.foo.findOne();
+    }, [], "read after logout");
+}
+
+function runSubShell(conn, cert, user, func) {
+    const args = [
+        'mongo',
+        '--ssl',
+        `--sslCAFile=${CA_CERT}`,
+        `--sslPEMKeyFile=${cert}`,
+        '--sslAllowInvalidHostnames',
+        '--authenticationDatabase=$external',
+        '--authenticationMechanism=MONGODB-X509',
+        `mongodb://${conn.host}`,
+        '--eval',
+        `(${func.toString()})('${cert}', '${user}');`
+    ];
+    const ret = _runMongoProgram(...args);
+    assert(ret == ErrorCodes.OK, 'subshell did not succeed');
+}
+
+function initUser(conn, user) {
+    const external = conn.getDB("$external");
+    external.createUser({
+        user: user,
+        roles: [
+            {'role': 'userAdminAnyDatabase', 'db': 'admin'},
+            {'role': 'readWriteAnyDatabase', 'db': 'admin'},
+            {'role': 'clusterMonitor', 'db': 'admin'},
+        ]
+    });
+
+    // Localhost exception should not be in place anymore
+    const test = conn.getDB("test");
+    assert.throws(function() {
+        test.foo.findOne();
+    }, [], "read without login");
+}
+
+const x509_options = {
+    sslMode: "requireSSL",
+    sslPEMKeyFile: SERVER_CERT,
+    sslCAFile: CA_CERT
+};
+
+const mongodOptions =
+    Object.merge(x509_options, {auth: "", setParameter: {enforceUserClusterSeparation: false}});
+
+const mongosOptions =
+    Object.merge(x509_options, {setParameter: {enforceUserClusterSeparation: false}});
+
+function runMongodTest(desc, func) {
+    print(desc);
+    const mongo = MongoRunner.runMongod(mongodOptions);
+    func(mongo);
+
+    MongoRunner.stopMongod(mongo);
+}
+
+function runMongodFailTest(desc, options) {
+    print(desc);
+    const mongo = MongoRunner.runMongod(Object.merge(mongodOptions, options));
+    assert(!mongo, "MongoD started successfully with bad options");
+}
+
+function runMongosTest(desc, func) {
+    print(desc);
+    const st = new ShardingTest({
+        shards: 1,
+        mongos: 1,
+        other: {
+            keyFile: 'jstests/libs/key1',
+            configOptions: mongodOptions,
+            mongosOptions: mongosOptions,
+            shardOptions: x509_options,
+            useHostname: false
+        }
+    });
+
+    const mongo = new Mongo(`localhost:${st.s0.port}`);
+    func(mongo);
+    st.stop();
+}
+
+function runMongosFailTest(desc, options) {
+    print(desc);
+    // We start the ShardingTest cleanly first because it throws and fails to clean up after itself.
+    const st = new ShardingTest({
+        config: 1,
+        shards: 1,
+        mongos: 1,
+        other: {
+            keyFile: 'jstests/libs/key1',
+            configOptions: mongodOptions,
+            mongosOptions: mongosOptions,
+            shardOptions: x509_options,
+            useHostname: false
+        }
+    });
+
+    const failOptions = Object.merge(mongosOptions, options);
+    print(`Fail options: ${tojson(failOptions)}`);
+
+    assert.throws(function() {
+        // Start a new mongos with bad options.
+        st.restartMongos(0, failOptions);
+    }, [], "MongoS restarted successfully with bad options");
+
+    // Avoid st.stop() because it will throw when it attempts to stop the second mongos.
+    st.stopAllShards();
+    st.stopAllConfigServers();
+}
+
+runMongodTest("1a. Testing x.509 auth to mongod with a client user/cert", function(conn) {
+    initUser(conn, CLIENT_USER);
+
+    runSubShell(conn, CLIENT_CERT, CLIENT_USER, authAndTest);
+});
+
+runMongodTest("1b. Testing x.509 auth to mongod with the server user/cert", function(conn) {
+    initUser(conn, SERVER_USER);
+
+    runSubShell(conn, SERVER_CERT, SERVER_USER, authAndTest);
+});
+
+runMongodTest("1c. Testing x.509 auth to mongod with a cluster user/cert", function(conn) {
+    initUser(conn, SERVER_SAN_USER);
+
+    runSubShell(conn, SERVER_SAN_CERT, SERVER_SAN_USER, authAndTest);
+});
+
+runMongodFailTest('1d. Testing x.509 cluster auth on mongod with "x509" option',
+                  {clusterAuthMode: "x509"});
+
+runMongodFailTest('1e. Testing x.509 cluster auth on mongod with "sendX509" option',
+                  {clusterAuthMode: "sendX509"});
+
+runMongodFailTest('1e. Testing x.509 cluster auth on mongod with "sendKeyFile" option',
+                  {clusterAuthMode: "sendKeyFile"});
+
+runMongosTest("2a. Testing x.509 auth to mongos with a client user/cert", function(conn) {
+    initUser(conn, CLIENT_USER);
+
+    runSubShell(conn, CLIENT_CERT, CLIENT_USER, authAndTest);
+});
+
+runMongosTest("2b. Testing x.509 auth to mongos with the server user/cert", function(conn) {
+    initUser(conn, SERVER_USER);
+
+    runSubShell(conn, SERVER_CERT, SERVER_USER, authAndTest);
+});
+
+runMongosTest("2c. Testing x.509 auth to mongos with a cluster user/cert", function(conn) {
+    initUser(conn, SERVER_SAN_USER);
+
+    runSubShell(conn, SERVER_SAN_CERT, SERVER_SAN_USER, authAndTest);
+});
+
+runMongosFailTest('2d. Testing x.509 cluster auth on mongos with "x509" option',
+                  {restart: true, clusterAuthMode: "x509"});
+
+runMongosFailTest('2e. Testing x.509 cluster auth on mongos with "sendX509" option',
+                  {clusterAuthMode: "sendX509"});
+
+runMongosFailTest('2f. Testing x.509 cluster auth on mongos with "sendKeyFile" option',
+                  {clusterAuthMode: "sendKeyFile"});
diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript
index 1c35425adb4..7faae0fa7c1 100644
--- a/src/mongo/db/auth/SConscript
+++ b/src/mongo/db/auth/SConscript
@@ -131,6 +131,17 @@ env.Library(
 )
 
 env.Library(
+    target='auth_options',
+    source=[
+        'auth_options.cpp',
+    ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/server_parameters',
+    ],
+)
+
+env.Library(
     target='role_graph_update',
     source=[
         'role_graph_update.cpp',
@@ -154,6 +165,7 @@ env.Library(
     LIBDEPS=[
         'address_restriction',
         'auth',
+        'auth_options',
         'auth_rolename',
         'authentication_restriction',
         'authorization_manager_global',
diff --git a/src/mongo/db/auth/auth_options.cpp b/src/mongo/db/auth/auth_options.cpp
new file mode 100644
index 00000000000..3894a97e560
--- /dev/null
+++ b/src/mongo/db/auth/auth_options.cpp
@@ -0,0 +1,47 @@
+
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kAccessControl
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/auth/auth_options.h"
+
+#include "mongo/db/server_parameters.h"
+
+namespace mongo {
+
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(enforceUserClusterSeparation, bool, true);
+
+bool shouldEnforceUserClusterSeparation() {
+    return enforceUserClusterSeparation;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/auth/auth_options.h b/src/mongo/db/auth/auth_options.h
new file mode 100644
index 00000000000..671d900f7fb
--- /dev/null
+++ b/src/mongo/db/auth/auth_options.h
@@ -0,0 +1,35 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+namespace mongo {
+
+// Return true if we should reject users that are ambiguously part of the cluster.
+bool shouldEnforceUserClusterSeparation();
+
+}  // namespace mongo
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index f8b88e55015..ad6d77e7920 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -144,6 +144,7 @@ env.Library(
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/client/clientdriver_minimal',
         '$BUILD_DIR/mongo/db/audit',
+        '$BUILD_DIR/mongo/db/auth/auth_options',
         '$BUILD_DIR/mongo/db/auth/sasl_options',
         '$BUILD_DIR/mongo/db/auth/user_document_parser',
         '$BUILD_DIR/mongo/db/commands',
diff --git a/src/mongo/db/commands/authentication_commands.cpp b/src/mongo/db/commands/authentication_commands.cpp
index fbc7ed593d9..b06bff81631 100644
--- a/src/mongo/db/commands/authentication_commands.cpp
+++ b/src/mongo/db/commands/authentication_commands.cpp
@@ -43,6 +43,7 @@
 #include "mongo/client/sasl_client_authenticate.h"
 #include "mongo/config.h"
 #include "mongo/db/audit.h"
+#include "mongo/db/auth/auth_options.h"
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/auth/privilege.h"
 #include "mongo/db/auth/sasl_options.h"
@@ -67,56 +68,81 @@ namespace {
 static bool _isX509AuthDisabled;
 static constexpr auto kX509AuthenticationDisabledMessage = "x.509 authentication is disabled."_sd;
 
+constexpr auto kExternalDB = "$external"_sd;
+
 #ifdef MONGO_CONFIG_SSL
 Status _authenticateX509(OperationContext* opCtx, const UserName& user, const BSONObj& cmdObj) {
     if (!getSSLManager()) {
         return Status(ErrorCodes::ProtocolError,
                       "SSL support is required for the MONGODB-X509 mechanism.");
     }
-    if (user.getDB() != "$external") {
-        return Status(ErrorCodes::ProtocolError,
-                      "X.509 authentication must always use the $external database.");
-    }
-
-    Client* client = Client::getCurrent();
-    AuthorizationSession* authorizationSession = AuthorizationSession::get(client);
-    auto clientName = SSLPeerInfo::forSession(client->session()).subjectName;
-    uassert(ErrorCodes::AuthenticationFailed,
-            "No verified subject name available from client",
-            !clientName.empty());
 
     if (!getSSLManager()->getSSLConfiguration().hasCA) {
         return Status(ErrorCodes::AuthenticationFailed,
                       "Unable to verify x.509 certificate, as no CA has been provided.");
-    } else if (user.getUser() != clientName.toString()) {
+    }
+
+    Client* client = opCtx->getClient();
+    auto clientName = SSLPeerInfo::forSession(client->session()).subjectName;
+    if (clientName.empty()) {
+        return Status(ErrorCodes::AuthenticationFailed,
+                      "No verified subject name available from client");
+    }
+
+    if (user.getDB() != kExternalDB) {
+        return Status(ErrorCodes::ProtocolError,
+                      "X.509 authentication must always use the $external database.");
+    }
+
+    if (user.getUser() != clientName.toString()) {
         return Status(ErrorCodes::AuthenticationFailed,
                       "There is no x.509 client certificate matching the user.");
-    } else {
+    }
+
+    AuthorizationSession* authorizationSession = AuthorizationSession::get(client);
+
+    auto isInternalClient = [&]() -> bool {
+        return opCtx->getClient()->session()->getTags() & transport::Session::kInternalClient;
+    };
+
+    auto authorizeExternalUser = [&]() -> Status {
+        if (_isX509AuthDisabled) {
+            return Status(ErrorCodes::BadValue, kX509AuthenticationDisabledMessage);
+        }
+        return authorizationSession->addAndAuthorizeUser(opCtx, user);
+    };
+
+    if (getSSLManager()->getSSLConfiguration().isClusterMember(clientName)) {
         // Handle internal cluster member auth, only applies to server-server connections
-        if (getSSLManager()->getSSLConfiguration().isClusterMember(clientName)) {
-            int clusterAuthMode = serverGlobalParams.clusterAuthMode.load();
-            if (clusterAuthMode == ServerGlobalParams::ClusterAuthMode_undefined ||
-                clusterAuthMode == ServerGlobalParams::ClusterAuthMode_keyFile) {
-                return Status(ErrorCodes::AuthenticationFailed,
-                              "The provided certificate "
-                              "can only be used for cluster authentication, not client "
-                              "authentication. The current configuration does not allow "
-                              "x.509 cluster authentication, check the --clusterAuthMode flag");
-            }
-            authorizationSession->grantInternalAuthorization(client);
+        switch (serverGlobalParams.clusterAuthMode.load()) {
+            case ServerGlobalParams::ClusterAuthMode_undefined:
+            case ServerGlobalParams::ClusterAuthMode_keyFile: {
+                uassert(ErrorCodes::AuthenticationFailed,
+                        "The provided certificate can only be used for cluster authentication, not "
+                        "client authentication. The current configuration does not allow x.509 "
+                        "cluster authentication, check the --clusterAuthMode flag",
+                        !shouldEnforceUserClusterSeparation());
+
+                return authorizeExternalUser();
+            } break;
+            case ServerGlobalParams::ClusterAuthMode_sendKeyFile:
+            case ServerGlobalParams::ClusterAuthMode_sendX509:
+            case ServerGlobalParams::ClusterAuthMode_x509: {
+                if (!isInternalClient()) {
+                    warning() << "Client isn't a mongod or mongos, but is connecting with a "
+                                 "certificate with cluster membership";
+                }
+
+                authorizationSession->grantInternalAuthorization(client);
+                return Status::OK();
+            } break;
         }
+    } else {
         // Handle normal client authentication, only applies to client-server connections
-        else {
-            if (_isX509AuthDisabled) {
-                return Status(ErrorCodes::BadValue, kX509AuthenticationDisabledMessage);
-            }
-            Status status = authorizationSession->addAndAuthorizeUser(opCtx, user);
-            if (!status.isOK()) {
-                return status;
-            }
-        }
-        return Status::OK();
+        return authorizeExternalUser();
     }
+
+    MONGO_UNREACHABLE;
 }
 #endif  // MONGO_CONFIG_SSL
 
diff --git a/src/mongo/db/commands/user_management_commands.cpp b/src/mongo/db/commands/user_management_commands.cpp
index 9293b6474e4..993fe6ef7dc 100644
--- a/src/mongo/db/commands/user_management_commands.cpp
+++ b/src/mongo/db/commands/user_management_commands.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/auth/action_set.h"
 #include "mongo/db/auth/action_type.h"
 #include "mongo/db/auth/address_restriction.h"
+#include "mongo/db/auth/auth_options.h"
 #include "mongo/db/auth/authorization_manager.h"
 #include "mongo/db/auth/authorization_manager_global.h"
 #include "mongo/db/auth/authorization_session.h"
@@ -92,8 +93,6 @@ using std::vector;
 
 namespace {
 
-MONGO_EXPORT_STARTUP_SERVER_PARAMETER(enforceUserClusterSeparation, bool, true);
-
 // Used to obtain mutex that guards modifications to persistent authorization data
 const auto getAuthzDataMutex = ServiceContext::declareDecoration<stdx::mutex>();
 
@@ -803,7 +802,7 @@ public:
 #ifdef MONGO_CONFIG_SSL
         if (getSSLManager() && dbname == "$external" &&
             getSSLManager()->getSSLConfiguration().isClusterMember(args.userName.getUser())) {
-            if (enforceUserClusterSeparation) {
+            if (shouldEnforceUserClusterSeparation()) {
                 uasserted(ErrorCodes::BadValue,
                           "Cannot create an x.509 user with a subjectname that would be "
                           "recognized as an internal cluster member");
diff --git a/src/mongo/db/commands/user_management_commands.idl b/src/mongo/db/commands/user_management_commands.idl
index 7df3ab4acda..1ae9c9658c5 100644
--- a/src/mongo/db/commands/user_management_commands.idl
+++ b/src/mongo/db/commands/user_management_commands.idl
@@ -32,12 +32,3 @@ imports:
     - "mongo/idl/basic_types.idl"
     - "mongo/db/auth/auth_types.idl"
     - "mongo/db/auth/address_restriction.idl"
-
-server_parameters:
-  enforceUserClusterSeparation:
-    description: "Prevents creation of users whose names would be interpreted as cluster members"
-    set_at: startup
-    cpp_varname: "gEnforceUserClusterSeparation"
-    cpp_vartype: bool
-    default: true
-
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 78472d843d5..dcb46c30df4 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -378,17 +378,18 @@ var ShardingTest = function(params) {
         }
     };
 
-    this.stopAllMongos = function(opts) {
-        for (var i = 0; i < this._mongos.length; i++) {
-            this.stopMongos(i, opts);
+    this.stopAllConfigServers = function(opts) {
+        if (this.configRS) {
+            this.configRS.stopSet(undefined, undefined, opts);
+        } else {
+            // Old style config triplet
+            for (var i = 0; i < this._configServers.length; i++) {
+                this.stopConfigServer(i, opts);
+            }
         }
     };
 
-    this.stop = function(opts) {
-        this.checkUUIDsConsistentAcrossCluster();
-
-        this.stopAllMongos(opts);
-
+    this.stopAllShards = function(opts) {
         for (var i = 0; i < this._connections.length; i++) {
             if (this._rs[i]) {
                 this._rs[i].test.stopSet(15, undefined, opts);
@@ -396,16 +397,25 @@ var ShardingTest = function(params) {
                 this.stopMongod(i, opts);
             }
         }
+    };
 
-        if (this.configRS) {
-            this.configRS.stopSet(undefined, undefined, opts);
-        } else {
-            // Old style config triplet
-            for (var i = 0; i < this._configServers.length; i++) {
-                this.stopConfigServer(i, opts);
-            }
+    this.stopAllMongos = function(opts) {
+        for (var i = 0; i < this._mongos.length; i++) {
+            this.stopMongos(i, opts);
         }
+    };
+
+    this.stop = function(opts) {
+        this.checkUUIDsConsistentAcrossCluster();
+
+        this.stopAllMongos(opts);
+
+        let startTime = new Date();  // Measure the execution time of shutting down shards.
+        this.stopAllShards(opts);
+        print("ShardingTest stopped all shards, took " + (new Date() - startTime) + "ms for " +
+              this._connections.length + " shards.");
 
+        this.stopAllConfigServers(opts);
         if (!opts || !opts.noCleanData) {
             print("ShardingTest stop deleting all dbpaths");
             for (var i = 0; i < _alldbpaths.length; i++) {
diff --git a/src/mongo/util/net/SConscript b/src/mongo/util/net/SConscript
index 5095c270e77..07de0db2487 100644
--- a/src/mongo/util/net/SConscript
+++ b/src/mongo/util/net/SConscript
@@ -35,6 +35,7 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/db/auth/auth_options',
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/server_options_core',
diff --git a/src/mongo/util/net/ssl_options.cpp b/src/mongo/util/net/ssl_options.cpp
index 29e2b7c510e..66d14a7fc15 100644
--- a/src/mongo/util/net/ssl_options.cpp
+++ b/src/mongo/util/net/ssl_options.cpp
@@ -39,6 +39,7 @@
 #include "mongo/base/init.h"
 #include "mongo/base/status.h"
 #include "mongo/config.h"
+#include "mongo/db/auth/auth_options.h"
 #include "mongo/db/server_options.h"
 #include "mongo/util/hex.h"
 #include "mongo/util/log.h"
@@ -605,6 +606,12 @@ Status storeSSLServerOptions(const moe::Environment& params) {
         if (sslGlobalParams.sslMode.load() == SSLParams::SSLMode_disabled) {
             return Status(ErrorCodes::BadValue, "need to enable SSL via the sslMode flag");
         }
+
+        if (!shouldEnforceUserClusterSeparation()) {
+            return Status(ErrorCodes::BadValue,
+                          "cannot have have x.509 cluster authentication while not enforcing user "
+                          "cluster separation");
+        }
     }
     if (sslGlobalParams.sslMode.load() == SSLParams::SSLMode_allowSSL) {
         // allowSSL and x509 is valid only when we are transitioning to auth.