SERVER-43773 Start up config server and shard replica sets in parallel in ShardingTest

2 files changed, 116 insertions, 69 deletions

diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index b609be89841..cb9b57d7224 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -572,8 +572,32 @@ var ReplSetTest = function(opts) {
      * @param options - The options passed to {@link MongoRunner.runMongod}
      */
     this.startSet = function(options, restart) {
-        print("ReplSetTest starting set");
-        let startTime = new Date();  // Measure the execution time of this function.
+        // If the caller has explicitly specified 'waitForConnect:false', then we will start up all
+        // replica set nodes and return without waiting to connect to any of them.
+        const skipWaitingForAllConnections = (options && options.waitForConnect === false);
+
+        // Start up without waiting for connections.
+        this.startSetAsync(options, restart);
+
+        // Avoid waiting for connections to each node.
+        if (skipWaitingForAllConnections) {
+            print("ReplSetTest startSet skipping waiting for connections to all nodes in set '" +
+                  this.name + "'");
+            return this.nodes;
+        }
+
+        return this.startSetAwait();
+    };
+
+    /**
+     * Starts each node in the replica set with the given options without waiting for a connection
+     * to any node. Call 'startSetAwait' subsequently to wait for startup of each node to complete.
+     *
+     * @param options - The options passed to {@link MongoRunner.runMongod}
+     */
+    this.startSetAsync = function(options, restart) {
+        print("ReplSetTest starting set '" + this.name + "'");
+        self.startSetStartTime = new Date();  // Measure the execution time of node startup.
 
         if (options && options.keyFile) {
             self.keyFile = options.keyFile;
@@ -591,10 +615,6 @@ var ReplSetTest = function(opts) {
             Random.setRandomSeed(jsTest.options().seed);
         }
 
-        // If the caller has explicitly specified 'waitForConnect:false', then we will start up all
-        // replica set nodes and return without waiting to connect to any of them.
-        const skipWaitingForAllConnections = (options && options.waitForConnect === false);
-
         // If the caller has explicitly set 'waitForConnect', then we prefer that. Otherwise we
         // default to not waiting for a connection. We merge the options object with a new field so
         // as to not modify the original options object that was passed in.
@@ -608,13 +628,14 @@ var ReplSetTest = function(opts) {
         for (let n = 0; n < this.ports.length; n++) {
             this.start(n, options, restart);
         }
+        return this.nodes;
+    };
 
-        // Avoid waiting for connections to each node.
-        if (skipWaitingForAllConnections) {
-            print("ReplSetTest startSet skipping waiting for connections to all nodes.");
-            return this.nodes;
-        }
-
+    /**
+     * Waits for startup of each replica set node to complete by waiting until a connection can be
+     * made to each.
+     */
+    this.startSetAwait = function() {
         // Wait until we can establish a connection to each node before proceeding.
         for (let n = 0; n < this.ports.length; n++) {
             this._waitForInitialConnection(n);
@@ -622,7 +643,7 @@ var ReplSetTest = function(opts) {
 
         print("ReplSetTest startSet, nodes: " + tojson(this.nodes));
 
-        print("ReplSetTest startSet took " + (new Date() - startTime) + "ms for " +
+        print("ReplSetTest startSet took " + (new Date() - self.startSetStartTime) + "ms for " +
               this.nodes.length + " nodes.");
         return this.nodes;
     };
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index e0fb313c1a0..bc73c48927b 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -1226,10 +1226,12 @@ var ShardingTest = function(params) {
     // Should we start up shards as replica sets.
     const shardsAsReplSets = (otherParams.rs || otherParams["rs" + i] || startShardsAsRS);
 
-    // Start the MongoD servers (shards)
+    //
+    // Start each shard replica set or standalone mongod.
+    //
     let startTime = new Date();  // Measure the execution time of startup and initiate.
     for (var i = 0; i < numShards; i++) {
-        if (otherParams.rs || otherParams["rs" + i] || startShardsAsRS) {
+        if (shardsAsReplSets) {
             var setName = testName + "-rs" + i;
 
             var rsDefaults = {
@@ -1309,24 +1311,11 @@ var ShardingTest = function(params) {
             });
 
             print("ShardingTest starting replica set for shard: " + setName);
-            this._rs[i] =
-                {setName: setName, test: rs, nodes: rs.startSet(rsDefaults), url: rs.getURL()};
-
-            // ReplSetTest.initiate() requires all nodes to be to be authorized to run
-            // replSetGetStatus.
-            // TODO(SERVER-14017): Remove this in favor of using initiate() everywhere.
-            print("ShardingTest initiating replica set for shard: " + setName);
-            rs.initiateWithAnyNodeAsPrimary();
-
-            this["rs" + i] = rs;
-            this._rsObjects[i] = rs;
 
-            _alldbpaths.push(null);
-            this._connections.push(null);
-
-            if (otherParams.useBridge) {
-                unbridgedConnections.push(null);
-            }
+            // Start up the replica set but don't wait for it to complete. This allows the startup
+            // of each shard to proceed in parallel.
+            this._rs[i] =
+                {setName: setName, test: rs, nodes: rs.startSetAsync(rsDefaults), url: rs.getURL()};
         } else {
             var options = {
                 useHostname: otherParams.useHostname,
@@ -1400,34 +1389,9 @@ var ShardingTest = function(params) {
         }
     }
 
-    // Do replication on replica sets if required
-    for (var i = 0; i < numShards; i++) {
-        if (!otherParams.rs && !otherParams["rs" + i] && !startShardsAsRS) {
-            continue;
-        }
-
-        var rs = this._rs[i].test;
-        rs.getPrimary().getDB("admin").foo.save({x: 1});
-
-        if (keyFile) {
-            authutil.asCluster(rs.nodes, keyFile, function() {
-                rs.awaitReplication();
-            });
-        }
-
-        rs.awaitSecondaryNodes();
-
-        var rsConn = new Mongo(rs.getURL());
-        rsConn.name = rs.getURL();
-
-        this._connections[i] = rsConn;
-        this["shard" + i] = rsConn;
-        rsConn.rs = rs;
-    }
-
-    print("ShardingTest startup and initiation for all shards took " + (new Date() - startTime) +
-          "ms for " + numShards + " shards.");
-
+    //
+    // Start up the config server replica set.
+    //
     this._configServers = [];
 
     // Using replica set for config servers
@@ -1467,17 +1431,83 @@ var ShardingTest = function(params) {
 
     rstOptions.nodes = nodeOptions;
 
-    const configServerStartTime =
-        new Date();  // Measure the execution time of config server startup and initiate.
-
-    // Start the config server's replica set
+    // Start the config server's replica set without waiting for it to complete. This allows it
+    // to proceed in parallel with the startup of each shard.
     this.configRS = new ReplSetTest(rstOptions);
-    this.configRS.startSet(startOptions);
+    this.configRS.startSetAsync(startOptions);
+
+    //
+    // Wait for each shard replica set to finish starting up.
+    //
+    if (shardsAsReplSets) {
+        for (let i = 0; i < numShards; i++) {
+            print("Waiting for shard " + this._rs[i].setName + " to finish starting up.");
+            this._rs[i].test.startSetAwait();
+        }
+    }
 
+    //
+    // Wait for the config server to finish starting up.
+    //
+    print("Waiting for the config server to finish starting up.");
+    this.configRS.startSetAwait();
     var config = this.configRS.getReplSetConfig();
     config.configsvr = true;
     config.settings = config.settings || {};
 
+    print("ShardingTest startup for all nodes took " + (new Date() - startTime) + "ms with " +
+          this.configRS.nodeList().length + " config server nodes and " +
+          totalNumShardNodes(shardsAsReplSets) + " total shard nodes.");
+
+    //
+    // Initiate each shard replica set.
+    //
+    if (shardsAsReplSets) {
+        for (var i = 0; i < numShards; i++) {
+            print("ShardingTest initiating replica set for shard: " + this._rs[i].setName);
+
+            // ReplSetTest.initiate() requires all nodes to be to be authorized to run
+            // replSetGetStatus.
+            // TODO(SERVER-14017): Remove this in favor of using initiate() everywhere.
+            this._rs[i].test.initiateWithAnyNodeAsPrimary();
+
+            this["rs" + i] = this._rs[i].test;
+            this._rsObjects[i] = this._rs[i].test;
+
+            _alldbpaths.push(null);
+            this._connections.push(null);
+
+            if (otherParams.useBridge) {
+                unbridgedConnections.push(null);
+            }
+        }
+    }
+
+    // Do replication on replica sets if required
+    for (var i = 0; i < numShards; i++) {
+        if (!shardsAsReplSets) {
+            continue;
+        }
+
+        var rs = this._rs[i].test;
+        rs.getPrimary().getDB("admin").foo.save({x: 1});
+
+        if (keyFile) {
+            authutil.asCluster(rs.nodes, keyFile, function() {
+                rs.awaitReplication();
+            });
+        }
+
+        rs.awaitSecondaryNodes();
+
+        var rsConn = new Mongo(rs.getURL());
+        rsConn.name = rs.getURL();
+
+        this._connections[i] = rsConn;
+        this["shard" + i] = rsConn;
+        rsConn.rs = rs;
+    }
+
     // ReplSetTest.initiate() requires all nodes to be to be authorized to run replSetGetStatus.
     // TODO(SERVER-14017): Remove this in favor of using initiate() everywhere.
     this.configRS.initiateWithAnyNodeAsPrimary(config);
@@ -1485,10 +1515,6 @@ var ShardingTest = function(params) {
     // Wait for master to be elected before starting mongos
     var csrsPrimary = this.configRS.getPrimary();
 
-    print("ShardingTest startup and initiation for the config server took " +
-          (new Date() - configServerStartTime) + "ms with " + this.configRS.nodeList().length +
-          " nodes.");
-
     print("ShardingTest startup and initiation for all nodes took " + (new Date() - startTime) +
           "ms with " + this.configRS.nodeList().length + " config server nodes and " +
           totalNumShardNodes(shardsAsReplSets) + " total shard nodes.");