SERVER-34448 Enable autosplitting on the shard and disable on mongos

SERVER-34449 Handle upgrade/downgrade logic for autosplitting on shard

SERVER-34448 Wait for chunk splitting in top chunk optimization tests

SERVER-34448 Require persistence in shard_existing_coll_chunk_count.js

7 files changed, 129 insertions, 14 deletions

diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 346b6dc6722..b9ff8d16d97 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -271,6 +271,7 @@ env.Library(
         'check_sharding_index_command.cpp',
         'cleanup_orphaned_cmd.cpp',
         'clone_catalog_data_command.cpp',
+        'clone_collection_options_from_primary_shard_cmd.cpp',
         'config/configsvr_add_shard_command.cpp',
         'config/configsvr_add_shard_to_zone_command.cpp',
         'config/configsvr_commit_chunk_migration_command.cpp',
@@ -302,11 +303,11 @@ env.Library(
         'set_shard_version_command.cpp',
         'sharding_server_status.cpp',
         'sharding_state_command.cpp',
+        'shardsvr_shard_collection.cpp',
         'split_chunk_command.cpp',
         'split_vector_command.cpp',
         'unset_sharding_command.cpp',
-        'shardsvr_shard_collection.cpp',
-        'clone_collection_options_from_primary_shard_cmd.cpp',
+        'wait_for_ongoing_chunk_splits_command.cpp',
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/bson/dotted_path_support',
diff --git a/src/mongo/db/s/chunk_splitter.cpp b/src/mongo/db/s/chunk_splitter.cpp
index 6ded7def07a..65dad51dbd4 100644
--- a/src/mongo/db/s/chunk_splitter.cpp
+++ b/src/mongo/db/s/chunk_splitter.cpp
@@ -38,6 +38,7 @@
 #include "mongo/db/dbdirectclient.h"
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/s/chunk_split_state_driver.h"
+#include "mongo/db/s/shard_filtering_metadata_refresh.h"
 #include "mongo/db/s/sharding_state.h"
 #include "mongo/db/s/split_chunk.h"
 #include "mongo/db/s/split_vector.h"
@@ -244,8 +245,7 @@ void ChunkSplitter::onStepUp() {
     }
     _isPrimary = true;
 
-    // log() << "The ChunkSplitter has started and will accept autosplit tasks.";
-    // TODO: Re-enable this log line when auto split is actively running on shards.
+    log() << "The ChunkSplitter has started and will accept autosplit tasks.";
 }
 
 void ChunkSplitter::onStepDown() {
@@ -255,9 +255,12 @@ void ChunkSplitter::onStepDown() {
     }
     _isPrimary = false;
 
-    // log() << "The ChunkSplitter has stopped and will no longer run new autosplit tasks. Any "
-    //       << "autosplit tasks that have already started will be allowed to finish.";
-    // TODO: Re-enable this log when auto split is actively running on shards.
+    log() << "The ChunkSplitter has stopped and will no longer run new autosplit tasks. Any "
+          << "autosplit tasks that have already started will be allowed to finish.";
+}
+
+void ChunkSplitter::waitForIdle() {
+    _threadPool.waitForIdle();
 }
 
 void ChunkSplitter::trySplitting(std::shared_ptr<ChunkSplitStateDriver> chunkSplitStateDriver,
@@ -268,7 +271,6 @@ void ChunkSplitter::trySplitting(std::shared_ptr<ChunkSplitStateDriver> chunkSpl
     if (!_isPrimary) {
         return;
     }
-
     uassertStatusOK(_threadPool.schedule(
         [ this, csd = std::move(chunkSplitStateDriver), nss, min, max, dataWritten ]() noexcept {
             _runAutosplit(csd, nss, min, max, dataWritten);
@@ -324,6 +326,9 @@ void ChunkSplitter::_runAutosplit(std::shared_ptr<ChunkSplitStateDriver> chunkSp
                                                        maxChunkSizeBytes));
 
         if (splitPoints.size() <= 1) {
+            LOG(1)
+                << "ChunkSplitter attempted split but not enough split points were found for chunk "
+                << redact(chunk.toString());
             // No split points means there isn't enough data to split on; 1 split point means we
             // have between half the chunk size to full chunk size so there is no need to split yet
             return;
@@ -375,6 +380,13 @@ void ChunkSplitter::_runAutosplit(std::shared_ptr<ChunkSplitStateDriver> chunkSp
               << (topChunkMinKey.isEmpty() ? "" : " (top chunk migration suggested" +
                           (std::string)(shouldBalance ? ")" : ", but no migrations allowed)"));
 
+        // Because the ShardServerOpObserver uses the metadata from the CSS for tracking incoming
+        // writes, if we split a chunk but do not force a CSS refresh, subsequent inserts will see
+        // stale metadata and so will not trigger a chunk split. If we force metadata refresh here,
+        // we can limit the amount of time that the op observer is tracking writes on the parent
+        // chunk rather than on its child chunks.
+        forceShardFilteringMetadataRefresh(opCtx.get(), nss, false);
+
         // Balance the resulting chunks if the autobalance option is enabled and if we split at the
         // first or last chunk on the collection as part of top chunk optimization.
         if (!shouldBalance || topChunkMinKey.isEmpty()) {
diff --git a/src/mongo/db/s/chunk_splitter.h b/src/mongo/db/s/chunk_splitter.h
index fcd826559d2..6f497af38f5 100644
--- a/src/mongo/db/s/chunk_splitter.h
+++ b/src/mongo/db/s/chunk_splitter.h
@@ -75,6 +75,12 @@ public:
     void onStepDown();
 
     /**
+     * Blocks until all chunk split tasks in the underlying thread pool have
+     * completed (that is, until the thread pool is idle)
+     */
+    void waitForIdle();
+
+    /**
      * Schedules an autosplit task. This function throws on scheduling failure.
      */
     void trySplitting(std::shared_ptr<ChunkSplitStateDriver> chunkSplitStateDriver,
diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp
index 5786820c8d3..8ad9ebd76d2 100644
--- a/src/mongo/db/s/shard_server_op_observer.cpp
+++ b/src/mongo/db/s/shard_server_op_observer.cpp
@@ -165,7 +165,6 @@ void incrementChunkOnInsertOrUpdate(OperationContext* opCtx,
     // Note that we can assume the simple collation, because shard keys do not support non-simple
     // collations.
     auto chunk = chunkManager.findIntersectingChunkWithSimpleCollation(shardKey);
-
     auto chunkWritesTracker = chunk.getWritesTracker();
     chunkWritesTracker->addBytesWritten(dataWritten);
 
@@ -174,10 +173,8 @@ void incrementChunkOnInsertOrUpdate(OperationContext* opCtx,
     if (chunkWritesTracker->shouldSplit(balancerConfig->getMaxChunkSizeBytes())) {
         auto chunkSplitStateDriver = ChunkSplitStateDriver::tryInitiateSplit(chunkWritesTracker);
         if (chunkSplitStateDriver) {
-            // TODO (SERVER-9287): Enable the following to trigger chunk splitting
-            // ChunkSplitter::get(opCtx).trySplitting(
-            //     std::move(chunkSplitStateDriver), nss, chunk.getMin(), chunk.getMax(),
-            //     dataWritten);
+            ChunkSplitter::get(opCtx).trySplitting(
+                std::move(chunkSplitStateDriver), nss, chunk.getMin(), chunk.getMax(), dataWritten);
         }
     }
 }
diff --git a/src/mongo/db/s/wait_for_ongoing_chunk_splits_command.cpp b/src/mongo/db/s/wait_for_ongoing_chunk_splits_command.cpp
new file mode 100644
index 00000000000..317811b9545
--- /dev/null
+++ b/src/mongo/db/s/wait_for_ongoing_chunk_splits_command.cpp
@@ -0,0 +1,92 @@
+/**
+ *    Copyright (C) 2018 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands.h"
+#include "mongo/db/commands/test_commands_enabled.h"
+#include "mongo/db/s/chunk_splitter.h"
+
+namespace mongo {
+
+namespace {
+class WaitForOngoingChunksSplitsCommand final : public BasicCommand {
+    MONGO_DISALLOW_COPYING(WaitForOngoingChunksSplitsCommand);
+
+public:
+    WaitForOngoingChunksSplitsCommand() : BasicCommand("waitForOngoingChunkSplits") {}
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kAlways;
+    }
+
+    bool adminOnly() const override {
+        return false;
+    }
+
+    bool supportsWriteConcern(const BSONObj& cmd) const override {
+        return false;
+    }
+
+    std::string help() const override {
+        return "block until all autosplit chunk splits have completed on a shard. Test command "
+               "only.";
+    }
+
+    bool requiresAuth() const override {
+        return false;
+    }
+
+    // No auth needed because it only works when enabled via command line.
+    Status checkAuthForOperation(OperationContext* opCtx,
+                                 const std::string& dbname,
+                                 const BSONObj& cmdObj) const override {
+        return Status::OK();
+    }
+
+    bool run(OperationContext* opCtx,
+             const std::string& db,
+             const BSONObj& cmdObj,
+             BSONObjBuilder& result) override {
+        ChunkSplitter::get(opCtx).waitForIdle();
+
+        return true;
+    }
+};
+
+MONGO_INITIALIZER(RegisterWaitForOngoingChunkSplitsCommand)(InitializerContext* context) {
+    if (getTestCommandsEnabled()) {
+        // Leaked intentionally: a Command registers itself when constructed.
+        new WaitForOngoingChunksSplitsCommand();
+    }
+    return Status::OK();
+}
+}
+}
diff --git a/src/mongo/s/write_ops/cluster_write.cpp b/src/mongo/s/write_ops/cluster_write.cpp
index 0f0772d7ec3..e1cc19ef124 100644
--- a/src/mongo/s/write_ops/cluster_write.cpp
+++ b/src/mongo/s/write_ops/cluster_write.cpp
@@ -232,7 +232,8 @@ void ClusterWriter::write(OperationContext* opCtx,
             BatchWriteExec::executeBatch(opCtx, targeter, request, response, stats);
         }
 
-        splitIfNeeded(opCtx, request.getNS(), targeterStats);
+        // TODO Re-enable for mixed-version scenarios
+        // splitIfNeeded(opCtx, request.getNS(), targeterStats);
     }
 }
 
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index 9ca112c48a2..1e23b0f9478 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -431,6 +431,12 @@ var ShardingTest = function(params) {
         throw _getErrorWithCode(res, "command " + tojson(cmd) + " failed: " + tojson(res));
     };
 
+    this.forEachConnection = function(fn) {
+        this._connections.forEach(function(conn) {
+            fn(conn);
+        });
+    };
+
     this.printChangeLog = function() {
         this.config.changelog.find().forEach(function(z) {
             var msg = z.server + "\t" + z.time + "\t" + z.what;