SERVER-28670 Add sharding CatalogCache and donor metrics to serverStatus

Includes metrics for refresh, clone and migration critical section
duration.

(cherry picked from commit c4142a8e0b486f3642b700c9efb208f909e3ff1d)

16 files changed, 495 insertions, 128 deletions

diff --git a/jstests/sharding/basic_sharding_params.js b/jstests/sharding/basic_sharding_params.js
index f21c4d17784..1c1fb11cb11 100644
--- a/jstests/sharding/basic_sharding_params.js
+++ b/jstests/sharding/basic_sharding_params.js
@@ -1,71 +1,77 @@
-// Test of complex sharding initialization
+/**
+ * Test of complex sharding initialization
+ */
 
-function shardingTestUsingObjects() {
-    var st = new ShardingTest({
+(function() {
+    'use strict';
 
-        mongos: {s0: {verbose: 6}, s1: {verbose: 5}},
-        config: {c0: {verbose: 4}},
-        shards: {d0: {verbose: 3}, rs1: {nodes: {d0: {verbose: 2}, a1: {verbose: 1}}}}
-    });
+    function shardingTestUsingObjects() {
+        var st = new ShardingTest({
 
-    var s0 = st.s0;
-    assert.eq(s0, st._mongos[0]);
+            mongos: {s0: {verbose: 6}, s1: {verbose: 5}},
+            config: {c0: {verbose: 4}},
+            shards: {d0: {verbose: 3}, rs1: {nodes: {d0: {verbose: 2}, a1: {verbose: 1}}}}
+        });
 
-    var s1 = st.s1;
-    assert.eq(s1, st._mongos[1]);
+        var s0 = st.s0;
+        assert.eq(s0, st._mongos[0]);
 
-    var c0 = st.c0;
-    assert.eq(c0, st._configServers[0]);
+        var s1 = st.s1;
+        assert.eq(s1, st._mongos[1]);
 
-    var d0 = st.d0;
-    assert.eq(d0, st._connections[0]);
+        var c0 = st.c0;
+        assert.eq(c0, st._configServers[0]);
 
-    var rs1 = st.rs1;
-    assert.eq(rs1, st._rsObjects[1]);
+        var d0 = st.d0;
+        assert.eq(d0, st._connections[0]);
 
-    var rs1_d0 = rs1.nodes[0];
-    var rs1_a1 = rs1.nodes[1];
+        var rs1 = st.rs1;
+        assert.eq(rs1, st._rsObjects[1]);
 
-    assert(s0.commandLine.hasOwnProperty("vvvvvv"));
-    assert(s1.commandLine.hasOwnProperty("vvvvv"));
-    assert(c0.commandLine.hasOwnProperty("vvvv"));
-    assert(d0.commandLine.hasOwnProperty("vvv"));
-    assert(rs1_d0.commandLine.hasOwnProperty("vv"));
-    assert(rs1_a1.commandLine.hasOwnProperty("v"));
+        var rs1_d0 = rs1.nodes[0];
+        var rs1_a1 = rs1.nodes[1];
 
-    st.stop();
-}
+        assert(s0.commandLine.hasOwnProperty("vvvvvv"));
+        assert(s1.commandLine.hasOwnProperty("vvvvv"));
+        assert(c0.commandLine.hasOwnProperty("vvvv"));
+        assert(d0.commandLine.hasOwnProperty("vvv"));
+        assert(rs1_d0.commandLine.hasOwnProperty("vv"));
+        assert(rs1_a1.commandLine.hasOwnProperty("v"));
 
-function shardingTestUsingArrays() {
-    var st = new ShardingTest({
-        mongos: [{verbose: 5}, {verbose: 4}],
-        config: [{verbose: 3}],
-        shards: [{verbose: 2}, {verbose: 1}]
-    });
+        st.stop();
+    }
 
-    var s0 = st.s0;
-    assert.eq(s0, st._mongos[0]);
+    function shardingTestUsingArrays() {
+        var st = new ShardingTest({
+            mongos: [{verbose: 5}, {verbose: 4}],
+            config: [{verbose: 3}],
+            shards: [{verbose: 2}, {verbose: 1}]
+        });
 
-    var s1 = st.s1;
-    assert.eq(s1, st._mongos[1]);
+        var s0 = st.s0;
+        assert.eq(s0, st._mongos[0]);
 
-    var c0 = st.c0;
-    assert.eq(c0, st._configServers[0]);
+        var s1 = st.s1;
+        assert.eq(s1, st._mongos[1]);
 
-    var d0 = st.d0;
-    assert.eq(d0, st._connections[0]);
+        var c0 = st.c0;
+        assert.eq(c0, st._configServers[0]);
 
-    var d1 = st.d1;
-    assert.eq(d1, st._connections[1]);
+        var d0 = st.d0;
+        assert.eq(d0, st._connections[0]);
 
-    assert(s0.commandLine.hasOwnProperty("vvvvv"));
-    assert(s1.commandLine.hasOwnProperty("vvvv"));
-    assert(c0.commandLine.hasOwnProperty("vvv"));
-    assert(d0.commandLine.hasOwnProperty("vv"));
-    assert(d1.commandLine.hasOwnProperty("v"));
+        var d1 = st.d1;
+        assert.eq(d1, st._connections[1]);
 
-    st.stop();
-}
+        assert(s0.commandLine.hasOwnProperty("vvvvv"));
+        assert(s1.commandLine.hasOwnProperty("vvvv"));
+        assert(c0.commandLine.hasOwnProperty("vvv"));
+        assert(d0.commandLine.hasOwnProperty("vv"));
+        assert(d1.commandLine.hasOwnProperty("v"));
 
-shardingTestUsingObjects();
-shardingTestUsingArrays();
+        st.stop();
+    }
+
+    shardingTestUsingObjects();
+    shardingTestUsingArrays();
+})();
diff --git a/jstests/sharding/move_chunk_basic.js b/jstests/sharding/move_chunk_basic.js
index 8ca26547c18..988d741e00e 100644
--- a/jstests/sharding/move_chunk_basic.js
+++ b/jstests/sharding/move_chunk_basic.js
@@ -81,5 +81,4 @@
     testNotHashed({a: 1, b: 1});
 
     st.stop();
-
 })();
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index fc9805460f1..b09b3ba71cc 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -61,6 +61,7 @@ env.Library(
         'sharding_initialization_mongod.cpp',
         'sharding_state.cpp',
         'sharding_state_recovery.cpp',
+        'sharding_statistics.cpp',
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
diff --git a/src/mongo/db/s/active_migrations_registry.cpp b/src/mongo/db/s/active_migrations_registry.cpp
index 53f250ef9fd..2358cbc1f23 100644
--- a/src/mongo/db/s/active_migrations_registry.cpp
+++ b/src/mongo/db/s/active_migrations_registry.cpp
@@ -109,7 +109,7 @@ BSONObj ActiveMigrationsRegistry::getActiveMigrationStatusReport(OperationContex
         AutoGetCollection autoColl(txn, nss.get(), MODE_IS);
 
         auto css = CollectionShardingState::get(txn, nss.get());
-        if (css && css->getMigrationSourceManager()) {
+        if (css->getMigrationSourceManager()) {
             return css->getMigrationSourceManager()->getMigrationStatusReport();
         }
     }
diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp
index 2c2df8cd3f2..419825a8bd0 100644
--- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp
+++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_commands.cpp
@@ -74,7 +74,7 @@ public:
         auto css = CollectionShardingState::get(txn, *nss);
         uassert(ErrorCodes::IllegalOperation,
                 str::stream() << "No active migrations were found for collection " << nss->ns(),
-                css && css->getMigrationSourceManager());
+                css->getMigrationSourceManager());
 
         // It is now safe to access the cloner
         _chunkCloner = dynamic_cast<MigrationChunkClonerSourceLegacy*>(
diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp
index b70617deee3..149a7b12397 100644
--- a/src/mongo/db/s/migration_source_manager.cpp
+++ b/src/mongo/db/s/migration_source_manager.cpp
@@ -42,6 +42,9 @@
 #include "mongo/db/s/operation_sharding_state.h"
 #include "mongo/db/s/sharding_state.h"
 #include "mongo/db/s/sharding_state_recovery.h"
+#include "mongo/db/s/sharding_statistics.h"
+#include "mongo/executor/task_executor.h"
+#include "mongo/executor/task_executor_pool.h"
 #include "mongo/s/catalog/sharding_catalog_client.h"
 #include "mongo/s/catalog/type_chunk.h"
 #include "mongo/s/client/shard_registry.h"
@@ -83,7 +86,7 @@ MigrationSourceManager::MigrationSourceManager(OperationContext* txn,
     : _args(std::move(request)),
       _donorConnStr(std::move(donorConnStr)),
       _recipientHost(std::move(recipientHost)),
-      _startTime() {
+      _stats(ShardingStatistics::get(txn)) {
     invariant(!txn->lockState()->isLocked());
 
     // Disallow moving a chunk to ourselves
@@ -160,6 +163,7 @@ MigrationSourceManager::MigrationSourceManager(OperationContext* txn,
 
 MigrationSourceManager::~MigrationSourceManager() {
     invariant(!_cloneDriver);
+    _stats.totalDonorMoveChunkTimeMillis.addAndFetch(_entireOpTimer.millis());
 }
 
 NamespaceString MigrationSourceManager::getNss() const {
@@ -170,6 +174,7 @@ Status MigrationSourceManager::startClone(OperationContext* txn) {
     invariant(!txn->lockState()->isLocked());
     invariant(_state == kCreated);
     auto scopedGuard = MakeGuard([&] { cleanupOnError(txn); });
+    _stats.countDonorMoveChunkStarted.addAndFetch(1);
 
     grid.catalogClient(txn)->logChange(txn,
                                        "moveChunk.start",
@@ -184,6 +189,8 @@ Status MigrationSourceManager::startClone(OperationContext* txn) {
     _cloneDriver = stdx::make_unique<MigrationChunkClonerSourceLegacy>(
         _args, _collectionMetadata->getKeyPattern(), _donorConnStr, _recipientHost);
 
+    _cloneAndCommitTimer.reset();
+
     {
         // Register for notifications from the replication subsystem
         ScopedTransaction scopedXact(txn, MODE_IX);
@@ -207,6 +214,8 @@ Status MigrationSourceManager::awaitToCatchUp(OperationContext* txn) {
     invariant(!txn->lockState()->isLocked());
     invariant(_state == kCloning);
     auto scopedGuard = MakeGuard([&] { cleanupOnError(txn); });
+    _stats.totalDonorChunkCloneTimeMillis.addAndFetch(_cloneAndCommitTimer.millis());
+    _cloneAndCommitTimer.reset();
 
     // Block until the cloner deems it appropriate to enter the critical section.
     Status catchUpStatus = _cloneDriver->awaitUntilCriticalSectionIsAppropriate(
@@ -224,6 +233,8 @@ Status MigrationSourceManager::enterCriticalSection(OperationContext* txn) {
     invariant(!txn->lockState()->isLocked());
     invariant(_state == kCloneCaughtUp);
     auto scopedGuard = MakeGuard([&] { cleanupOnError(txn); });
+    _stats.totalDonorChunkCloneTimeMillis.addAndFetch(_cloneAndCommitTimer.millis());
+    _cloneAndCommitTimer.reset();
 
     // Mark the shard as running critical operation, which requires recovery on crash
     Status status = ShardingStateRecovery::startMetadataOp(txn);
@@ -319,6 +330,8 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn
 
     builder.append(kWriteConcernField, kMajorityWriteConcern.toBSON());
 
+    Timer t;
+
     auto commitChunkMigrationResponse =
         grid.shardRegistry()->getConfigShard()->runCommandWithFixedRetryAttempts(
             txn,
@@ -426,6 +439,9 @@ Status MigrationSourceManager::commitChunkMetadataOnConfig(OperationContext* txn
     MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeLeavingCriticalSection);
 
     scopedGuard.Dismiss();
+
+    _stats.totalCriticalSectionCommitTimeMillis.addAndFetch(t.millis());
+
     _cleanup(txn);
 
     grid.catalogClient(txn)->logChange(txn,
@@ -486,6 +502,8 @@ void MigrationSourceManager::_cleanup(OperationContext* txn) {
     // Decrement the metadata op counter outside of the collection lock in order to hold it for as
     // short as possible.
     if (_state == kCriticalSection || _state == kCloneCompleted) {
+        _stats.totalCriticalSectionTimeMillis.addAndFetch(_cloneAndCommitTimer.millis());
+
         ShardingStateRecovery::endMetadataOp(txn);
     }
 
diff --git a/src/mongo/db/s/migration_source_manager.h b/src/mongo/db/s/migration_source_manager.h
index 27d4480a665..02b6c5fc3ff 100644
--- a/src/mongo/db/s/migration_source_manager.h
+++ b/src/mongo/db/s/migration_source_manager.h
@@ -41,6 +41,7 @@ namespace mongo {
 
 class MigrationChunkClonerSource;
 class OperationContext;
+struct ShardingStatistics;
 
 /**
  * The donor-side migration state machine. This object must be created and owned by a single thread,
@@ -211,8 +212,15 @@ private:
     // The resolved primary of the recipient shard
     const HostAndPort _recipientHost;
 
-    // Gets initialized at creation time and will time the entire move chunk operation
-    const Timer _startTime;
+    // Stores a reference to the process sharding statistics object which needs to be updated
+    ShardingStatistics& _stats;
+
+    // Times the entire moveChunk operation
+    const Timer _entireOpTimer;
+
+    // Starts counting from creation time and is used to time various parts from the lifetime of the
+    // move chunk sequence
+    Timer _cloneAndCommitTimer;
 
     // The current state. Used only for diagnostics and validation.
     State _state{kCreated};
diff --git a/src/mongo/db/s/sharding_server_status.cpp b/src/mongo/db/s/sharding_server_status.cpp
index 8f58e24600b..ac3a638d7b6 100644
--- a/src/mongo/db/s/sharding_server_status.cpp
+++ b/src/mongo/db/s/sharding_server_status.cpp
@@ -31,38 +31,57 @@
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/db/commands/server_status.h"
 #include "mongo/db/s/sharding_state.h"
+#include "mongo/db/s/sharding_statistics.h"
 #include "mongo/db/server_options.h"
+#include "mongo/s/balancer_configuration.h"
+#include "mongo/s/catalog_cache.h"
+#include "mongo/s/client/shard_registry.h"
 #include "mongo/s/grid.h"
 
 namespace mongo {
 namespace {
 
-class ShardingServerStatus : public ServerStatusSection {
+bool isClusterNode() {
+    return serverGlobalParams.clusterRole != ClusterRole::None;
+}
+
+class ShardingServerStatus final : public ServerStatusSection {
 public:
     ShardingServerStatus() : ServerStatusSection("sharding") {}
 
-    bool includeByDefault() const final {
-        return true;
+    bool includeByDefault() const override {
+        return isClusterNode();
     }
 
-    BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const final {
+    BSONObj generateSection(OperationContext* txn,
+                            const BSONElement& configElement) const override {
+        if (!isClusterNode())
+            return {};
+
+        auto const shardingState = ShardingState::get(txn);
+        if (!shardingState->enabled())
+            return {};
+
+        auto const grid = Grid::get(txn);
+        auto const shardRegistry = grid->shardRegistry();
+
         BSONObjBuilder result;
 
-        auto shardingState = ShardingState::get(txn);
-        if (shardingState->enabled() &&
-            serverGlobalParams.clusterRole != ClusterRole::ConfigServer) {
-            result.append("configsvrConnectionString",
-                          shardingState->getConfigServer(txn).toString());
-
-            Grid::get(txn)->configOpTime().append(&result, "lastSeenConfigServerOpTime");
-
-            // Get a migration status report if a migration is active for which this is the source
-            // shard. ShardingState::getActiveMigrationStatusReport will take an IS lock on the
-            // namespace of the active migration if there is one that is active.
-            BSONObj migrationStatus = ShardingState::get(txn)->getActiveMigrationStatusReport(txn);
-            if (!migrationStatus.isEmpty()) {
-                result.append("migrations", migrationStatus);
-            }
+        result.append("configsvrConnectionString",
+                      shardRegistry->getConfigServerConnectionString().toString());
+
+        grid->configOpTime().append(&result, "lastSeenConfigServerOpTime");
+
+        const long long maxChunkSizeInBytes =
+            grid->getBalancerConfiguration()->getMaxChunkSizeBytes();
+        result.append("maxChunkSizeInBytes", maxChunkSizeInBytes);
+
+        // Get a migration status report if a migration is active for which this is the source
+        // shard. ShardingState::getActiveMigrationStatusReport will take an IS lock on the
+        // namespace of the active migration if there is one that is active.
+        BSONObj migrationStatus = shardingState->getActiveMigrationStatusReport(txn);
+        if (!migrationStatus.isEmpty()) {
+            result.append("migrations", migrationStatus);
         }
 
         return result.obj();
@@ -70,5 +89,33 @@ public:
 
 } shardingServerStatus;
 
+class ShardingStatisticsServerStatus final : public ServerStatusSection {
+public:
+    ShardingStatisticsServerStatus() : ServerStatusSection("shardingStatistics") {}
+
+    bool includeByDefault() const override {
+        return isClusterNode();
+    }
+
+    BSONObj generateSection(OperationContext* opCtx,
+                            const BSONElement& configElement) const override {
+        if (!isClusterNode())
+            return {};
+
+        auto const shardingState = ShardingState::get(opCtx);
+        if (!shardingState->enabled())
+            return {};
+
+        auto const grid = Grid::get(opCtx);
+        auto const catalogCache = grid->catalogCache();
+
+        BSONObjBuilder result;
+        ShardingStatistics::get(opCtx).report(&result);
+        catalogCache->report(&result);
+        return result.obj();
+    }
+
+} shardingStatisticsServerStatus;
+
 }  // namespace
 }  // namespace mongo
diff --git a/src/mongo/db/s/sharding_state.cpp b/src/mongo/db/s/sharding_state.cpp
index 02c75e287fe..22838fcdb26 100644
--- a/src/mongo/db/s/sharding_state.cpp
+++ b/src/mongo/db/s/sharding_state.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/s/operation_sharding_state.h"
 #include "mongo/db/s/sharded_connection_info.h"
 #include "mongo/db/s/sharding_initialization_mongod.h"
+#include "mongo/db/s/sharding_statistics.h"
 #include "mongo/db/s/type_shard_identity.h"
 #include "mongo/executor/network_interface_factory.h"
 #include "mongo/executor/network_interface_thread_pool.h"
@@ -230,6 +231,8 @@ Status ShardingState::onStaleShardVersion(OperationContext* txn,
     LOG(2) << "metadata refresh requested for " << nss.ns() << " at shard version "
            << expectedVersion;
 
+    ShardingStatistics::get(txn).countStaleConfigErrors.addAndFetch(1);
+
     // Ensure any ongoing migrations have completed
     auto& oss = OperationShardingState::get(txn);
     oss.waitForMigrationCriticalSectionSignal(txn);
diff --git a/src/mongo/db/s/sharding_statistics.cpp b/src/mongo/db/s/sharding_statistics.cpp
new file mode 100644
index 00000000000..905f1869bf9
--- /dev/null
+++ b/src/mongo/db/s/sharding_statistics.cpp
@@ -0,0 +1,62 @@
+/**
+ *    Copyright (C) 2018 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/sharding_statistics.h"
+
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/operation_context.h"
+#include "mongo/db/service_context.h"
+
+namespace mongo {
+namespace {
+
+const auto getShardingStatistics = ServiceContext::declareDecoration<ShardingStatistics>();
+
+}  // namespace
+
+ShardingStatistics& ShardingStatistics::get(ServiceContext* serviceContext) {
+    return getShardingStatistics(serviceContext);
+}
+
+ShardingStatistics& ShardingStatistics::get(OperationContext* opCtx) {
+    return get(opCtx->getServiceContext());
+}
+
+void ShardingStatistics::report(BSONObjBuilder* builder) const {
+    builder->append("countStaleConfigErrors", countStaleConfigErrors.load());
+
+    builder->append("countDonorMoveChunkStarted", countDonorMoveChunkStarted.load());
+    builder->append("totalDonorChunkCloneTimeMillis", totalDonorChunkCloneTimeMillis.load());
+    builder->append("totalCriticalSectionCommitTimeMillis",
+                    totalCriticalSectionCommitTimeMillis.load());
+    builder->append("totalCriticalSectionTimeMillis", totalCriticalSectionTimeMillis.load());
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/s/sharding_statistics.h b/src/mongo/db/s/sharding_statistics.h
new file mode 100644
index 00000000000..c4eb7abd728
--- /dev/null
+++ b/src/mongo/db/s/sharding_statistics.h
@@ -0,0 +1,85 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects
+ *    for all of the code used other than as permitted herein. If you modify
+ *    file(s) with this exception, you may extend this exception to your
+ *    version of the file(s), but you are not obligated to do so. If you do not
+ *    wish to do so, delete this exception statement from your version. If you
+ *    delete this exception statement from all source files in the program,
+ *    then also delete it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/platform/atomic_word.h"
+
+namespace mongo {
+
+class BSONObjBuilder;
+class OperationContext;
+class ServiceContext;
+
+/**
+ * Encapsulates per-process statistics for the sharding subsystem.
+ */
+struct ShardingStatistics {
+    // Counts how many times threads hit stale config exception (which is what triggers metadata
+    // refreshes)
+    AtomicInt64 countStaleConfigErrors{0};
+
+    // Cumulative, always-increasing counter of how many chunks did this node start donating
+    // (whether they succeeded or not)
+    AtomicInt64 countDonorMoveChunkStarted{0};
+
+    // Cumulative, always-increasing counter of how much time the entire move chunk operation took
+    // (excluding range deletion)
+    AtomicInt64 totalDonorMoveChunkTimeMillis{0};
+
+    // Cumulative, always-increasing counter of how much time the clone phase took on the donor
+    // node, before it was appropriate to enter the critical section
+    AtomicInt64 totalDonorChunkCloneTimeMillis{0};
+
+    // Cumulative, always-increasing counter of how much time the critical section's commit phase
+    // took (this is the period of time when all operations on the collection are blocked, not just
+    // the reads)
+    AtomicInt64 totalCriticalSectionCommitTimeMillis{0};
+
+    // Cumulative, always-increasing counter of how much time the entire critical section took. It
+    // includes the time the recipient took to fetch the latest modifications from the donor and
+    // persist them plus the critical section commit time.
+    //
+    // The value of totalCriticalSectionTimeMillis - totalCriticalSectionCommitTimeMillis gives the
+    // duration of the catch-up phase of the critical section (where the last mods are transferred
+    // from the donor to the recipient).
+    AtomicInt64 totalCriticalSectionTimeMillis{0};
+
+    /**
+     * Obtains the per-process instance of the sharding statistics object.
+     */
+    static ShardingStatistics& get(ServiceContext* serviceContext);
+    static ShardingStatistics& get(OperationContext* opCtx);
+
+    /**
+     * Reports the accumulated statistics for serverStatus.
+     */
+    void report(BSONObjBuilder* builder) const;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/s/catalog_cache.cpp b/src/mongo/s/catalog_cache.cpp
index 103a69550a2..fba58664e19 100644
--- a/src/mongo/s/catalog_cache.cpp
+++ b/src/mongo/s/catalog_cache.cpp
@@ -34,6 +34,7 @@
 
 #include "mongo/base/status.h"
 #include "mongo/base/status_with.h"
+#include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/query/collation/collator_factory_interface.h"
 #include "mongo/db/repl/optime_with.h"
@@ -45,6 +46,7 @@
 #include "mongo/s/grid.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/util/log.h"
+#include "mongo/util/scopeguard.h"
 #include "mongo/util/timer.h"
 
 namespace mongo {
@@ -213,7 +215,16 @@ StatusWith<CachedCollectionRoutingInfo> CatalogCache::getCollectionRoutingInfo(
             ul.unlock();
 
             auto refreshStatus = [&]() {
+                Timer t;
+                ON_BLOCK_EXIT([&] { _stats.totalRefreshWaitTimeMicros.addAndFetch(t.micros()); });
+
                 try {
+                    const Milliseconds kReportingInterval{250};
+                    while (!refreshNotification->waitFor(opCtx, kReportingInterval)) {
+                        _stats.totalRefreshWaitTimeMicros.addAndFetch(t.micros());
+                        t.reset();
+                    }
+
                     return refreshNotification->get(opCtx);
                 } catch (const DBException& ex) {
                     return ex.toStatus();
@@ -256,6 +267,8 @@ StatusWith<CachedCollectionRoutingInfo> CatalogCache::getShardedCollectionRoutin
 }
 
 void CatalogCache::onStaleConfigError(CachedCollectionRoutingInfo&& ccriToInvalidate) {
+    _stats.countStaleConfigErrors.addAndFetch(1);
+
     // Ensure the move constructor of CachedCollectionRoutingInfo is invoked in order to clear the
     // input argument so it can't be used anymore
     auto ccri(ccriToInvalidate);
@@ -327,6 +340,25 @@ void CatalogCache::purgeAllDatabases() {
     _databases.clear();
 }
 
+void CatalogCache::report(BSONObjBuilder* builder) const {
+    BSONObjBuilder cacheStatsBuilder(builder->subobjStart("catalogCache"));
+
+    size_t numDatabaseEntries;
+    size_t numCollectionEntries{0};
+    {
+        stdx::lock_guard<stdx::mutex> ul(_mutex);
+        numDatabaseEntries = _databases.size();
+        for (const auto& entry : _databases) {
+            numCollectionEntries += entry.second->collections.size();
+        }
+    }
+
+    cacheStatsBuilder.append("numDatabaseEntries", static_cast<long long>(numDatabaseEntries));
+    cacheStatsBuilder.append("numCollectionEntries", static_cast<long long>(numCollectionEntries));
+
+    _stats.report(&cacheStatsBuilder);
+}
+
 std::shared_ptr<CatalogCache::DatabaseInfoEntry> CatalogCache::_getDatabase(OperationContext* opCtx,
                                                                             StringData dbName) {
     stdx::lock_guard<stdx::mutex> lg(_mutex);
@@ -368,15 +400,47 @@ void CatalogCache::_scheduleCollectionRefresh_inlock(
     std::shared_ptr<ChunkManager> existingRoutingInfo,
     const NamespaceString& nss,
     int refreshAttempt) {
+
+    // If we have an existing chunk manager, the refresh is considered "incremental", regardless of
+    // how many chunks are in the differential
+    const bool isIncremental(existingRoutingInfo);
+
+    if (isIncremental) {
+        _stats.numActiveIncrementalRefreshes.addAndFetch(1);
+        _stats.countIncrementalRefreshesStarted.addAndFetch(1);
+    } else {
+        _stats.numActiveFullRefreshes.addAndFetch(1);
+        _stats.countFullRefreshesStarted.addAndFetch(1);
+    }
+
     Timer t;
+    // Invoked when one iteration of getChunksSince has completed, whether with success or error
+    const auto onRefreshCompleted =
+        [this, t, nss, isIncremental](const Status& status, ChunkManager* routingInfoAfterRefresh) {
+            if (isIncremental) {
+                _stats.numActiveIncrementalRefreshes.subtractAndFetch(1);
+            } else {
+                _stats.numActiveFullRefreshes.subtractAndFetch(1);
+            }
 
-    const ChunkVersion startingCollectionVersion =
-        (existingRoutingInfo ? existingRoutingInfo->getVersion() : ChunkVersion::UNSHARDED());
+            if (!status.isOK()) {
+                _stats.countFailedRefreshes.addAndFetch(1);
+
+                log() << "Refresh for collection " << nss << " took " << t.millis()
+                      << " ms and failed" << causedBy(redact(status));
+            } else if (routingInfoAfterRefresh) {
+                log() << "Refresh for collection " << nss << " took " << t.millis()
+                      << " ms and found version " << routingInfoAfterRefresh->getVersion();
+            } else {
+                log() << "Refresh for collection " << nss << " took " << t.millis()
+                      << " ms and found the collection is not sharded";
+            }
+        };
 
+    // Invoked if getChunksSince resulted in error
     const auto refreshFailed_inlock =
-        [ this, t, dbEntry, nss, refreshAttempt ](const Status& status) noexcept {
-        log() << "Refresh for collection " << nss << " took " << t.millis() << " ms and failed"
-              << causedBy(redact(status));
+        [ this, dbEntry, nss, refreshAttempt, onRefreshCompleted ](const Status& status) noexcept {
+        onRefreshCompleted(status, nullptr);
 
         auto& collections = dbEntry->collections;
         auto it = collections.find(nss.ns());
@@ -397,13 +461,15 @@ void CatalogCache::_scheduleCollectionRefresh_inlock(
     };
 
     const auto refreshCallback =
-        [ this, t, dbEntry, nss, existingRoutingInfo, refreshFailed_inlock ](
+        [ this, dbEntry, nss, existingRoutingInfo, refreshFailed_inlock, onRefreshCompleted ](
             OperationContext * opCtx,
             StatusWith<CatalogCacheLoader::CollectionAndChangedChunks> swCollAndChunks) noexcept {
         std::shared_ptr<ChunkManager> newRoutingInfo;
         try {
             newRoutingInfo = refreshCollectionRoutingInfo(
                 opCtx, nss, std::move(existingRoutingInfo), std::move(swCollAndChunks));
+
+            onRefreshCompleted(Status::OK(), newRoutingInfo.get());
         } catch (const DBException& ex) {
             stdx::lock_guard<stdx::mutex> lg(_mutex);
             refreshFailed_inlock(ex.toStatus());
@@ -411,6 +477,7 @@ void CatalogCache::_scheduleCollectionRefresh_inlock(
         }
 
         stdx::lock_guard<stdx::mutex> lg(_mutex);
+
         auto& collections = dbEntry->collections;
         auto it = collections.find(nss.ns());
         invariant(it != collections.end());
@@ -421,18 +488,15 @@ void CatalogCache::_scheduleCollectionRefresh_inlock(
         collEntry.refreshCompletionNotification = nullptr;
 
         if (!newRoutingInfo) {
-            log() << "Refresh for collection " << nss << " took " << t.millis()
-                  << " and found the collection is not sharded";
-
             collections.erase(it);
         } else {
-            log() << "Refresh for collection " << nss << " took " << t.millis()
-                  << " ms and found version " << newRoutingInfo->getVersion();
-
             collEntry.routingInfo = std::move(newRoutingInfo);
         }
     };
 
+    const ChunkVersion startingCollectionVersion =
+        (existingRoutingInfo ? existingRoutingInfo->getVersion() : ChunkVersion::UNSHARDED());
+
     log() << "Refreshing chunks for collection " << nss << " based on version "
           << startingCollectionVersion;
 
@@ -451,6 +515,20 @@ void CatalogCache::_scheduleCollectionRefresh_inlock(
     }
 }
 
+void CatalogCache::Stats::report(BSONObjBuilder* builder) const {
+    builder->append("countStaleConfigErrors", countStaleConfigErrors.load());
+
+    builder->append("totalRefreshWaitTimeMicros", totalRefreshWaitTimeMicros.load());
+
+    builder->append("numActiveIncrementalRefreshes", numActiveIncrementalRefreshes.load());
+    builder->append("countIncrementalRefreshesStarted", countIncrementalRefreshesStarted.load());
+
+    builder->append("numActiveFullRefreshes", numActiveFullRefreshes.load());
+    builder->append("countFullRefreshesStarted", countFullRefreshesStarted.load());
+
+    builder->append("countFailedRefreshes", countFailedRefreshes.load());
+}
+
 CachedDatabaseInfo::CachedDatabaseInfo(std::shared_ptr<CatalogCache::DatabaseInfoEntry> db)
     : _db(std::move(db)) {}
 
diff --git a/src/mongo/s/catalog_cache.h b/src/mongo/s/catalog_cache.h
index 5631c6fa00b..c7a3940c1a7 100644
--- a/src/mongo/s/catalog_cache.h
+++ b/src/mongo/s/catalog_cache.h
@@ -30,6 +30,7 @@
 
 #include "mongo/base/disallow_copying.h"
 #include "mongo/base/string_data.h"
+#include "mongo/platform/atomic_word.h"
 #include "mongo/s/catalog_cache_loader.h"
 #include "mongo/s/chunk_manager.h"
 #include "mongo/s/chunk_version.h"
@@ -40,6 +41,7 @@
 
 namespace mongo {
 
+class BSONObjBuilder;
 class CachedDatabaseInfo;
 class CachedCollectionRoutingInfo;
 class OperationContext;
@@ -113,6 +115,11 @@ public:
      */
     void purgeAllDatabases();
 
+    /**
+     * Reports statistics about the catalog cache to be used by serverStatus
+     */
+    void report(BSONObjBuilder* builder) const;
+
 private:
     // Make the cache entries friends so they can access the private classes below
     friend class CachedDatabaseInfo;
@@ -165,8 +172,42 @@ private:
     // Interface from which chunks will be retrieved
     const std::unique_ptr<CatalogCacheLoader> _cacheLoader;
 
+    // Encapsulates runtime statistics across all collections in the catalog cache
+    struct Stats {
+        // Counts how many times threads hit stale config exception (which is what triggers metadata
+        // refreshes)
+        AtomicInt64 countStaleConfigErrors{0};
+
+        // Cumulative, always-increasing counter of how much time threads waiting for refresh
+        // combined
+        AtomicInt64 totalRefreshWaitTimeMicros{0};
+
+        // Tracks how many incremental refreshes are waiting to complete currently
+        AtomicInt64 numActiveIncrementalRefreshes{0};
+
+        // Cumulative, always-increasing counter of how many incremental refreshes have been kicked
+        // off
+        AtomicInt64 countIncrementalRefreshesStarted{0};
+
+        // Tracks how many full refreshes are waiting to complete currently
+        AtomicInt64 numActiveFullRefreshes{0};
+
+        // Cumulative, always-increasing counter of how many full refreshes have been kicked off
+        AtomicInt64 countFullRefreshesStarted{0};
+
+        // Cumulative, always-increasing counter of how many full or incremental refreshes failed
+        // for whatever reason
+        AtomicInt64 countFailedRefreshes{0};
+
+        /**
+         * Reports the accumulated statistics for serverStatus.
+         */
+        void report(BSONObjBuilder* builder) const;
+
+    } _stats;
+
     // Mutex to serialize access to the structures below
-    stdx::mutex _mutex;
+    mutable stdx::mutex _mutex;
 
     // Map from DB name to the info for that database
     DatabaseInfoMap _databases;
diff --git a/src/mongo/s/s_sharding_server_status.cpp b/src/mongo/s/s_sharding_server_status.cpp
index 1d37f83c9c4..d9100f7e3ca 100644
--- a/src/mongo/s/s_sharding_server_status.cpp
+++ b/src/mongo/s/s_sharding_server_status.cpp
@@ -30,6 +30,8 @@
 
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/db/commands/server_status.h"
+#include "mongo/s/balancer_configuration.h"
+#include "mongo/s/catalog_cache.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/grid.h"
 
@@ -39,35 +41,52 @@ namespace {
 
 class ShardingServerStatus : public ServerStatusSection {
 public:
-    ShardingServerStatus();
+    ShardingServerStatus() : ServerStatusSection("sharding") {}
 
-    bool includeByDefault() const final;
+    bool includeByDefault() const override {
+        return true;
+    }
 
-    BSONObj generateSection(OperationContext* txn, const BSONElement& configElement) const final;
-};
+    BSONObj generateSection(OperationContext* txn,
+                            const BSONElement& configElement) const override {
+        auto const grid = Grid::get(txn);
+        auto const shardRegistry = grid->shardRegistry();
 
-}  // namespace
+        BSONObjBuilder result;
+
+        result.append("configsvrConnectionString",
+                      shardRegistry->getConfigServerConnectionString().toString());
+
+        grid->configOpTime().append(&result, "lastSeenConfigServerOpTime");
 
-ShardingServerStatus shardingServerStatus;
+        const long long maxChunkSizeInBytes =
+            grid->getBalancerConfiguration()->getMaxChunkSizeBytes();
+        result.append("maxChunkSizeInBytes", maxChunkSizeInBytes);
 
-ShardingServerStatus::ShardingServerStatus() : ServerStatusSection("sharding") {}
+        return result.obj();
+    }
 
-bool ShardingServerStatus::includeByDefault() const {
-    return true;
-}
+} shardingServerStatus;
 
-// This implementation runs on mongoS.
-BSONObj ShardingServerStatus::generateSection(OperationContext* txn,
-                                              const BSONElement& configElement) const {
-    invariant(grid.shardRegistry());
+class ShardingStatisticsServerStatus final : public ServerStatusSection {
+public:
+    ShardingStatisticsServerStatus() : ServerStatusSection("shardingStatistics") {}
+
+    bool includeByDefault() const override {
+        return true;
+    }
 
-    BSONObjBuilder result;
-    result.append("configsvrConnectionString",
-                  grid.shardRegistry()->getConfigServerConnectionString().toString());
+    BSONObj generateSection(OperationContext* txn,
+                            const BSONElement& configElement) const override {
+        auto const grid = Grid::get(txn);
+        auto const catalogCache = grid->catalogCache();
 
-    grid.configOpTime().append(&result, "lastSeenConfigServerOpTime");
+        BSONObjBuilder result;
+        catalogCache->report(&result);
+        return result.obj();
+    }
 
-    return result.obj();
-}
+} shardingStatisticsServerStatus;
 
+}  // namespace
 }  // namespace mongo
diff --git a/src/mongo/util/concurrency/notification.h b/src/mongo/util/concurrency/notification.h
index 32ce9caa068..d24fc84e5f9 100644
--- a/src/mongo/util/concurrency/notification.h
+++ b/src/mongo/util/concurrency/notification.h
@@ -97,8 +97,10 @@ public:
     }
 
     /**
-     * If the notification is not set, blocks either until it becomes set or until the waitTimeout
-     * expires. If the wait is interrupted, throws an exception. Otherwise, returns immediately.
+     * If the notification is set, returns immediately. Otherwise, blocks until it either becomes
+     * set or the waitTimeout expires, whichever comes first. Returns true if the notification is
+     * set (in which case a subsequent call to get is guaranteed to not block) or false otherwise.
+     * If the wait is interrupted, throws an exception.
      */
     bool waitFor(OperationContext* txn, Microseconds waitTimeout) {
         const auto waitDeadline = Date_t::now() + waitTimeout;
diff --git a/src/mongo/util/timer.h b/src/mongo/util/timer.h
index 05a5936c69e..3b1ab146970 100644
--- a/src/mongo/util/timer.h
+++ b/src/mongo/util/timer.h
@@ -1,6 +1,5 @@
-// @file timer.h
-
-/*    Copyright 2010 10gen Inc.
+/**
+ *    Copyright (C) 2018 MongoDB Inc.
  *
  *    This program is free software: you can redistribute it and/or  modify
  *    it under the terms of the GNU Affero General Public License, version 3,
@@ -31,7 +30,6 @@
 
 #include "mongo/util/time_support.h"
 
-
 namespace mongo {
 
 class TickSource;
@@ -39,26 +37,29 @@ class TickSource;
 /**
  * Time tracking object.
  */
-class Timer /*copyable*/ {
+class Timer {
 public:
     /**
-     * Creates a timer with the system default tick source. Should not be created before
-     * global initialization completes.
+     * Creates a timer with the system default tick source. Should not be created before global
+     * initialization completes.
      */
     Timer();
 
     /**
-     * Creates a timer using the specified tick source. Caller retains ownership of
-     * TickSource, and must keep it in scope until Timer goes out of scope.
+     * Creates a timer using the specified tick source. Caller retains ownership of TickSource and
+     * must keep it in scope until Timer goes out of scope.
      */
     explicit Timer(TickSource* tickSource);
 
-    int seconds() const {
-        return static_cast<int>(micros() / 1000000);
+    long long micros() const {
+        return static_cast<long long>((now() - _old) * _microsPerCount);
     }
     int millis() const {
         return static_cast<int>(micros() / 1000);
     }
+    int seconds() const {
+        return static_cast<int>(micros() / 1000000);
+    }
     int minutes() const {
         return seconds() / 60;
     }
@@ -74,15 +75,11 @@ public:
         return static_cast<int>(deltaMicros / 1000);
     }
 
-    inline long long micros() const {
-        return static_cast<long long>((now() - _old) * _microsPerCount);
-    }
-
     Microseconds elapsed() const {
         return Microseconds{micros()};
     }
 
-    inline void reset() {
+    void reset() {
         _old = now();
     }
 
@@ -97,4 +94,5 @@ private:
 
     long long _old;
 };
+
 }  // namespace mongo