SERVER-28431 shard chunk loader interface implementation

16 files changed, 80 insertions, 23 deletions

diff --git a/src/mongo/s/catalog/type_chunk.cpp b/src/mongo/s/catalog/type_chunk.cpp
index eb286b7a631..5d40cc2ecdf 100644
--- a/src/mongo/s/catalog/type_chunk.cpp
+++ b/src/mongo/s/catalog/type_chunk.cpp
@@ -343,10 +343,6 @@ std::string ChunkType::genID(StringData ns, const BSONObj& o) {
 }
 
 Status ChunkType::validate() const {
-    if (!_ns.is_initialized() || _ns->empty()) {
-        return Status(ErrorCodes::NoSuchKey, str::stream() << "missing " << ns.name() << " field");
-    }
-
     if (!_min.is_initialized() || _min->isEmpty()) {
         return Status(ErrorCodes::NoSuchKey, str::stream() << "missing " << min.name() << " field");
     }
diff --git a/src/mongo/s/catalog/type_chunk.h b/src/mongo/s/catalog/type_chunk.h
index 96ee1588a6a..e80a7df3029 100644
--- a/src/mongo/s/catalog/type_chunk.h
+++ b/src/mongo/s/catalog/type_chunk.h
@@ -255,7 +255,7 @@ public:
 private:
     // Convention: (M)andatory, (O)ptional, (S)pecial; (C)onfig, (S)hard.
 
-    // (M)(C)     collection this chunk is in
+    // (O)(C)     collection this chunk is in
     boost::optional<std::string> _ns;
     // (M)(C)(S)  first key of the range, inclusive
     boost::optional<BSONObj> _min;
diff --git a/src/mongo/s/catalog/type_shard_collection.h b/src/mongo/s/catalog/type_shard_collection.h
index 43118b4c0d5..c3637e2c5aa 100644
--- a/src/mongo/s/catalog/type_shard_collection.h
+++ b/src/mongo/s/catalog/type_shard_collection.h
@@ -79,6 +79,13 @@ public:
     static const BSONField<bool> refreshing;
     static const BSONField<long long> refreshSequenceNumber;
 
+    explicit ShardCollectionType(const NamespaceString& uuid,
+                                 const NamespaceString& nss,
+                                 const OID& epoch,
+                                 const KeyPattern& keyPattern,
+                                 const BSONObj& defaultCollation,
+                                 const bool& unique);
+
     /**
      * Constructs a new ShardCollectionType object from BSON. Also does validation of the contents.
      */
@@ -145,13 +152,6 @@ public:
     }
 
 private:
-    ShardCollectionType(const NamespaceString& uuid,
-                        const NamespaceString& nss,
-                        const OID& epoch,
-                        const KeyPattern& keyPattern,
-                        const BSONObj& defaultCollation,
-                        const bool& unique);
-
     // Will become the UUID when available. Currently a duplicate of '_nss'.
     NamespaceString _uuid;
 
diff --git a/src/mongo/s/catalog_cache.cpp b/src/mongo/s/catalog_cache.cpp
index 87f53ebadad..d81f222bfde 100644
--- a/src/mongo/s/catalog_cache.cpp
+++ b/src/mongo/s/catalog_cache.cpp
@@ -40,9 +40,7 @@
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_database.h"
 #include "mongo/s/client/shard_registry.h"
-#include "mongo/s/config_server_catalog_cache_loader.h"
 #include "mongo/s/grid.h"
-#include "mongo/stdx/memory.h"
 #include "mongo/util/log.h"
 #include "mongo/util/timer.h"
 
@@ -157,6 +155,9 @@ std::shared_ptr<ChunkManager> refreshCollectionRoutingInfo(
 
 CatalogCache::CatalogCache() : _cacheLoader(stdx::make_unique<ConfigServerCatalogCacheLoader>()) {}
 
+CatalogCache::CatalogCache(std::unique_ptr<CatalogCacheLoader> cacheLoader)
+    : _cacheLoader(std::move(cacheLoader)) {}
+
 CatalogCache::~CatalogCache() = default;
 
 StatusWith<CachedDatabaseInfo> CatalogCache::getDatabase(OperationContext* opCtx,
diff --git a/src/mongo/s/catalog_cache.h b/src/mongo/s/catalog_cache.h
index e13a2b0c557..fa81d0a7100 100644
--- a/src/mongo/s/catalog_cache.h
+++ b/src/mongo/s/catalog_cache.h
@@ -34,6 +34,8 @@
 #include "mongo/s/chunk_manager.h"
 #include "mongo/s/chunk_version.h"
 #include "mongo/s/client/shard.h"
+#include "mongo/s/config_server_catalog_cache_loader.h"
+#include "mongo/stdx/memory.h"
 #include "mongo/stdx/mutex.h"
 #include "mongo/util/concurrency/notification.h"
 #include "mongo/util/string_map.h"
@@ -55,7 +57,13 @@ class CatalogCache {
     MONGO_DISALLOW_COPYING(CatalogCache);
 
 public:
+    /**
+     * Defaults to instantiating a ConfigServerCatalogCacheLoader.
+     */
     CatalogCache();
+
+    CatalogCache(std::unique_ptr<CatalogCacheLoader> cacheLoader);
+
     ~CatalogCache();
 
     /**
diff --git a/src/mongo/s/catalog_cache_loader.h b/src/mongo/s/catalog_cache_loader.h
index 6cdfc487513..21702da77e3 100644
--- a/src/mongo/s/catalog_cache_loader.h
+++ b/src/mongo/s/catalog_cache_loader.h
@@ -68,7 +68,7 @@ public:
 
     /**
      * Non-blocking call, which requests the chunks changed since the specified version to be
-     * fetched from the persistent matadata store and invokes the callback function with the result.
+     * fetched from the persistent metadata store and invokes the callback function with the result.
      * The callback function must never throw - it is a fatal error to do so.
      *
      * If for some reason the asynchronous fetch operation cannot be dispatched (for example on
diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp
index b26df2d73ae..6b63846e592 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.cpp
+++ b/src/mongo/s/config_server_catalog_cache_loader.cpp
@@ -47,7 +47,7 @@ namespace {
  */
 ThreadPool::Options makeDefaultThreadPoolOptions() {
     ThreadPool::Options options;
-    options.poolName = "CatalogCacheLoader";
+    options.poolName = "ConfigServerCatalogCacheLoader";
     options.minThreads = 0;
     options.maxThreads = 6;
 
diff --git a/src/mongo/s/config_server_test_fixture.cpp b/src/mongo/s/config_server_test_fixture.cpp
index c5640333022..5bd8d4a9aa1 100644
--- a/src/mongo/s/config_server_test_fixture.cpp
+++ b/src/mongo/s/config_server_test_fixture.cpp
@@ -66,6 +66,7 @@
 #include "mongo/s/client/shard_local.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/client/shard_remote.h"
+#include "mongo/s/config_server_catalog_cache_loader.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/query/cluster_cursor_manager.h"
 #include "mongo/s/set_shard_version_request.h"
@@ -144,8 +145,14 @@ std::unique_ptr<ShardingCatalogManager> ConfigServerTestFixture::makeShardingCat
     return stdx::make_unique<ShardingCatalogManagerImpl>(std::move(specialExec));
 }
 
-std::unique_ptr<CatalogCache> ConfigServerTestFixture::makeCatalogCache() {
-    return stdx::make_unique<CatalogCache>();
+std::unique_ptr<CatalogCacheLoader> ConfigServerTestFixture::makeCatalogCacheLoader() {
+    return stdx::make_unique<ConfigServerCatalogCacheLoader>();
+}
+
+std::unique_ptr<CatalogCache> ConfigServerTestFixture::makeCatalogCache(
+    std::unique_ptr<CatalogCacheLoader> catalogCacheLoader) {
+    invariant(catalogCacheLoader);
+    return stdx::make_unique<CatalogCache>(std::move(catalogCacheLoader));
 }
 
 std::unique_ptr<BalancerConfiguration> ConfigServerTestFixture::makeBalancerConfiguration() {
diff --git a/src/mongo/s/config_server_test_fixture.h b/src/mongo/s/config_server_test_fixture.h
index 69d08afe17d..9eeb0fa7398 100644
--- a/src/mongo/s/config_server_test_fixture.h
+++ b/src/mongo/s/config_server_test_fixture.h
@@ -132,7 +132,10 @@ protected:
     std::unique_ptr<ShardingCatalogManager> makeShardingCatalogManager(
         ShardingCatalogClient* catalogClient) override;
 
-    std::unique_ptr<CatalogCache> makeCatalogCache() override;
+    std::unique_ptr<CatalogCacheLoader> makeCatalogCacheLoader() override;
+
+    std::unique_ptr<CatalogCache> makeCatalogCache(
+        std::unique_ptr<CatalogCacheLoader> catalogCacheLoader) override;
 
     std::unique_ptr<ClusterCursorManager> makeClusterCursorManager() override;
 
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index a88d123e535..7e4c84bd01f 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -67,6 +67,7 @@
 #include "mongo/s/balancer_configuration.h"
 #include "mongo/s/catalog/sharding_catalog_client.h"
 #include "mongo/s/catalog/sharding_catalog_manager.h"
+#include "mongo/s/catalog_cache.h"
 #include "mongo/s/client/shard_connection.h"
 #include "mongo/s/client/shard_factory.h"
 #include "mongo/s/client/shard_registry.h"
@@ -203,6 +204,7 @@ static Status initializeSharding(OperationContext* opCtx) {
         mongosGlobalParams.configdbs,
         generateDistLockProcessId(opCtx),
         std::move(shardFactory),
+        stdx::make_unique<CatalogCache>(),
         [opCtx]() {
             auto hookList = stdx::make_unique<rpc::EgressMetadataHookList>();
             hookList->addHook(
diff --git a/src/mongo/s/shard_server_test_fixture.cpp b/src/mongo/s/shard_server_test_fixture.cpp
index 272d6a00641..2e0396248e7 100644
--- a/src/mongo/s/shard_server_test_fixture.cpp
+++ b/src/mongo/s/shard_server_test_fixture.cpp
@@ -33,9 +33,11 @@
 #include "mongo/client/remote_command_targeter_mock.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/repl/replication_coordinator_mock.h"
+#include "mongo/db/s/shard_server_catalog_cache_loader.h"
 #include "mongo/s/catalog/dist_lock_catalog_mock.h"
 #include "mongo/s/catalog/dist_lock_manager_mock.h"
 #include "mongo/s/catalog/sharding_catalog_client_impl.h"
+#include "mongo/s/catalog_cache.h"
 #include "mongo/stdx/memory.h"
 
 namespace mongo {
@@ -106,4 +108,15 @@ std::unique_ptr<ShardingCatalogClient> ShardServerTestFixture::makeShardingCatal
     return stdx::make_unique<ShardingCatalogClientImpl>(std::move(distLockManager));
 }
 
+std::unique_ptr<CatalogCacheLoader> ShardServerTestFixture::makeCatalogCacheLoader() {
+    return stdx::make_unique<ShardServerCatalogCacheLoader>(
+        stdx::make_unique<ConfigServerCatalogCacheLoader>());
+}
+
+std::unique_ptr<CatalogCache> ShardServerTestFixture::makeCatalogCache(
+    std::unique_ptr<CatalogCacheLoader> catalogCacheLoader) {
+    invariant(catalogCacheLoader);
+    return stdx::make_unique<CatalogCache>(std::move(catalogCacheLoader));
+}
+
 }  // namespace mongo
diff --git a/src/mongo/s/shard_server_test_fixture.h b/src/mongo/s/shard_server_test_fixture.h
index e3e15a4d780..52e202536d3 100644
--- a/src/mongo/s/shard_server_test_fixture.h
+++ b/src/mongo/s/shard_server_test_fixture.h
@@ -83,6 +83,14 @@ protected:
      */
     std::unique_ptr<ShardingCatalogClient> makeShardingCatalogClient(
         std::unique_ptr<DistLockManager> distLockManager) override;
+
+    /**
+     * Creates a ShardServerCatalogCacheLoader.
+     */
+    std::unique_ptr<CatalogCacheLoader> makeCatalogCacheLoader();
+
+    std::unique_ptr<CatalogCache> makeCatalogCache(
+        std::unique_ptr<CatalogCacheLoader> catalogCacheLoader);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp
index 9220176f29e..570440f817b 100644
--- a/src/mongo/s/sharding_initialization.cpp
+++ b/src/mongo/s/sharding_initialization.cpp
@@ -160,6 +160,7 @@ Status initializeGlobalShardingState(OperationContext* opCtx,
                                      const ConnectionString& configCS,
                                      StringData distLockProcessId,
                                      std::unique_ptr<ShardFactory> shardFactory,
+                                     std::unique_ptr<CatalogCache> catalogCache,
                                      rpc::ShardingEgressMetadataHookBuilder hookBuilder,
                                      ShardingCatalogManagerBuilder catalogManagerBuilder) {
     if (configCS.type() == ConnectionString::INVALID) {
@@ -202,7 +203,7 @@ Status initializeGlobalShardingState(OperationContext* opCtx,
     grid.init(
         std::move(catalogClient),
         std::move(catalogManager),
-        stdx::make_unique<CatalogCache>(),
+        std::move(catalogCache),
         std::move(shardRegistry),
         stdx::make_unique<ClusterCursorManager>(getGlobalServiceContext()->getPreciseClockSource()),
         stdx::make_unique<BalancerConfiguration>(),
diff --git a/src/mongo/s/sharding_initialization.h b/src/mongo/s/sharding_initialization.h
index f2e3d12db8b..6d4629c7994 100644
--- a/src/mongo/s/sharding_initialization.h
+++ b/src/mongo/s/sharding_initialization.h
@@ -41,12 +41,14 @@ namespace executor {
 class TaskExecutor;
 }  // namespace executor
 
+class CatalogCache;
 class ConnectionString;
 class OperationContext;
 class ShardFactory;
 class Status;
 class ShardingCatalogClient;
 class ShardingCatalogManager;
+
 using ShardingCatalogManagerBuilder = stdx::function<std::unique_ptr<ShardingCatalogManager>(
     ShardingCatalogClient*, std::unique_ptr<executor::TaskExecutor>)>;
 
@@ -73,6 +75,7 @@ Status initializeGlobalShardingState(OperationContext* opCtx,
                                      const ConnectionString& configCS,
                                      StringData distLockProcessId,
                                      std::unique_ptr<ShardFactory> shardFactory,
+                                     std::unique_ptr<CatalogCache> catalogCache,
                                      rpc::ShardingEgressMetadataHookBuilder hookBuilder,
                                      ShardingCatalogManagerBuilder catalogManagerBuilder);
 
diff --git a/src/mongo/s/sharding_mongod_test_fixture.cpp b/src/mongo/s/sharding_mongod_test_fixture.cpp
index 8685b7ebc6d..5c4369f5fa3 100644
--- a/src/mongo/s/sharding_mongod_test_fixture.cpp
+++ b/src/mongo/s/sharding_mongod_test_fixture.cpp
@@ -65,6 +65,7 @@
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_shard.h"
 #include "mongo/s/catalog_cache.h"
+#include "mongo/s/catalog_cache_loader.h"
 #include "mongo/s/client/shard_factory.h"
 #include "mongo/s/client/shard_local.h"
 #include "mongo/s/client/shard_registry.h"
@@ -235,7 +236,12 @@ std::unique_ptr<ShardingCatalogManager> ShardingMongodTestFixture::makeShardingC
     return nullptr;
 }
 
-std::unique_ptr<CatalogCache> ShardingMongodTestFixture::makeCatalogCache() {
+std::unique_ptr<CatalogCacheLoader> ShardingMongodTestFixture::makeCatalogCacheLoader() {
+    return nullptr;
+}
+
+std::unique_ptr<CatalogCache> ShardingMongodTestFixture::makeCatalogCache(
+    std::unique_ptr<CatalogCacheLoader> catalogCacheLoader) {
     return nullptr;
 }
 
@@ -269,7 +275,9 @@ Status ShardingMongodTestFixture::initializeGlobalShardingStateForMongodForTest(
 
     auto catalogClientPtr = makeShardingCatalogClient(std::move(distLockManagerPtr));
     auto catalogManagerPtr = makeShardingCatalogManager(catalogClientPtr.get());
-    auto catalogCachePtr = makeCatalogCache();
+
+    auto catalogCacheLoaderPtr = makeCatalogCacheLoader();
+    auto catalogCachePtr = makeCatalogCache(std::move(catalogCacheLoaderPtr));
 
     auto clusterCursorManagerPtr = makeClusterCursorManager();
 
diff --git a/src/mongo/s/sharding_mongod_test_fixture.h b/src/mongo/s/sharding_mongod_test_fixture.h
index 62435c97e79..fca43abd45d 100644
--- a/src/mongo/s/sharding_mongod_test_fixture.h
+++ b/src/mongo/s/sharding_mongod_test_fixture.h
@@ -40,6 +40,7 @@ namespace mongo {
 
 class BalancerConfiguration;
 class CatalogCache;
+class CatalogCacheLoader;
 class ConnectionString;
 class ClusterCursorManager;
 class DistLockCatalog;
@@ -229,7 +230,13 @@ protected:
     /**
      * Base class returns nullptr.
      */
-    virtual std::unique_ptr<CatalogCache> makeCatalogCache();
+    virtual std::unique_ptr<CatalogCacheLoader> makeCatalogCacheLoader();
+
+    /**
+     * Base class returns nullptr.
+     */
+    virtual std::unique_ptr<CatalogCache> makeCatalogCache(
+        std::unique_ptr<CatalogCacheLoader> catalogCacheLoader);
 
     /**
      * Base class returns nullptr.