SERVER-29760 propagate UUID from primary shard to config server on shardCollection

9 files changed, 168 insertions, 5 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
index e68f17fa28d..e38582486f3 100644
--- a/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_last_stable_mongos_and_mixed_shards.yml
@@ -39,6 +39,7 @@ selector:
   - jstests/sharding/session_info_in_oplog.js
   - jstests/sharding/enable_sharding_basic.js
   - jstests/sharding/retryable_writes.js
+  - jstests/sharding/uuid_propagated_to_config_server_on_shardCollection.js
   # New feature in v3.6 mongo shell.
   - jstests/sharding/causal_consistency_shell_support.js
   - jstests/sharding/keys_rotation_interval_sec.js
diff --git a/jstests/libs/uuid_util.js b/jstests/libs/uuid_util.js
new file mode 100644
index 00000000000..65ec55e7817
--- /dev/null
+++ b/jstests/libs/uuid_util.js
@@ -0,0 +1,25 @@
+//  Utilities for testing UUIDs.
+
+/**
+ * Reads the collection entry for 'nss' from config.collections, asserts that such an entry exists,
+ * and returns its 'uuid' field, which may be undefined.
+ */
+function getUUIDFromConfigCollections(mongosConn, nss) {
+    let collEntry = mongosConn.getDB("config").collections.findOne({_id: nss});
+    assert.neq(undefined, collEntry);
+    return collEntry.uuid;
+}
+
+/**
+ * Calls listCollections on a connection to 'db' with a filter for 'collName', asserts that a result
+ * for 'collName' was returned, and returns its 'uuid' field, which may be undefined.
+ */
+function getUUIDFromListCollections(db, collName) {
+    let listCollsRes = db.runCommand({listCollections: 1, filter: {name: collName}});
+    assert.commandWorked(listCollsRes);
+    assert.neq(undefined, listCollsRes.cursor);
+    assert.neq(undefined, listCollsRes.cursor.firstBatch);
+    assert.eq(1, listCollsRes.cursor.firstBatch.length);
+    assert.neq(undefined, listCollsRes.cursor.firstBatch[0].info);
+    return listCollsRes.cursor.firstBatch[0].info.uuid;
+}
diff --git a/jstests/sharding/shard1.js b/jstests/sharding/shard1.js
index 3b97bbc0306..baacbd96ea4 100644
--- a/jstests/sharding/shard1.js
+++ b/jstests/sharding/shard1.js
@@ -39,6 +39,7 @@ assert(cconfig, "why no collection entry for test.foo");
 delete cconfig.lastmod;
 delete cconfig.dropped;
 delete cconfig.lastmodEpoch;
+delete cconfig.uuid;
 
 assert.eq(cconfig, {_id: "test.foo", key: {num: 1}, unique: false}, "Sharded content mismatch");
 
diff --git a/jstests/sharding/uuid_propagated_to_config_server_on_shardCollection.js b/jstests/sharding/uuid_propagated_to_config_server_on_shardCollection.js
new file mode 100644
index 00000000000..d8dffb2f455
--- /dev/null
+++ b/jstests/sharding/uuid_propagated_to_config_server_on_shardCollection.js
@@ -0,0 +1,63 @@
+/**
+ * 1) Tests that in fcv=3.4, the shardCollection command does not persist a UUID in
+ * config.collections.
+ *
+ * 2) Tests that in fcv=3.6, the shardCollection command obtains the collection's UUID from the
+ * primary shard and persists it in config.collections.
+ */
+(function() {
+
+    load('jstests/libs/uuid_util.js');
+
+    let db = "test";
+
+    let st = new ShardingTest({shards: {rs0: {nodes: 1}, rs1: {nodes: 1}}, other: {config: 3}});
+
+    assert.commandWorked(st.s.adminCommand({enableSharding: db}));
+    st.ensurePrimaryShard(db, st.shard0.shardName);
+
+    // Check that in fcv=3.4, shardCollection does not persist a UUID.
+    assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "3.4"}));
+    for (let i = 0; i < 3; i++) {
+        let coll = "foo" + i;
+        let nss = db + "." + coll;
+
+        // It shouldn't matter whether the collection existed on the shard already or not; test
+        // both cases.
+        if (i === 0) {
+            assert.writeOK(st.s.getDB(db).getCollection(coll).insert({x: 1}));
+        }
+
+        assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: 1}}));
+
+        // Check that the entry in config.collections does not have a uuid field.
+        assert.eq(undefined, getUUIDFromConfigCollections(st.s, nss));
+    }
+
+    // Check that in fcv=3.6, shardCollection propagates and persists UUIDs.
+    assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: "3.6"}));
+    for (let i = 0; i < 3; i++) {
+        // Use different collection names because the ones starting with 'foo' are already sharded
+        // and will have a UUID generated for them on setFCV=3.6.
+        let coll = "bar" + i;
+        let nss = db + "." + coll;
+
+        // It shouldn't matter whether the collection existed on the shard already or not; test
+        // both cases.
+        if (i === 0) {
+            assert.writeOK(st.s.getDB(db).getCollection(coll).insert({x: 1}));
+        }
+
+        assert.commandWorked(st.s.adminCommand({shardCollection: nss, key: {_id: 1}}));
+
+        // Check that the entry for the collection in config.collections has a uuid field.
+        let collEntryUUID = getUUIDFromConfigCollections(st.s, nss);
+        assert.neq(undefined, collEntryUUID);
+
+        // Check that the uuid field in the config.collections entry matches the uuid on the shard.
+        let listCollsUUID = getUUIDFromListCollections(st.shard0.getDB(db), coll);
+        assert.neq(undefined, listCollsUUID);
+        assert.eq(listCollsUUID, collEntryUUID);
+    }
+
+})();
diff --git a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
index 65112eec4ff..b2edd4da4c3 100644
--- a/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
+++ b/src/mongo/db/s/config/configsvr_shard_collection_command.cpp
@@ -629,6 +629,55 @@ void migrateAndFurtherSplitInitialChunks(OperationContext* opCtx,
     }
 }
 
+boost::optional<UUID> getUUIDFromPrimaryShard(const NamespaceString& nss,
+                                              ScopedDbConnection& conn) {
+    // UUIDs were introduced in featureCompatibilityVersion 3.6.
+    if (serverGlobalParams.featureCompatibility.version.load() <
+        ServerGlobalParams::FeatureCompatibility::Version::k36) {
+        return boost::none;
+    }
+
+    // Obtain the collection's UUID from the primary shard's listCollections response.
+    BSONObj res;
+    {
+        std::list<BSONObj> all =
+            conn->getCollectionInfos(nss.db().toString(), BSON("name" << nss.coll()));
+        if (!all.empty()) {
+            res = all.front().getOwned();
+        }
+    }
+
+    uassert(ErrorCodes::InternalError,
+            str::stream() << "expected the primary shard host " << conn.getHost()
+                          << " for database "
+                          << nss.db()
+                          << " to return an entry for "
+                          << nss.ns()
+                          << " in its listCollections response, but it did not",
+            !res.isEmpty());
+
+    BSONObj collectionInfo;
+    if (res["info"].type() == BSONType::Object) {
+        collectionInfo = res["info"].Obj();
+    }
+
+    uassert(ErrorCodes::InternalError,
+            str::stream() << "expected primary shard to return 'info' field as part of "
+                             "listCollections for "
+                          << nss.ns()
+                          << " because the cluster is in featureCompatibilityVersion=3.6, but got "
+                          << res,
+            !collectionInfo.isEmpty());
+
+    uassert(ErrorCodes::InternalError,
+            str::stream() << "expected primary shard to return a UUID for collection " << nss.ns()
+                          << " as part of 'info' field but got "
+                          << res,
+            collectionInfo.hasField("uuid"));
+
+    return uassertStatusOK(UUID::parse(collectionInfo["uuid"]));
+}
+
 /**
  * Internal sharding command run on config servers to add a shard to the cluster.
  */
@@ -728,10 +777,13 @@ public:
         validateShardKeyAgainstExistingIndexes(
             opCtx, nss, proposedKey, shardKeyPattern, primaryShard, conn, request);
 
+        // Step 4.
+        auto uuid = getUUIDFromPrimaryShard(nss, conn);
+
         // isEmpty is used by multiple steps below.
         bool isEmpty = (conn->count(nss.ns()) == 0);
 
-        // Step 4.
+        // Step 5.
         std::vector<BSONObj> initSplits;  // there will be at most numShards-1 of these
         std::vector<BSONObj> allSplits;   // all of the initial desired split points
         determinePresplittingPoints(opCtx,
@@ -753,9 +805,10 @@ public:
         // (below) if using a hashed shard key.
         const bool distributeInitialChunks = request.getInitialSplitPoints().is_initialized();
 
-        // Step 5. Actually shard the collection.
+        // Step 6. Actually shard the collection.
         catalogManager->shardCollection(opCtx,
                                         nss.ns(),
+                                        uuid,
                                         shardKeyPattern,
                                         *request.getCollation(),
                                         request.getUnique(),
@@ -770,7 +823,7 @@ public:
         // proceed.
         scopedDistLock.reset();
 
-        // Step 6. Migrate initial chunks to distribute them across shards.
+        // Step 7. Migrate initial chunks to distribute them across shards.
         migrateAndFurtherSplitInitialChunks(
             opCtx, nss, numShards, shardIds, isEmpty, shardKeyPattern, allSplits);
 
diff --git a/src/mongo/s/catalog/sharding_catalog_manager.h b/src/mongo/s/catalog/sharding_catalog_manager.h
index 254a894aaa4..8d54f47c7d0 100644
--- a/src/mongo/s/catalog/sharding_catalog_manager.h
+++ b/src/mongo/s/catalog/sharding_catalog_manager.h
@@ -43,6 +43,7 @@ namespace mongo {
 class OperationContext;
 class RemoteCommandTargeter;
 class ServiceContext;
+class UUID;
 
 /**
  * Implements modifications to the sharding catalog metadata.
@@ -199,6 +200,7 @@ public:
      * Shards a collection. Assumes that the database is enabled for sharding.
      *
      * @param ns: namespace of collection to shard
+     * @param uuid: the collection's UUID. Optional because new in 3.6.
      * @param fieldsAndOrder: shardKey pattern
      * @param defaultCollation: the default collation for the collection, to be written to
      *     config.collections. If empty, the collection default collation is simple binary
@@ -211,6 +213,7 @@ public:
      */
     void shardCollection(OperationContext* opCtx,
                          const std::string& ns,
+                         const boost::optional<UUID> uuid,
                          const ShardKeyPattern& fieldsAndOrder,
                          const BSONObj& defaultCollation,
                          bool unique,
diff --git a/src/mongo/s/catalog/sharding_catalog_manager_collection_operations.cpp b/src/mongo/s/catalog/sharding_catalog_manager_collection_operations.cpp
index dbaeee4b209..017ca2f8481 100644
--- a/src/mongo/s/catalog/sharding_catalog_manager_collection_operations.cpp
+++ b/src/mongo/s/catalog/sharding_catalog_manager_collection_operations.cpp
@@ -216,6 +216,7 @@ void checkForExistingChunks(OperationContext* opCtx, const string& ns) {
 
 void ShardingCatalogManager::shardCollection(OperationContext* opCtx,
                                              const string& ns,
+                                             const boost::optional<UUID> uuid,
                                              const ShardKeyPattern& fieldsAndOrder,
                                              const BSONObj& defaultCollation,
                                              bool unique,
@@ -236,6 +237,9 @@ void ShardingCatalogManager::shardCollection(OperationContext* opCtx,
         BSONObjBuilder collectionDetail;
         collectionDetail.append("shardKey", fieldsAndOrder.toBSON());
         collectionDetail.append("collection", ns);
+        if (uuid) {
+            uuid->appendToBuilder(&collectionDetail, "uuid");
+        }
         collectionDetail.append("primary", primaryShard->toString());
         collectionDetail.append("numChunks", static_cast<int>(initPoints.size() + 1));
         catalogClient
@@ -262,6 +266,9 @@ void ShardingCatalogManager::shardCollection(OperationContext* opCtx,
     {
         CollectionType coll;
         coll.setNs(nss);
+        if (uuid) {
+            coll.setUUID(*uuid);
+        }
         coll.setEpoch(collVersion.epoch());
 
         // TODO(schwerin): The following isn't really a date, but is stored as one in-memory and in
diff --git a/src/mongo/s/catalog/sharding_catalog_shard_collection_test.cpp b/src/mongo/s/catalog/sharding_catalog_shard_collection_test.cpp
index d2b0f3d1446..ae0ea1b4b56 100644
--- a/src/mongo/s/catalog/sharding_catalog_shard_collection_test.cpp
+++ b/src/mongo/s/catalog/sharding_catalog_shard_collection_test.cpp
@@ -135,6 +135,7 @@ TEST_F(ShardCollectionTest, anotherMongosSharding) {
     ASSERT_THROWS_CODE(ShardingCatalogManager::get(operationContext())
                            ->shardCollection(operationContext(),
                                              nss.ns(),
+                                             boost::none,  // UUID
                                              shardKeyPattern,
                                              defaultCollation,
                                              false,
@@ -174,6 +175,7 @@ TEST_F(ShardCollectionTest, noInitialChunksOrData) {
         ShardingCatalogManager::get(operationContext())
             ->shardCollection(opCtx.get(),
                               nss.ns(),
+                              boost::none,  // UUID
                               shardKeyPattern,
                               defaultCollation,
                               false,
@@ -302,6 +304,7 @@ TEST_F(ShardCollectionTest, withInitialChunks) {
         ShardingCatalogManager::get(operationContext())
             ->shardCollection(opCtx.get(),
                               ns,
+                              boost::none,  // UUID
                               keyPattern,
                               defaultCollation,
                               true,
@@ -398,8 +401,14 @@ TEST_F(ShardCollectionTest, withInitialData) {
         Client::initThreadIfNotAlready("Test");
         auto opCtx = cc().makeOperationContext();
         ShardingCatalogManager::get(operationContext())
-            ->shardCollection(
-                opCtx.get(), ns, keyPattern, defaultCollation, false, vector<BSONObj>{}, false);
+            ->shardCollection(opCtx.get(),
+                              ns,
+                              boost::none,  // UUID
+                              keyPattern,
+                              defaultCollation,
+                              false,
+                              vector<BSONObj>{},
+                              false);
     });
 
     // Report that documents exist for the given collection on the primary shard, so that calling
diff --git a/src/mongo/util/uuid.h b/src/mongo/util/uuid.h
index 09a6751fc1f..d8f75d0d7ce 100644
--- a/src/mongo/util/uuid.h
+++ b/src/mongo/util/uuid.h
@@ -53,6 +53,7 @@ class UUID {
     using UUIDStorage = std::array<unsigned char, 16>;
 
     // Make the IDL generated parser a friend
+    friend class ConfigsvrShardCollection;
     friend class One_UUID;
     friend class LogicalSessionId;
     friend class LogicalSessionToClient;