SERVER-33453 - Add timestamp support to the ChunkManager

7 files changed, 64 insertions, 18 deletions

diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp
index 120c0b3ee58..862188f5509 100644
--- a/src/mongo/db/s/collection_metadata_test.cpp
+++ b/src/mongo/db/s/collection_metadata_test.cpp
@@ -43,13 +43,17 @@ using unittest::assertGet;
 
 std::unique_ptr<CollectionMetadata> makeCollectionMetadataImpl(
     const KeyPattern& shardKeyPattern,
-    const std::vector<std::pair<BSONObj, BSONObj>>& thisShardsChunks) {
+    const std::vector<std::pair<BSONObj, BSONObj>>& thisShardsChunks,
+    bool staleChunkManager) {
 
     const OID epoch = OID::gen();
     const NamespaceString kNss("test.foo");
     const ShardId kThisShard("thisShard");
     const ShardId kOtherShard("otherShard");
 
+    const Timestamp kRouting(100, 0);
+    const Timestamp kChunkManager(staleChunkManager ? 99 : 100, 0);
+
     std::vector<ChunkType> allChunks;
     auto nextMinKey = shardKeyPattern.globalMin();
     ChunkVersion version{1, 0, epoch};
@@ -58,20 +62,25 @@ std::unique_ptr<CollectionMetadata> makeCollectionMetadataImpl(
             // Need to add a chunk to the other shard from nextMinKey to myNextChunk.first.
             allChunks.emplace_back(
                 kNss, ChunkRange{nextMinKey, myNextChunk.first}, version, kOtherShard);
+            allChunks.back().setHistory({ChunkHistory(kRouting, kOtherShard)});
             version.incMajor();
         }
         allChunks.emplace_back(
             kNss, ChunkRange{myNextChunk.first, myNextChunk.second}, version, kThisShard);
+        allChunks.back().setHistory({ChunkHistory(kRouting, kThisShard)});
         version.incMajor();
         nextMinKey = myNextChunk.second;
     }
     if (SimpleBSONObjComparator::kInstance.evaluate(nextMinKey < shardKeyPattern.globalMax())) {
         allChunks.emplace_back(
             kNss, ChunkRange{nextMinKey, shardKeyPattern.globalMax()}, version, kOtherShard);
+        allChunks.back().setHistory({ChunkHistory(kRouting, kOtherShard)});
     }
 
     UUID uuid(UUID::gen());
-    auto cm = ChunkManager::makeNew(kNss, uuid, shardKeyPattern, nullptr, false, epoch, allChunks);
+    auto rt =
+        RoutingTableHistory::makeNew(kNss, uuid, shardKeyPattern, nullptr, false, epoch, allChunks);
+    std::shared_ptr<ChunkManager> cm = std::make_shared<ChunkManager>(rt, kChunkManager);
     return stdx::make_unique<CollectionMetadata>(cm, kThisShard);
 }
 
@@ -83,7 +92,7 @@ struct ConstructedRangeMap : public RangeMap {
 class NoChunkFixture : public unittest::Test {
 protected:
     std::unique_ptr<CollectionMetadata> makeCollectionMetadata() const {
-        return makeCollectionMetadataImpl(KeyPattern(BSON("a" << 1)), {});
+        return makeCollectionMetadataImpl(KeyPattern(BSON("a" << 1)), {}, false);
     }
 };
 
@@ -168,8 +177,8 @@ TEST_F(NoChunkFixture, OrphanedDataRangeEnd) {
 class SingleChunkFixture : public unittest::Test {
 protected:
     std::unique_ptr<CollectionMetadata> makeCollectionMetadata() const {
-        return makeCollectionMetadataImpl(KeyPattern(BSON("a" << 1)),
-                                          {std::make_pair(BSON("a" << 10), BSON("a" << 20))});
+        return makeCollectionMetadataImpl(
+            KeyPattern(BSON("a" << 1)), {std::make_pair(BSON("a" << 10), BSON("a" << 20))}, false);
     }
 };
 
@@ -247,7 +256,8 @@ protected:
         const KeyPattern shardKeyPattern(BSON("a" << 1 << "b" << 1));
         return makeCollectionMetadataImpl(
             shardKeyPattern,
-            {std::make_pair(shardKeyPattern.globalMin(), shardKeyPattern.globalMax())});
+            {std::make_pair(shardKeyPattern.globalMin(), shardKeyPattern.globalMax())},
+            false);
     }
 };
 
@@ -271,7 +281,8 @@ protected:
         return makeCollectionMetadataImpl(
             KeyPattern(BSON("a" << 1 << "b" << 1)),
             {std::make_pair(BSON("a" << 10 << "b" << 0), BSON("a" << 20 << "b" << 0)),
-             std::make_pair(BSON("a" << 30 << "b" << 0), BSON("a" << 40 << "b" << 0))});
+             std::make_pair(BSON("a" << 30 << "b" << 0), BSON("a" << 40 << "b" << 0))},
+            false);
     }
 };
 
@@ -308,7 +319,8 @@ protected:
         return makeCollectionMetadataImpl(KeyPattern(BSON("a" << 1)),
                                           {std::make_pair(BSON("a" << MINKEY), BSON("a" << 10)),
                                            std::make_pair(BSON("a" << 10), BSON("a" << 20)),
-                                           std::make_pair(BSON("a" << 30), BSON("a" << MAXKEY))});
+                                           std::make_pair(BSON("a" << 30), BSON("a" << MAXKEY))},
+                                          false);
     }
 };
 
@@ -390,5 +402,27 @@ TEST_F(ThreeChunkWithRangeGapFixture, GetDifferentChunkFromLast) {
     ASSERT_EQUALS(0, differentChunk.getMax().woCompare(BSON("a" << 10)));
 }
 
+/**
+ * Fixture with single chunk containing:
+ * [10->20)
+ */
+class StaleChunkFixture : public unittest::Test {
+protected:
+    std::unique_ptr<CollectionMetadata> makeCollectionMetadata() const {
+        return makeCollectionMetadataImpl(
+            KeyPattern(BSON("a" << 1)), {std::make_pair(BSON("a" << 10), BSON("a" << 20))}, true);
+    }
+};
+
+TEST_F(StaleChunkFixture, KeyBelongsToMe) {
+    ASSERT_THROWS_CODE(makeCollectionMetadata()->keyBelongsToMe(BSON("a" << 10)),
+                       AssertionException,
+                       ErrorCodes::StaleChunkHistory);
+
+    ASSERT_THROWS_CODE(makeCollectionMetadata()->keyBelongsToMe(BSON("a" << 0)),
+                       AssertionException,
+                       ErrorCodes::StaleChunkHistory);
+}
+
 }  // namespace
 }  // namespace mongo
diff --git a/src/mongo/db/s/collection_range_deleter_test.cpp b/src/mongo/db/s/collection_range_deleter_test.cpp
index beddabc66de..393d20c667b 100644
--- a/src/mongo/db/s/collection_range_deleter_test.cpp
+++ b/src/mongo/db/s/collection_range_deleter_test.cpp
@@ -70,7 +70,7 @@ protected:
         client.createCollection(kNss.ns());
 
         const KeyPattern keyPattern(kShardKeyPattern);
-        auto cm = ChunkManager::makeNew(
+        auto rt = RoutingTableHistory::makeNew(
             kNss,
             UUID::gen(),
             keyPattern,
@@ -81,6 +81,7 @@ protected:
                        ChunkRange{keyPattern.globalMin(), keyPattern.globalMax()},
                        ChunkVersion(1, 0, epoch()),
                        ShardId("otherShard"))});
+        std::shared_ptr<ChunkManager> cm = std::make_shared<ChunkManager>(rt, Timestamp(100, 0));
 
         AutoGetCollection autoColl(operationContext(), kNss, MODE_IX);
         auto const css = CollectionShardingState::get(operationContext(), kNss);
diff --git a/src/mongo/db/s/collection_sharding_state_test.cpp b/src/mongo/db/s/collection_sharding_state_test.cpp
index d50d0b10d86..63db860c50d 100644
--- a/src/mongo/db/s/collection_sharding_state_test.cpp
+++ b/src/mongo/db/s/collection_sharding_state_test.cpp
@@ -132,8 +132,10 @@ std::unique_ptr<CollectionMetadata> makeAMetadata(BSONObj const& keyPattern) {
     const OID epoch = OID::gen();
     auto range = ChunkRange(BSON("key" << MINKEY), BSON("key" << MAXKEY));
     auto chunk = ChunkType(kTestNss, std::move(range), ChunkVersion(1, 0, epoch), ShardId("other"));
-    auto cm = ChunkManager::makeNew(
+    auto rt = RoutingTableHistory::makeNew(
         kTestNss, UUID::gen(), KeyPattern(keyPattern), nullptr, false, epoch, {std::move(chunk)});
+    std::shared_ptr<ChunkManager> cm = std::make_shared<ChunkManager>(rt, Timestamp(100, 0));
+
     return stdx::make_unique<CollectionMetadata>(std::move(cm), ShardId("this"));
 }
 
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
index c4fc7ca563e..50d25843d50 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp
@@ -691,6 +691,7 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration(
     // Generate the new versions of migratedChunk and controlChunk. Migrating chunk's minor version
     // will be 0.
     ChunkType newMigratedChunk = migratedChunk;
+    newMigratedChunk.setShard(toShard);
     newMigratedChunk.setVersion(ChunkVersion(
         currentCollectionVersion.majorVersion() + 1, 0, currentCollectionVersion.epoch()));
 
@@ -718,6 +719,9 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration(
                                   << validAfter.get().toString()};
         }
         newHistory.emplace(newHistory.begin(), ChunkHistory(validAfter.get(), toShard));
+    } else {
+        // TODO: SERVER-33781 FCV 3.6 should not have any history
+        newHistory.clear();
     }
     newMigratedChunk.setHistory(std::move(newHistory));
 
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_commit_chunk_migration_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_commit_chunk_migration_test.cpp
index dd1357e8b4a..5a817120c27 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_commit_chunk_migration_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_commit_chunk_migration_test.cpp
@@ -64,7 +64,7 @@ TEST_F(CommitChunkMigrate, CheckCorrectOpsCommandWithCtl) {
     chunk0.setNS(kNamespace);
     chunk0.setVersion(origVersion);
     chunk0.setShard(shard0.getName());
-    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX"))});
+    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), shard0.getName())});
 
     // apportion
     auto chunkMin = BSON("a" << 1);
@@ -143,7 +143,7 @@ TEST_F(CommitChunkMigrate, CheckCorrectOpsCommandNoCtl) {
     chunk0.setNS(kNamespace);
     chunk0.setVersion(origVersion);
     chunk0.setShard(shard0.getName());
-    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX"))});
+    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), shard0.getName())});
 
     // apportion
     auto chunkMin = BSON("a" << 1);
@@ -204,7 +204,7 @@ TEST_F(CommitChunkMigrate, CheckCorrectOpsCommandNoCtlTrimHistory) {
     chunk0.setNS(kNamespace);
     chunk0.setVersion(origVersion);
     chunk0.setShard(shard0.getName());
-    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX"))});
+    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), shard0.getName())});
 
     // apportion
     auto chunkMin = BSON("a" << 1);
@@ -266,7 +266,7 @@ TEST_F(CommitChunkMigrate, RejectOutOfOrderHistory) {
     chunk0.setNS(kNamespace);
     chunk0.setVersion(origVersion);
     chunk0.setShard(shard0.getName());
-    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX"))});
+    chunk0.setHistory({ChunkHistory(Timestamp(100, 0), shard0.getName())});
 
     // apportion
     auto chunkMin = BSON("a" << 1);
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp
index 932bc374745..336f1139e4e 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_split_chunk_test.cpp
@@ -53,7 +53,7 @@ TEST_F(SplitChunkTest, SplitExistingChunkCorrectlyShouldSucceed) {
     auto chunkMax = BSON("a" << 10);
     chunk.setMin(chunkMin);
     chunk.setMax(chunkMax);
-    chunk.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX")),
+    chunk.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shard0000")),
                       ChunkHistory(Timestamp(90, 0), ShardId("shardY"))});
 
     auto chunkSplitPoint = BSON("a" << 5);
@@ -113,7 +113,7 @@ TEST_F(SplitChunkTest, MultipleSplitsOnExistingChunkShouldSucceed) {
     auto chunkMax = BSON("a" << 10);
     chunk.setMin(chunkMin);
     chunk.setMax(chunkMax);
-    chunk.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shardX")),
+    chunk.setHistory({ChunkHistory(Timestamp(100, 0), ShardId("shard0000")),
                       ChunkHistory(Timestamp(90, 0), ShardId("shardY"))});
 
     auto chunkSplitPoint = BSON("a" << 5);
diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp
index 9acd513a5ab..7a260c7f08f 100644
--- a/src/mongo/db/s/metadata_manager_test.cpp
+++ b/src/mongo/db/s/metadata_manager_test.cpp
@@ -71,7 +71,7 @@ protected:
     static std::unique_ptr<CollectionMetadata> makeEmptyMetadata() {
         const OID epoch = OID::gen();
 
-        auto cm = ChunkManager::makeNew(
+        auto rt = RoutingTableHistory::makeNew(
             kNss,
             UUID::gen(),
             kShardKeyPattern,
@@ -82,6 +82,9 @@ protected:
                        ChunkRange{BSON(kPattern << MINKEY), BSON(kPattern << MAXKEY)},
                        ChunkVersion(1, 0, epoch),
                        kOtherShard}});
+
+        std::shared_ptr<ChunkManager> cm = std::make_shared<ChunkManager>(rt, Timestamp(100, 0));
+
         return stdx::make_unique<CollectionMetadata>(cm, kThisShard);
     }
 
@@ -110,10 +113,12 @@ protected:
         v2.incMajor();
         auto v3 = v2;
         v3.incMajor();
-        cm = cm->makeUpdated(
+        auto rt = cm->getRoutingHistory().makeUpdated(
             {ChunkType{kNss, ChunkRange{chunkToSplit->getMin(), minKey}, v1, kOtherShard},
              ChunkType{kNss, ChunkRange{minKey, maxKey}, v2, kThisShard},
              ChunkType{kNss, ChunkRange{maxKey, chunkToSplit->getMax()}, v3, kOtherShard}});
+        cm = std::make_shared<ChunkManager>(rt, Timestamp(100, 0));
+
         return stdx::make_unique<CollectionMetadata>(cm, kThisShard);
     }