SERVER-71615 Add top-level `onCurrentShardSince` field to `config.chunk` entries

3 files changed, 89 insertions, 14 deletions

diff --git a/src/mongo/s/catalog/type_chunk.cpp b/src/mongo/s/catalog/type_chunk.cpp
index b330697db0d..1d07942ecfd 100644
--- a/src/mongo/s/catalog/type_chunk.cpp
+++ b/src/mongo/s/catalog/type_chunk.cpp
@@ -60,6 +60,7 @@ const BSONField<bool> ChunkType::jumbo("jumbo");
 const BSONField<Date_t> ChunkType::lastmod("lastmod");
 const BSONField<BSONObj> ChunkType::history("history");
 const BSONField<int64_t> ChunkType::estimatedSizeBytes("estimatedDataSizeBytes");
+const BSONField<Timestamp> ChunkType::onCurrentShardSince("onCurrentShardSince");
 const BSONField<bool> ChunkType::historyIsAt40("historyIsAt40");
 
 namespace {
@@ -238,6 +239,14 @@ StatusWith<ChunkType> ChunkType::_parseChunkBase(const BSONObj& source) {
     }
 
     {
+        Timestamp onCurrentShardSinceValue;
+        Status status = bsonExtractTimestampField(
+            source, onCurrentShardSince.name(), &onCurrentShardSinceValue);
+        chunk._onCurrentShardSince =
+            (status.isOK() ? (boost::optional<Timestamp>)onCurrentShardSinceValue : boost::none);
+    }
+
+    {
         BSONElement historyObj;
         Status status = bsonExtractTypedField(source, history.name(), Array, &historyObj);
         if (status.isOK()) {
@@ -246,6 +255,8 @@ StatusWith<ChunkType> ChunkType::_parseChunkBase(const BSONObj& source) {
                 return history.getStatus();
 
             chunk._history = std::move(history.getValue());
+            dassert(!chunk._onCurrentShardSince.has_value() ||
+                    *chunk._onCurrentShardSince == chunk._history.front().getValidAfter());
         } else if (status == ErrorCodes::NoSuchKey) {
             // History is missing, so it will be presumed empty
         } else {
@@ -470,6 +481,11 @@ BSONObj ChunkType::toConfigBSON() const {
                              static_cast<long long>(*_estimatedSizeBytes));
     if (_jumbo)
         builder.append(jumbo.name(), getJumbo());
+    if (_onCurrentShardSince) {
+        dassert(!_history.empty());
+        dassert(_history.front().getValidAfter() == *_onCurrentShardSince);
+        builder.append(onCurrentShardSince.name(), *_onCurrentShardSince);
+    }
     addHistoryToBSON(builder);
     return builder.obj();
 }
@@ -484,6 +500,8 @@ BSONObj ChunkType::toShardBSON() const {
     builder.append(max.name(), getMax());
     builder.append(shard.name(), getShard().toString());
     builder.appendTimestamp(lastmod.name(), _version->toLong());
+    if (_onCurrentShardSince)
+        builder.append(onCurrentShardSince.name(), *_onCurrentShardSince);
     addHistoryToBSON(builder);
     return builder.obj();
 }
@@ -532,6 +550,10 @@ void ChunkType::setJumbo(bool jumbo) {
     _jumbo = jumbo;
 }
 
+void ChunkType::setOnCurrentShardSince(const Timestamp& onCurrentShardSince) {
+    _onCurrentShardSince = onCurrentShardSince;
+}
+
 void ChunkType::addHistoryToBSON(BSONObjBuilder& builder) const {
     if (_history.size()) {
         BSONArrayBuilder arrayBuilder(builder.subarrayStart(history.name()));
@@ -590,6 +612,14 @@ Status ChunkType::validate() const {
                     str::stream() << "History contains an invalid shard "
                                   << _history.front().getShard()};
         }
+        if (_onCurrentShardSince.has_value() &&
+            _history.front().getValidAfter() != *_onCurrentShardSince) {
+            return {ErrorCodes::BadValue,
+                    str::stream() << "The first `validAfter` in the chunk's `history` is not "
+                                     "consistent with `onCurrentShardSince`: validAfter is "
+                                  << _history.front().getValidAfter()
+                                  << " while onCurrentShardSince is " << *_onCurrentShardSince};
+        }
     }
 
     return Status::OK();
diff --git a/src/mongo/s/catalog/type_chunk.h b/src/mongo/s/catalog/type_chunk.h
index c2797112686..06768cc40cb 100644
--- a/src/mongo/s/catalog/type_chunk.h
+++ b/src/mongo/s/catalog/type_chunk.h
@@ -214,6 +214,7 @@ public:
     static const BSONField<Date_t> lastmod;
     static const BSONField<BSONObj> history;
     static const BSONField<int64_t> estimatedSizeBytes;
+    static const BSONField<Timestamp> onCurrentShardSince;
     static const BSONField<bool> historyIsAt40;
 
     ChunkType();
@@ -222,14 +223,14 @@ public:
     /**
      * Constructs a new ChunkType object from BSON with the following format:
      * {min: <>, max: <>, shard: <>, uuid: <>, history: <>, jumbo: <>, lastmod: <>,
-     * lastmodEpoch: <>, lastmodTimestamp: <>}
+     * lastmodEpoch: <>, lastmodTimestamp: <>, onCurrentShardSince: <>}
      */
     static StatusWith<ChunkType> parseFromNetworkRequest(const BSONObj& source);
 
     /**
      * Constructs a new ChunkType object from BSON with the following format:
      * {_id: <>, min: <>, max: <>, shard: <>, uuid: <>, history: <>, jumbo: <>, lastmod: <>,
-     * estimatedSizeByte: <>}
+     * estimatedSizeByte: <>, onCurrentShardSince: <>}
      *
      * Returns ErrorCodes::NoSuchKey if the '_id' field is missing
      */
@@ -239,7 +240,7 @@ public:
 
     /**
      * Constructs a new ChunkType object from BSON with the following format:
-     * {_id: <>, max: <>, shard: <>, history: <>, lastmod: <>}
+     * {_id: <>, max: <>, shard: <>, history: <>, lastmod: <>, onCurrentShardSince: <>}
      * Also does validation of the contents.
      */
     static StatusWith<ChunkType> parseFromShardBSON(const BSONObj& source,
@@ -308,6 +309,11 @@ public:
     }
     void setJumbo(bool jumbo);
 
+    const boost::optional<Timestamp>& getOnCurrentShardSince() const {
+        return _onCurrentShardSince;
+    }
+    void setOnCurrentShardSince(const Timestamp& onCurrentShardSince);
+
     void setHistory(std::vector<ChunkHistory> history) {
         _history = std::move(history);
         if (!_history.empty()) {
@@ -334,7 +340,7 @@ public:
 private:
     /**
      * Parses the base chunk data on all usages:
-     * {history: <>, shard: <>}
+     * {history: <>, shard: <>, onCurrentShardSince: <>}
      */
     static StatusWith<ChunkType> _parseChunkBase(const BSONObj& source);
 
@@ -357,6 +363,8 @@ private:
     boost::optional<int64_t> _estimatedSizeBytes;
     // (O)(C)     too big to move?
     boost::optional<bool> _jumbo;
+    // (M)(C)(S)  timestamp since this chunk belongs to the current shard
+    boost::optional<Timestamp> _onCurrentShardSince;
     // history of the chunk
     std::vector<ChunkHistory> _history;
 };
diff --git a/src/mongo/s/catalog/type_chunk_test.cpp b/src/mongo/s/catalog/type_chunk_test.cpp
index 18c199b69ea..b7ad47a97fb 100644
--- a/src/mongo/s/catalog/type_chunk_test.cpp
+++ b/src/mongo/s/catalog/type_chunk_test.cpp
@@ -134,12 +134,17 @@ TEST(ChunkType, MinAndMaxShardKeysDifferInNumberOfKeys) {
     const auto collTimestamp = Timestamp(1);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2);
     BSONObj obj = BSON(
         ChunkType::name(OID::gen())
         << ChunkType::collectionUUID() << collUuid << ChunkType::min(BSON("a" << 10 << "b" << 10))
         << ChunkType::max(BSON("a" << 20)) << "lastmod" << Timestamp(chunkVersion.toLong())
         << "lastmodEpoch" << chunkVersion.epoch() << "lastmodTimestamp"
-        << chunkVersion.getTimestamp() << ChunkType::shard("shard0001"));
+        << chunkVersion.getTimestamp() << ChunkType::shard("shard0001")
+        << ChunkType::onCurrentShardSince() << onCurrentShardSince << ChunkType::history()
+        << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                           << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                           << "shard0001")));
     StatusWith<ChunkType> chunkRes = ChunkType::parseFromConfigBSON(obj, collEpoch, collTimestamp);
     ASSERT_OK(chunkRes.getStatus());
     ASSERT_FALSE(chunkRes.getValue().validate().isOK());
@@ -151,12 +156,17 @@ TEST(ChunkType, MinAndMaxShardKeysDifferInKeyNames) {
     const auto collTimestamp = Timestamp(1);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2);
     BSONObj obj =
         BSON(ChunkType::name(OID::gen())
              << ChunkType::collectionUUID() << collUuid << ChunkType::min(BSON("a" << 10))
              << ChunkType::max(BSON("b" << 20)) << "lastmod" << Timestamp(chunkVersion.toLong())
              << "lastmodEpoch" << chunkVersion.epoch() << "lastmodTimestamp"
-             << chunkVersion.getTimestamp() << ChunkType::shard("shard0001"));
+             << chunkVersion.getTimestamp() << ChunkType::shard("shard0001")
+             << ChunkType::onCurrentShardSince() << onCurrentShardSince << ChunkType::history()
+             << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                                << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                                << "shard0001")));
     StatusWith<ChunkType> chunkRes = ChunkType::parseFromConfigBSON(obj, collEpoch, collTimestamp);
     ASSERT_OK(chunkRes.getStatus());
     ASSERT_FALSE(chunkRes.getValue().validate().isOK());
@@ -168,11 +178,16 @@ TEST(ChunkType, MinToMaxNotAscending) {
     const auto collTimestamp = Timestamp(1);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2);
     BSONObj obj =
         BSON(ChunkType::name(OID::gen())
              << ChunkType::collectionUUID() << collUuid << ChunkType::min(BSON("a" << 20))
              << ChunkType::max(BSON("a" << 10)) << "lastmod" << Timestamp(chunkVersion.toLong())
-             << "lastmodEpoch" << chunkVersion.epoch() << ChunkType::shard("shard0001"));
+             << "lastmodEpoch" << chunkVersion.epoch() << ChunkType::shard("shard0001")
+             << ChunkType::onCurrentShardSince() << onCurrentShardSince << ChunkType::history()
+             << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                                << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                                << "shard0001")));
     StatusWith<ChunkType> chunkRes = ChunkType::parseFromConfigBSON(obj, collEpoch, collTimestamp);
     ASSERT_EQ(ErrorCodes::FailedToParse, chunkRes.getStatus());
 }
@@ -183,11 +198,17 @@ TEST(ChunkType, ToFromConfigBSON) {
     const auto collTimestamp = Timestamp(1);
 
     const auto chunkID = OID::gen();
+    const auto onCurrentShardSince = Timestamp(4);
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
-    BSONObj obj = BSON(ChunkType::name(chunkID)
-                       << ChunkType::collectionUUID() << collUuid << ChunkType::min(BSON("a" << 10))
-                       << ChunkType::max(BSON("a" << 20)) << ChunkType::shard("shard0001")
-                       << "lastmod" << Timestamp(chunkVersion.toLong()));
+    BSONObj obj =
+        BSON(ChunkType::name(chunkID)
+             << ChunkType::collectionUUID() << collUuid << ChunkType::min(BSON("a" << 10))
+             << ChunkType::max(BSON("a" << 20)) << ChunkType::shard("shard0001") << "lastmod"
+             << Timestamp(chunkVersion.toLong()) << ChunkType::onCurrentShardSince()
+             << onCurrentShardSince << ChunkType::history()
+             << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                                << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                                << "shard0001")));
     StatusWith<ChunkType> chunkRes = ChunkType::parseFromConfigBSON(obj, collEpoch, collTimestamp);
     ASSERT_OK(chunkRes.getStatus());
     ChunkType chunk = chunkRes.getValue();
@@ -201,6 +222,7 @@ TEST(ChunkType, ToFromConfigBSON) {
     ASSERT_EQUALS(chunk.getVersion().toLong(), chunkVersion.toLong());
     ASSERT_EQUALS(chunk.getVersion().epoch(), chunkVersion.epoch());
     ASSERT_EQUALS(chunk.getShard(), "shard0001");
+    ASSERT_EQUALS(*chunk.getOnCurrentShardSince(), onCurrentShardSince);
     ASSERT_OK(chunk.validate());
 }
 
@@ -219,6 +241,7 @@ TEST(ChunkType, BothNsAndUUID) {
     const auto collTimestamp = Timestamp(1);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2);
 
     BSONObj objModNS =
         BSON(ChunkType::name(OID::gen())
@@ -226,7 +249,11 @@ TEST(ChunkType, BothNsAndUUID) {
              << mongo::UUID::gen() << ChunkType::min(BSON("a" << 10 << "b" << 10))
              << ChunkType::max(BSON("a" << 20)) << "lastmod" << Timestamp(chunkVersion.toLong())
              << "lastmodEpoch" << chunkVersion.epoch() << "lastmodTimestamp"
-             << chunkVersion.getTimestamp() << ChunkType::shard("shard0001"));
+             << chunkVersion.getTimestamp() << ChunkType::shard("shard0001")
+             << ChunkType::onCurrentShardSince() << onCurrentShardSince << ChunkType::history()
+             << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                                << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                                << "shard0001")));
     StatusWith<ChunkType> chunkRes =
         ChunkType::parseFromConfigBSON(objModNS, collEpoch, collTimestamp);
     ASSERT_TRUE(chunkRes.isOK());
@@ -237,13 +264,18 @@ TEST(ChunkType, UUIDPresentAndNsMissing) {
     const auto collTimestamp = Timestamp(1);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2);
 
     BSONObj objModNS = BSON(
         ChunkType::name(OID::gen())
         << ChunkType::collectionUUID() << mongo::UUID::gen()
         << ChunkType::min(BSON("a" << 10 << "b" << 10)) << ChunkType::max(BSON("a" << 20))
         << "lastmod" << Timestamp(chunkVersion.toLong()) << "lastmodEpoch" << chunkVersion.epoch()
-        << "lastmodTimestamp" << chunkVersion.getTimestamp() << ChunkType::shard("shard0001"));
+        << "lastmodTimestamp" << chunkVersion.getTimestamp() << ChunkType::shard("shard0001")
+        << ChunkType::onCurrentShardSince() << onCurrentShardSince << ChunkType::history()
+        << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                           << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                           << "shard0001")));
     StatusWith<ChunkType> chunkRes =
         ChunkType::parseFromConfigBSON(objModNS, collEpoch, collTimestamp);
     ASSERT_TRUE(chunkRes.isOK());
@@ -254,6 +286,7 @@ TEST(ChunkType, ParseFromNetworkRequest) {
     const auto collTimestamp = Timestamp(1, 0);
 
     ChunkVersion chunkVersion({collEpoch, collTimestamp}, {1, 2});
+    const auto onCurrentShardSince = Timestamp(2, 0);
 
     auto chunk = assertGet(ChunkType::parseFromNetworkRequest(
         BSON(ChunkType::name(OID::gen())
@@ -262,7 +295,11 @@ TEST(ChunkType, ParseFromNetworkRequest) {
              << "lastmod"
              << BSON("e" << chunkVersion.epoch() << "t" << chunkVersion.getTimestamp() << "v"
                          << Timestamp(chunkVersion.toLong()))
-             << ChunkType::shard("shard0001"))));
+             << ChunkType::shard("shard0001") << ChunkType::onCurrentShardSince()
+             << onCurrentShardSince << ChunkType::history()
+             << BSON_ARRAY(BSON(ChunkHistoryBase::kValidAfterFieldName
+                                << onCurrentShardSince << ChunkHistoryBase::kShardFieldName
+                                << "shard0001")))));
 
     ASSERT_EQ("shard0001", chunk.getShard());
     ASSERT_EQ(chunkVersion, chunk.getVersion());