SERVER-57060 Rename of a sharded collection must bump the collection version

14 files changed, 240 insertions, 33 deletions

diff --git a/jstests/core/rename_collection.js b/jstests/core/rename_collection.js
index 570e6009551..6f32699a733 100644
--- a/jstests/core/rename_collection.js
+++ b/jstests/core/rename_collection.js
@@ -24,7 +24,26 @@ function getNewColl() {
     return coll;
 }
 
-jsTest.log("Rename collection with documents");
+jsTest.log("Rename collection with documents (create SRC and then DST)");
+{
+    const src = getNewColl();
+    const dstName = getNewCollName();
+
+    src.save({x: 1});
+    src.save({x: 2});
+    src.save({x: 3});
+
+    assert.eq(3, src.countDocuments({}));
+
+    assert.commandWorked(src.renameCollection(dstName));
+
+    assert.eq(0, src.countDocuments({}));
+    const dst = db[dstName];
+    assert.eq(3, dst.countDocuments({}));
+    dst.drop();
+}
+
+jsTest.log("Rename collection with documents (create DST and then SRC)");
 {
     const src = getNewColl();
     const dstName = getNewCollName();
@@ -73,8 +92,8 @@ jsTest.log("Rename collection with indexes");
 
 jsTest.log("Rename collection with existing target");
 {
-    const dst = getNewColl();
     const src = getNewColl();
+    const dst = getNewColl();
 
     src.save({x: 1});
     dst.save({x: 2});
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index c59b03baac3..cbf327b1493 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -99,6 +99,7 @@ let viewsCommandTests = {
     _configsvrMovePrimary: {skip: isAnInternalCommand},
     _configsvrRefineCollectionShardKey: {skip: isAnInternalCommand},
     _configsvrRenameCollection: {skip: isAnInternalCommand},
+    _configsvrRenameCollectionMetadata: {skip: isAnInternalCommand},
     _configsvrRemoveShard: {skip: isAnInternalCommand},
     _configsvrRemoveShardFromZone: {skip: isAnInternalCommand},
     _configsvrReshardCollection: {skip: isAnInternalCommand},
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index b2d4827c788..1cc2ffb30d5 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -50,6 +50,7 @@ const allCommands = {
     _configsvrRefineCollectionShardKey: {skip: isPrimaryOnly},
     _configsvrRemoveShard: {skip: isPrimaryOnly},
     _configsvrRemoveShardFromZone: {skip: isPrimaryOnly},
+    _configsvrRenameCollectionMetadata: {skip: isAnInternalCommand},
     _configsvrReshardCollection: {skip: isPrimaryOnly},
     _configsvrSetAllowMigrations: {skip: isPrimaryOnly},
     _configsvrShardCollection: {skip: isPrimaryOnly},
diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js
index c43c764a904..73f9d47617d 100644
--- a/jstests/sharding/read_write_concern_defaults_application.js
+++ b/jstests/sharding/read_write_concern_defaults_application.js
@@ -100,6 +100,7 @@ let testCases = {
     _configsvrRemoveShard: {skip: "internal command"},
     _configsvrRemoveShardFromZone: {skip: "internal command"},
     _configsvrRenameCollection: {skip: "internal command"},
+    _configsvrRenameCollectionMetadata: {skip: "internal command"},
     _configsvrReshardCollection: {skip: "internal command"},
     _configsvrSetAllowMigrations: {skip: "internal command"},
     _configsvrShardCollection: {skip: "internal command"},
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 8675525b656..3d2c1a0e698 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -317,6 +317,7 @@ env.Library(
         'config/configsvr_refine_collection_shard_key_command.cpp',
         'config/configsvr_remove_shard_command.cpp',
         'config/configsvr_remove_shard_from_zone_command.cpp',
+        'config/configsvr_rename_collection_metadata_command.cpp',
         'config/configsvr_reshard_collection_cmd.cpp',
         'config/configsvr_set_allow_migrations_command.cpp',
         'config/configsvr_shard_collection_command.cpp',
diff --git a/src/mongo/db/s/config/configsvr_rename_collection_metadata_command.cpp b/src/mongo/db/s/config/configsvr_rename_collection_metadata_command.cpp
new file mode 100644
index 00000000000..4e18dd3ea1d
--- /dev/null
+++ b/src/mongo/db/s/config/configsvr_rename_collection_metadata_command.cpp
@@ -0,0 +1,94 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/sharding_ddl_util.h"
+#include "mongo/s/request_types/sharded_ddl_commands_gen.h"
+
+namespace mongo {
+namespace {
+
+class ConfigsvrRenameCollectionMetadataCommand final
+    : public TypedCommand<ConfigsvrRenameCollectionMetadataCommand> {
+public:
+    using Request = ConfigsvrRenameCollectionMetadata;
+
+    std::string help() const override {
+        return "Internal command. Do not call directly. Renames a collection.";
+    }
+
+    bool adminOnly() const override {
+        return false;
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kNever;
+    }
+
+    class Invocation final : public InvocationBase {
+    public:
+        using InvocationBase::InvocationBase;
+
+        void typedRun(OperationContext* opCtx) {
+            invariant(serverGlobalParams.clusterRole == ClusterRole::ConfigServer);
+
+            const auto& req = request();
+
+            ShardingCatalogManager::get(opCtx)->renameShardedMetadata(
+                opCtx, ns(), req.getTo(), req.getOptFromCollection());
+        }
+
+    private:
+        NamespaceString ns() const override {
+            return request().getNamespace();
+        }
+
+        bool supportsWriteConcern() const override {
+            return true;
+        }
+
+        void doCheckAuthorization(OperationContext* opCtx) const override {
+            uassert(ErrorCodes::Unauthorized,
+                    "Unauthorized",
+                    AuthorizationSession::get(opCtx->getClient())
+                        ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
+                                                           ActionType::internal));
+        }
+    };
+
+} _configsvrRenameCollectionMetadata;
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index c1ac1602848..c710902aa0e 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -463,6 +463,17 @@ public:
      */
     void downgradeMetadataToPre50Phase2(OperationContext* opCtx);
 
+    /*
+     * Rename collection metadata as part of a renameCollection operation.
+     *
+     * - Updates the FROM collection entry if the source collection is sharded
+     * - Removes the TO collection entry if the target collection was sharded
+     */
+    void renameShardedMetadata(OperationContext* opCtx,
+                               const NamespaceString& from,
+                               const NamespaceString& to,
+                               boost::optional<CollectionType> optFromCollType);
+
     //
     // For Diagnostics
     //
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
index 9cbbe8f9676..1af7c6d40f3 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp
@@ -53,6 +53,7 @@
 #include "mongo/db/operation_context.h"
 #include "mongo/db/query/collation/collator_factory_interface.h"
 #include "mongo/db/repl/repl_client_info.h"
+#include "mongo/db/s/sharding_ddl_util.h"
 #include "mongo/db/s/sharding_logging.h"
 #include "mongo/db/vector_clock.h"
 #include "mongo/executor/network_interface.h"
@@ -459,4 +460,41 @@ void ShardingCatalogManager::updateShardingCatalogEntryForCollectionInTxn(
     }
 }
 
+void ShardingCatalogManager::renameShardedMetadata(
+    OperationContext* opCtx,
+    const NamespaceString& from,
+    const NamespaceString& to,
+    boost::optional<CollectionType> optFromCollType) {
+    // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and
+    // migrations. Take _kZoneOpLock in exclusive mode to prevent concurrent zone operations.
+    // TODO(SERVER-25359): Replace with a collection-specific lock map to allow splits/merges/
+    // move chunks on different collections to proceed in parallel.
+    Lock::ExclusiveLock chunkLk(opCtx->lockState(), _kChunkOpLock);
+    Lock::ExclusiveLock zoneLk(opCtx->lockState(), _kZoneOpLock);
+
+    std::string logMsg = str::stream() << from << " to " << to;
+    if (optFromCollType) {
+        // Rename CSRS metadata in case the source collection is sharded
+        auto collType = *optFromCollType;
+        sharding_ddl_util::shardedRenameMetadata(opCtx, collType, to);
+        ShardingLogging::get(opCtx)->logChange(
+            opCtx,
+            "renameCollection.metadata",
+            str::stream() << logMsg << ": dropped target collection and renamed source collection",
+            BSON("newCollMetadata" << collType.toBSON()),
+            ShardingCatalogClient::kLocalWriteConcern);
+    } else {
+        // Remove stale CSRS metadata in case the source collection is unsharded and the
+        // target collection was sharded
+        sharding_ddl_util::removeCollMetadataFromConfig(opCtx, to);
+        ShardingLogging::get(opCtx)->logChange(opCtx,
+                                               "renameCollection.metadata",
+                                               str::stream()
+                                                   << logMsg << " : dropped target collection.",
+                                               BSONObj(),
+                                               ShardingCatalogClient::kLocalWriteConcern);
+    }
+}
+
+
 }  // namespace mongo
diff --git a/src/mongo/db/s/rename_collection_coordinator.cpp b/src/mongo/db/s/rename_collection_coordinator.cpp
index d8db3483980..508eb01f51f 100644
--- a/src/mongo/db/s/rename_collection_coordinator.cpp
+++ b/src/mongo/db/s/rename_collection_coordinator.cpp
@@ -280,23 +280,24 @@ ExecutorFuture<void> RenameCollectionCoordinator::_runImpl(
                                     participants,
                                     **executor);
                             }))
-        .then(_executePhase(
-            Phase::kRenameMetadata,
-            [this, anchor = shared_from_this()] {
-                auto opCtxHolder = cc().makeOperationContext();
-                auto* opCtx = opCtxHolder.get();
-                getForwardableOpMetadata().setOn(opCtx);
+        .then(_executePhase(Phase::kRenameMetadata,
+                            [this, anchor = shared_from_this()] {
+                                auto opCtxHolder = cc().makeOperationContext();
+                                auto* opCtx = opCtxHolder.get();
+                                getForwardableOpMetadata().setOn(opCtx);
 
-                const auto& optFromCollType = _doc.getOptShardedCollInfo();
-                if (optFromCollType) {
-                    // Rename CSRS metadata
-                    auto collType = *optFromCollType;
-                    sharding_ddl_util::shardedRenameMetadata(opCtx, collType, _doc.getTo());
-                } else if (_doc.getTargetIsSharded()) {
-                    // Remove stale target CSRS metadata
-                    sharding_ddl_util::removeCollMetadataFromConfig(opCtx, _doc.getTo());
-                }
-            }))
+                                ConfigsvrRenameCollectionMetadata req(nss(), _doc.getTo());
+                                req.setOptFromCollection(_doc.getOptShardedCollInfo());
+
+                                const auto& configShard =
+                                    Grid::get(opCtx)->shardRegistry()->getConfigShard();
+                                uassertStatusOK(configShard->runCommand(
+                                    opCtx,
+                                    ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+                                    "admin",
+                                    CommandHelpers::appendMajorityWriteConcern(req.toBSON({})),
+                                    Shard::RetryPolicy::kIdempotent));
+                            }))
         .then(_executePhase(
             Phase::kUnblockCRUD,
             [this, executor = executor, anchor = shared_from_this()] {
diff --git a/src/mongo/db/s/sharded_rename_collection.idl b/src/mongo/db/s/sharded_rename_collection.idl
index efa4a88f495..942cc857995 100644
--- a/src/mongo/db/s/sharded_rename_collection.idl
+++ b/src/mongo/db/s/sharded_rename_collection.idl
@@ -62,14 +62,6 @@ enums:
             kDeleteFromRangeDeletions: "deleteFromRangeDeletions"
             kUnblockCRUD: "unblockCRUD"
 
-types:
-    CollectionInfo:
-        description: "Information of the sharded collection to rename."
-        bson_serialization_type: object
-        cpp_type: CollectionType
-        serializer: "mongo::CollectionType::toBSON"
-        deserializer: "mongo::CollectionType"
-
 commands:
     _shardsvrRenameCollectionParticipant:
         command_name: _shardsvrRenameCollectionParticipant
@@ -117,7 +109,7 @@ structs:
                 default: false
             optShardedCollInfo:
                 description: "CollectionType of the collection currently being renamed (if sharded)."
-                type: CollectionInfo
+                type: CollectionType
                 optional: true
             sourceUUID:
                 type: uuid
diff --git a/src/mongo/db/s/sharding_ddl_util.cpp b/src/mongo/db/s/sharding_ddl_util.cpp
index 6f0a9dabc3b..1b43e958599 100644
--- a/src/mongo/db/s/sharding_ddl_util.cpp
+++ b/src/mongo/db/s/sharding_ddl_util.cpp
@@ -34,11 +34,13 @@
 #include "mongo/db/s/sharding_ddl_util.h"
 
 #include "mongo/db/catalog/collection_catalog.h"
+#include "mongo/db/commands/feature_compatibility_version.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/s/collection_sharding_runtime.h"
 #include "mongo/db/s/shard_filtering_metadata_refresh.h"
 #include "mongo/db/s/sharding_logging.h"
 #include "mongo/db/s/sharding_util.h"
+#include "mongo/db/vector_clock.h"
 #include "mongo/logv2/log.h"
 #include "mongo/rpc/metadata/impersonated_user_metadata.h"
 #include "mongo/s/catalog/type_chunk.h"
@@ -231,6 +233,8 @@ bool removeCollMetadataFromConfig(OperationContext* opCtx, const NamespaceString
 void shardedRenameMetadata(OperationContext* opCtx,
                            CollectionType& fromCollType,
                            const NamespaceString& toNss) {
+    invariant(serverGlobalParams.clusterRole == ClusterRole::ConfigServer);
+
     auto catalogClient = Grid::get(opCtx)->catalogClient();
     auto fromNss = fromCollType.getNss();
 
@@ -255,8 +259,21 @@ void shardedRenameMetadata(OperationContext* opCtx,
     // Update FROM tags to TO
     updateTags(opCtx, fromNss, toNss);
 
-    // Insert the TO collection entry
+    // Update namespace and bump timestamp of the FROM collection entry
     fromCollType.setNss(toNss);
+    auto now = VectorClock::get(opCtx)->getTime();
+    auto newTimestamp = now.clusterTime().asTimestamp();
+    fromCollType.setTimestamp(newTimestamp);
+    {
+        // Only bump the epoch if the whole cluster is in FCV 5.0, so chunks do not contain epochs.
+        FixedFCVRegion fixedFCVRegion(opCtx);
+        if (serverGlobalParams.featureCompatibility.isGreaterThanOrEqualTo(
+                ServerGlobalParams::FeatureCompatibility::Version::kVersion50)) {
+            fromCollType.setEpoch(OID::gen());
+        }
+    }
+
+    // Insert the TO collection entry
     uassertStatusOK(
         catalogClient->insertConfigDocument(opCtx,
                                             CollectionType::ConfigNS,
diff --git a/src/mongo/db/s/sharding_ddl_util.h b/src/mongo/db/s/sharding_ddl_util.h
index 17bf2ba8ff7..54f2e41dd66 100644
--- a/src/mongo/db/s/sharding_ddl_util.h
+++ b/src/mongo/db/s/sharding_ddl_util.h
@@ -80,7 +80,7 @@ bool removeCollMetadataFromConfig(OperationContext* opCtx, const NamespaceString
  * - Update namespace associated with tags (FROM -> TO)
  * - Update FROM collection entry to TO
  *
- * This function is idempotent.
+ * This function is idempotent and can just be invoked by the CSRS.
  */
 void shardedRenameMetadata(OperationContext* opCtx,
                            CollectionType& fromCollType,
diff --git a/src/mongo/db/s/sharding_ddl_util_test.cpp b/src/mongo/db/s/sharding_ddl_util_test.cpp
index ac97bd60042..f8283a49731 100644
--- a/src/mongo/db/s/sharding_ddl_util_test.cpp
+++ b/src/mongo/db/s/sharding_ddl_util_test.cpp
@@ -154,12 +154,17 @@ TEST_F(ShardingDDLUtilTest, ShardedRenameMetadata) {
     std::vector<BSONObj> toChunks;
     client.findN(toChunks, ChunkType::ConfigNS.ns(), toChunksQuery, nChunks);
 
-    // Check that the original epoch is preserved in config.collections entry
-    ASSERT(fromCollection.getEpoch() == toCollection.getEpoch());
+    // Check that original epoch/timestamp are changed in config.collections entry
+    ASSERT(fromCollection.getEpoch() != toCollection.getEpoch());
+    ASSERT(fromCollection.getTimestamp() != toCollection.getTimestamp());
 
     // Check that no other CollectionType field has been changed
-    auto fromUnchangedFields = fromDoc.removeField(CollectionType::kNssFieldName);
-    auto toUnchangedFields = toDoc.removeField(CollectionType::kNssFieldName);
+    auto fromUnchangedFields = fromDoc.removeField(CollectionType::kNssFieldName)
+                                   .removeField(CollectionType::kEpochFieldName)
+                                   .removeField(CollectionType::kTimestampFieldName);
+    auto toUnchangedFields = toDoc.removeField(CollectionType::kNssFieldName)
+                                 .removeField(CollectionType::kEpochFieldName)
+                                 .removeField(CollectionType::kTimestampFieldName);
     ASSERT_EQ(fromUnchangedFields.woCompare(toUnchangedFields), 0);
 
     // Check that chunk documents remain unchanged
diff --git a/src/mongo/s/request_types/sharded_ddl_commands.idl b/src/mongo/s/request_types/sharded_ddl_commands.idl
index e736dedec09..0a5a07b0884 100644
--- a/src/mongo/s/request_types/sharded_ddl_commands.idl
+++ b/src/mongo/s/request_types/sharded_ddl_commands.idl
@@ -28,6 +28,8 @@
 
 global:
     cpp_namespace: "mongo"
+    cpp_includes:
+        - "mongo/s/catalog/type_collection.h"
 
 imports:
     - "mongo/db/commands/rename_collection.idl"
@@ -38,6 +40,14 @@ imports:
     - "mongo/s/database_version.idl"
     - "mongo/s/resharding/common_types.idl"
 
+types:
+    CollectionType:
+        description: "Information of a sharded collection."
+        bson_serialization_type: object
+        cpp_type: CollectionType
+        serializer: "mongo::CollectionType::toBSON"
+        deserializer: "mongo::CollectionType"
+
 structs:
 
     ConfigsvrCreateDatabaseResponse:
@@ -209,6 +219,22 @@ commands:
         chained_structs:
             RenameCollectionRequest: RenameCollectionRequest
 
+    _configsvrRenameCollectionMetadata:
+        command_name: _configsvrRenameCollectionMetadata
+        cpp_name: ConfigsvrRenameCollectionMetadata
+        description: "Internal command for renaming collection metadata on the CSRS"
+        strict: false
+        namespace: concatenate_with_db
+        api_version: ""
+        fields:
+            optFromCollection:
+                description: "Information of the source collection to rename, used only for sharded collection."
+                type: CollectionType
+                optional: true
+            to:
+                type: namespacestring
+                description: "The new namespace for the collection being renamed."
+
     _shardsvrReshardCollection:
         command_name: _shardsvrReshardCollection
         cpp_name: ShardsvrReshardCollection