SERVER-76489 Fixing implicit create collection on sharded DDL ops

8 files changed, 100 insertions, 9 deletions

diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index d81f840be05..dd05fae9550 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -839,4 +839,6 @@ last-lts:
     ticket: SERVER-76550
   - test_file: jstests/sharding/analyze_shard_key/monotonicity_hashed_sharding_compound.js
     ticket: SERVER-76719
+  - test_file: jstests/sharding/implicit_create_collection_triggered_by_DDLs.js
+    ticket: SERVER-76489
   suites: null
diff --git a/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js b/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js
new file mode 100644
index 00000000000..46547779d2c
--- /dev/null
+++ b/jstests/sharding/implicit_create_collection_triggered_by_DDLs.js
@@ -0,0 +1,54 @@
+(function() {
+"use strict";
+
+function shardKnowledgeIsShardedOrUnknown(shard, nss) {
+    let res = assert.commandWorked(shard.adminCommand({getShardVersion: nss, fullMetadata: true}));
+    return (typeof res.global == 'string' && res.global == 'UNKNOWN') ||
+        (typeof res.metadata == 'object' && typeof res.metadata.collVersion != 'undefined');
+}
+
+const st = new ShardingTest({shards: 2, mongos: 1});
+
+void function testOptimizedShardCollection() {
+    const dbName = 'testDB1';
+    const collName = 'testColl1';
+
+    jsTest.log("Testing that implicit collection creation triggered by optimized " +
+               "shardCollection leaves all shards with the expected knowledge");
+
+    assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.name}));
+
+    assert.commandWorked(
+        st.s.adminCommand({shardCollection: `${dbName}.${collName}`, key: {_id: 'hashed'}}));
+
+    assert(shardKnowledgeIsShardedOrUnknown(st.shard0, `${dbName}.${collName}`),
+           "Unexpected sharding state in Shard 0");
+    assert(shardKnowledgeIsShardedOrUnknown(st.shard1, `${dbName}.${collName}`),
+           "Unexpected sharding state in Shard 1");
+}();
+
+void function testmovePrimary() {
+    const dbName = 'testDB2';
+    const collName = 'testColl2';
+
+    jsTest.log("Testing that implicit collection creation triggered by movePrimary " +
+               "leaves all shards with the expected knowledge");
+
+    assert.commandWorked(st.s.adminCommand({enableSharding: dbName, primaryShard: st.shard0.name}));
+
+    assert.commandWorked(
+        st.s.adminCommand({shardCollection: `${dbName}.${collName}`, key: {_id: 1}}));
+
+    assert.commandWorked(st.s.adminCommand({
+        movePrimary: dbName,
+        to: st.shard1.name,
+    }));
+
+    assert(shardKnowledgeIsShardedOrUnknown(st.shard0, `${dbName}.${collName}`),
+           "Unexpected sharding state in Shard 0");
+    assert(shardKnowledgeIsShardedOrUnknown(st.shard1, `${dbName}.${collName}`),
+           "Unexpected sharding state in Shard 1");
+}();
+
+st.stop();
+})();
\ No newline at end of file
diff --git a/src/mongo/db/cloner.cpp b/src/mongo/db/cloner.cpp
index c3aae94fd88..68c683385e2 100644
--- a/src/mongo/db/cloner.cpp
+++ b/src/mongo/db/cloner.cpp
@@ -418,7 +418,8 @@ Status DefaultClonerImpl::_createCollectionsForDb(
 
             {
                 OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE
-                    unsafeCreateCollection(opCtx);
+                    unsafeCreateCollection(opCtx,
+                                           /* forceCSRAsUnknownAfterCollectionCreation */ true);
                 Status createStatus = db->userCreateNS(
                     opCtx, nss, collectionOptions, createDefaultIndexes, params.idIndexSpec);
                 if (!createStatus.isOK()) {
diff --git a/src/mongo/db/s/implicit_collection_creation_test.cpp b/src/mongo/db/s/implicit_collection_creation_test.cpp
index ae1016e4644..a7957a225c6 100644
--- a/src/mongo/db/s/implicit_collection_creation_test.cpp
+++ b/src/mongo/db/s/implicit_collection_creation_test.cpp
@@ -31,6 +31,7 @@
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/catalog_raii.h"
+#include "mongo/db/s/collection_sharding_runtime.h"
 #include "mongo/db/s/operation_sharding_state.h"
 #include "mongo/db/s/shard_server_test_fixture.h"
 #include "mongo/unittest/unittest.h"
@@ -66,6 +67,26 @@ TEST_F(ImplicitCollectionCreationTest, AllowImplicitCollectionCreate) {
     WriteUnitOfWork wuow(operationContext());
     ASSERT_OK(db->userCreateNS(operationContext(), nss, CollectionOptions{}));
     wuow.commit();
+
+    const auto scopedCsr =
+        CollectionShardingRuntime::assertCollectionLockedAndAcquireShared(operationContext(), nss);
+    ASSERT_TRUE(scopedCsr->getCurrentMetadataIfKnown());
+}
+
+TEST_F(ImplicitCollectionCreationTest, AllowImplicitCollectionCreateWithSetCSRAsUnknown) {
+    NamespaceString nss =
+        NamespaceString::createNamespaceString_forTest("AllowImplicitCollectionCreateDB.TestColl");
+    OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE unsafeCreateCollection(
+        operationContext(), /* forceCSRAsUnknownAfterCollectionCreation */ true);
+    AutoGetCollection autoColl(operationContext(), nss, MODE_IX);
+    auto db = autoColl.ensureDbExists(operationContext());
+    WriteUnitOfWork wuow(operationContext());
+    ASSERT_OK(db->userCreateNS(operationContext(), nss, CollectionOptions{}));
+    wuow.commit();
+
+    const auto scopedCsr =
+        CollectionShardingRuntime::assertCollectionLockedAndAcquireShared(operationContext(), nss);
+    ASSERT_FALSE(scopedCsr->getCurrentMetadataIfKnown());
 }
 
 }  // namespace
diff --git a/src/mongo/db/s/migration_destination_manager.cpp b/src/mongo/db/s/migration_destination_manager.cpp
index 47558dbb88a..9210fc9cc2c 100644
--- a/src/mongo/db/s/migration_destination_manager.cpp
+++ b/src/mongo/db/s/migration_destination_manager.cpp
@@ -1058,10 +1058,9 @@ void MigrationDestinationManager::cloneCollectionIndexesAndOptions(
                               << collectionByUUID->ns().toStringForErrorMsg());
             }
 
-            // We do not have a collection by this name. Create the collection with the donor's
-            // options.
+            // We do not have a collection by this name. Create it with the donor's options.
             OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE
-                unsafeCreateCollection(opCtx);
+                unsafeCreateCollection(opCtx, /* forceCSRAsUnknownAfterCollectionCreation */ true);
             WriteUnitOfWork wuow(opCtx);
             CollectionOptions collectionOptions = uassertStatusOK(
                 CollectionOptions::parse(collectionOptionsAndIndexes.options,
diff --git a/src/mongo/db/s/operation_sharding_state.cpp b/src/mongo/db/s/operation_sharding_state.cpp
index 1c6d9bf94f5..cd34b05f8a8 100644
--- a/src/mongo/db/s/operation_sharding_state.cpp
+++ b/src/mongo/db/s/operation_sharding_state.cpp
@@ -169,17 +169,19 @@ using ScopedAllowImplicitCollectionCreate_UNSAFE =
     OperationShardingState::ScopedAllowImplicitCollectionCreate_UNSAFE;
 
 ScopedAllowImplicitCollectionCreate_UNSAFE::ScopedAllowImplicitCollectionCreate_UNSAFE(
-    OperationContext* opCtx)
+    OperationContext* opCtx, bool forceCSRAsUnknownAfterCollectionCreation)
     : _opCtx(opCtx) {
     auto& oss = get(_opCtx);
     invariant(!oss._allowCollectionCreation);
     oss._allowCollectionCreation = true;
+    oss._forceCSRAsUnknownAfterCollectionCreation = forceCSRAsUnknownAfterCollectionCreation;
 }
 
 ScopedAllowImplicitCollectionCreate_UNSAFE::~ScopedAllowImplicitCollectionCreate_UNSAFE() {
     auto& oss = get(_opCtx);
     invariant(oss._allowCollectionCreation);
     oss._allowCollectionCreation = false;
+    oss._forceCSRAsUnknownAfterCollectionCreation = false;
 }
 
 ScopedSetShardRole::ScopedSetShardRole(OperationContext* opCtx,
diff --git a/src/mongo/db/s/operation_sharding_state.h b/src/mongo/db/s/operation_sharding_state.h
index baf7fc62214..05e3533941e 100644
--- a/src/mongo/db/s/operation_sharding_state.h
+++ b/src/mongo/db/s/operation_sharding_state.h
@@ -96,11 +96,17 @@ public:
      *
      * Instantiating this object on the stack indicates to the storage execution subsystem that it
      * is allowed to create any collection in this context and that the caller will be responsible
-     * for notifying the shard Sharding sybsystem of the collection creation.
+     * for notifying the shard Sharding subsystem of the collection creation. Note that in most of
+     * cases the CollectionShardingRuntime associated to that nss will be set as UNSHARDED. However,
+     * there are some scenarios in which it is required to set is as UNKNOWN: that's the reason why
+     * the constructor has the 'forceCSRAsUnknownAfterCollectionCreation' parameter. You can find
+     * more information about how the CSR is modified in ShardServerOpObserver::onCreateCollection.
      */
     class ScopedAllowImplicitCollectionCreate_UNSAFE {
     public:
-        ScopedAllowImplicitCollectionCreate_UNSAFE(OperationContext* opCtx);
+        /* Please read the comment associated to this class */
+        ScopedAllowImplicitCollectionCreate_UNSAFE(
+            OperationContext* opCtx, bool forceCSRAsUnknownAfterCollectionCreation = false);
         ~ScopedAllowImplicitCollectionCreate_UNSAFE();
 
     private:
@@ -170,6 +176,9 @@ private:
 
     // Specifies whether the request is allowed to create database/collection implicitly
     bool _allowCollectionCreation{false};
+    // Specifies whether the CollectionShardingRuntime should be set as unknown after collection
+    // creation
+    bool _forceCSRAsUnknownAfterCollectionCreation{false};
 
     // Stores the shard version expected for each collection that will be accessed
     struct ShardVersionTracker {
diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp
index 417ef53acd9..d324610decc 100644
--- a/src/mongo/db/s/shard_server_op_observer.cpp
+++ b/src/mongo/db/s/shard_server_op_observer.cpp
@@ -709,6 +709,7 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx,
                                                const OplogSlot& createOpTime,
                                                bool fromMigrate) {
     // Only the shard primay nodes control the collection creation and secondaries just follow
+    // Secondaries CSR will be the defaulted one (UNKNOWN in most of the cases)
     if (!opCtx->writesAreReplicated()) {
         return;
     }
@@ -733,10 +734,12 @@ void ShardServerOpObserver::onCreateCollection(OperationContext* opCtx,
             oss._allowCollectionCreation);
 
     // If the check above passes, this means the collection doesn't exist and is being created and
-    // that the caller will be responsible to eventially set the proper placement version
+    // that the caller will be responsible to eventially set the proper placement version.
     auto scopedCsr =
         CollectionShardingRuntime::assertCollectionLockedAndAcquireExclusive(opCtx, collectionName);
-    if (!scopedCsr->getCurrentMetadataIfKnown()) {
+    if (oss._forceCSRAsUnknownAfterCollectionCreation) {
+        scopedCsr->clearFilteringMetadata(opCtx);
+    } else if (!scopedCsr->getCurrentMetadataIfKnown()) {
         scopedCsr->setFilteringMetadata(opCtx, CollectionMetadata());
     }
 }