SERVER-67796 Persist the result of doCompactOperation in the compaction coordinator

8 files changed, 67 insertions, 18 deletions

diff --git a/src/mongo/db/commands/fle2_compact.idl b/src/mongo/db/commands/fle2_compact.idl
index 9adcc209503..c0602a864d6 100644
--- a/src/mongo/db/commands/fle2_compact.idl
+++ b/src/mongo/db/commands/fle2_compact.idl
@@ -82,5 +82,5 @@ commands:
         reply_type: CompactStructuredEncryptionDataCommandReply
         fields:
             compactionTokens:
-                description: "Map of field path to EOCToken"
+                description: "Map of field path to ECOCToken"
                 type: object
diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index dbea7398a5a..2466fb480c3 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -359,6 +359,16 @@ public:
                         ->waitForCoordinatorsOfGivenTypeToComplete(
                             opCtx, DDLCoordinatorTypeEnum::kCollMod);
                 }
+
+                // TODO SERVER-68373 remove once 7.0 becomes last LTS
+                if (actualVersion > requestedVersion) {
+                    // Drain the QE compact coordinator because it persists state that is
+                    // not backwards compatible with earlier versions.
+                    ShardingDDLCoordinatorService::getService(opCtx)
+                        ->waitForCoordinatorsOfGivenTypeToComplete(
+                            opCtx, DDLCoordinatorTypeEnum::kCompactStructuredEncryptionData);
+                }
+
                 // If we are only running phase-1, then we are done
                 return true;
             }
diff --git a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp
index 6700e4c99b1..a76e7f07faf 100644
--- a/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp
+++ b/src/mongo/db/s/compact_structured_encryption_data_coordinator.cpp
@@ -211,11 +211,21 @@ ExecutorFuture<void> CompactStructuredEncryptionDataCoordinator::_runImpl(
                                 _doc.setSkipCompact(_skipCompact);
                                 _doc.setEcocRenameUuid(_ecocRenameUuid);
                             }))
-        .then(_executePhase(
-            Phase::kCompactStructuredEncryptionData,
-            [this, anchor = shared_from_this()]() { _response = doCompactOperation(_doc); }))
-        .then(_executePhase(Phase::kDropTempCollection,
-                            [this, anchor = shared_from_this()] { doDropOperation(_doc); }));
+        .then(_executePhase(Phase::kCompactStructuredEncryptionData,
+                            [this, anchor = shared_from_this()]() {
+                                _response = doCompactOperation(_doc);
+                                if (!_isPre61Compatible()) {
+                                    stdx::lock_guard lg(_docMutex);
+                                    _doc.setResponse(_response);
+                                }
+                            }))
+        .then(_executePhase(Phase::kDropTempCollection, [this, anchor = shared_from_this()] {
+            if (!_isPre61Compatible()) {
+                invariant(_doc.getResponse());
+                _response = *_doc.getResponse();
+            }
+            doDropOperation(_doc);
+        }));
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/compact_structured_encryption_data_coordinator.h b/src/mongo/db/s/compact_structured_encryption_data_coordinator.h
index f8e9ba84aa7..bb0667add70 100644
--- a/src/mongo/db/s/compact_structured_encryption_data_coordinator.h
+++ b/src/mongo/db/s/compact_structured_encryption_data_coordinator.h
@@ -73,6 +73,12 @@ private:
     ExecutorFuture<void> _runImpl(std::shared_ptr<executor::ScopedTaskExecutor> executor,
                                   const CancellationToken& token) noexcept final;
 
+    // TODO SERVER-68373 remove once 7.0 becomes last LTS
+    bool _isPre61Compatible() const {
+        return operationType() ==
+            DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre61Compatible;
+    }
+
 private:
     boost::optional<CompactStructuredEncryptionDataCommandReply> _response;
     bool _skipCompact{false};
diff --git a/src/mongo/db/s/compact_structured_encryption_data_coordinator.idl b/src/mongo/db/s/compact_structured_encryption_data_coordinator.idl
index a21ba2e4f83..5a725d3ec82 100644
--- a/src/mongo/db/s/compact_structured_encryption_data_coordinator.idl
+++ b/src/mongo/db/s/compact_structured_encryption_data_coordinator.idl
@@ -82,3 +82,7 @@ structs:
             compactionTokens:
                 description: "Compaction tokens for the compact operation"
                 type: object_owned
+            response:
+                description: "Response to the compactStructuredEncryptedData command"
+                type: CompactStructuredEncryptionDataCommandReply
+                optional: true
diff --git a/src/mongo/db/s/sharding_ddl_coordinator.idl b/src/mongo/db/s/sharding_ddl_coordinator.idl
index 89ae4cd6fb6..e9df9baa315 100644
--- a/src/mongo/db/s/sharding_ddl_coordinator.idl
+++ b/src/mongo/db/s/sharding_ddl_coordinator.idl
@@ -52,9 +52,11 @@ enums:
             kSetAllowMigrations: "setAllowMigrations"
             kCollMod: "collMod_V3"
             # TODO SERVER-68008: Remove once 7.0 becomes last LTS
-            kCollModPre61Compatible: "collMod_V2" 
+            kCollModPre61Compatible: "collMod_V2"
             kReshardCollection: "reshardCollection"
-            kCompactStructuredEncryptionData: "compactStructuredEncryptionData"
+            kCompactStructuredEncryptionData: "compactStructuredEncryptionData_V2"
+            # TODO SERVER-68373 remove once 7.0 becomes last LTS
+            kCompactStructuredEncryptionDataPre61Compatible: "compactStructuredEncryptionData"
 
 types:
     ForwardableOperationMetadata:
@@ -92,7 +94,7 @@ structs:
                 type: LogicalSessionId
             txnNumber:
                 type: TxnNumber
-    
+
     ShardingDDLCoordinatorMetadata:
         description: "Commong metadata for all sharding DDL coordinator."
         generate_comparison_operators: false
diff --git a/src/mongo/db/s/sharding_ddl_coordinator_service.cpp b/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
index 954c5cb66c3..1fc117046d2 100644
--- a/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
+++ b/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
@@ -95,6 +95,8 @@ std::shared_ptr<ShardingDDLCoordinator> constructShardingDDLCoordinatorInstance(
         case DDLCoordinatorTypeEnum::kReshardCollection:
             return std::make_shared<ReshardCollectionCoordinator>(service, std::move(initialState));
             break;
+        case DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre61Compatible:
+            // TODO SERVER-68373 remove once 7.0 becomes last LTS
         case DDLCoordinatorTypeEnum::kCompactStructuredEncryptionData:
             return std::make_shared<CompactStructuredEncryptionDataCoordinator>(
                 service, std::move(initialState));
diff --git a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp
index 1bbb55c6da3..dad00676c50 100644
--- a/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp
+++ b/src/mongo/db/s/shardsvr_compact_structured_encryption_data_command.cpp
@@ -77,19 +77,27 @@ public:
         using InvocationBase::InvocationBase;
 
         Reply typedRun(OperationContext* opCtx) {
-            FixedFCVRegion fixedFcvRegion(opCtx);
 
-            auto compact = makeRequest(opCtx);
-            if (!compact) {
+            auto compactCoordinator =
+                [&]() -> std::shared_ptr<CompactStructuredEncryptionDataCoordinator> {
+                FixedFCVRegion fixedFcvRegion(opCtx);
+
+                auto compact = makeRequest(opCtx);
+                if (!compact) {
+                    return nullptr;
+                }
+                return checked_pointer_cast<CompactStructuredEncryptionDataCoordinator>(
+                    ShardingDDLCoordinatorService::getService(opCtx)->getOrCreateInstance(
+                        opCtx, compact->toBSON()));
+            }();
+
+            if (!compactCoordinator) {
                 // Nothing to do.
                 LOGV2(6548305, "Skipping compaction as there is no ECOC collection to compact");
                 return CompactStats({}, {}, {});
             }
 
-            return checked_pointer_cast<CompactStructuredEncryptionDataCoordinator>(
-                       ShardingDDLCoordinatorService::getService(opCtx)->getOrCreateInstance(
-                           opCtx, compact->toBSON()))
-                ->getResponse(opCtx);
+            return compactCoordinator->getResponse(opCtx);
         }
 
     private:
@@ -116,6 +124,14 @@ public:
             }
 
             CompactStructuredEncryptionDataState compact;
+            auto coordinatorType = DDLCoordinatorTypeEnum::kCompactStructuredEncryptionData;
+
+            if (serverGlobalParams.featureCompatibility.isLessThan(
+                    multiversion::FeatureCompatibilityVersion::kVersion_6_1)) {
+                // TODO SERVER-68373 remove once 7.0 becomes last LTS
+                coordinatorType =
+                    DDLCoordinatorTypeEnum::kCompactStructuredEncryptionDataPre61Compatible;
+            }
 
             if (ecocColl.getCollection()) {
                 compact.setEcocUuid(ecocColl->uuid());
@@ -124,8 +140,7 @@ public:
                 compact.setEcocRenameUuid(ecocTempColl->uuid());
             }
 
-            compact.setShardingDDLCoordinatorMetadata(
-                {{nss, DDLCoordinatorTypeEnum::kCompactStructuredEncryptionData}});
+            compact.setShardingDDLCoordinatorMetadata({{nss, coordinatorType}});
             compact.setEscNss(namespaces.escNss);
             compact.setEccNss(namespaces.eccNss);
             compact.setEcocNss(namespaces.ecocNss);