SERVER-68122 Remove encryption from storageOptions on secondary replication

(cherry picked from commit ef120ac5552574fb9b84d36d842ead8519588c31)

15 files changed, 211 insertions, 5 deletions

diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp
index 0066240cc2f..fe1cc4b34a2 100644
--- a/src/mongo/db/repl/collection_cloner.cpp
+++ b/src/mongo/db/repl/collection_cloner.cpp
@@ -29,6 +29,8 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplicationInitialSync
 
+#include "mongo/db/index/index_descriptor_fwd.h"
+#include "mongo/db/service_context.h"
 #include "mongo/platform/basic.h"
 
 #include "mongo/base/string_data.h"
@@ -206,8 +208,20 @@ BaseCloner::AfterStageBehavior CollectionCloner::listIndexesStage() {
                       "source"_attr = getSource());
     }
 
+    const auto storageEngine = getGlobalServiceContext()->getStorageEngine();
     // Parse the index specs into their respective state, ready or unfinished.
     for (auto&& spec : indexSpecs) {
+        // Sanitize storage engine options to remove options which might not apply to this node. See
+        // SERVER-68122.
+        if (auto storageEngineElem = spec.getField(IndexDescriptor::kStorageEngineFieldName)) {
+            auto sanitizedStorageEngineOpts =
+                storageEngine->getSanitizedStorageOptionsForSecondaryReplication(
+                    storageEngineElem.embeddedObject());
+            fassert(6812200, sanitizedStorageEngineOpts);
+            spec = spec.addFields(BSON(IndexDescriptor::kStorageEngineFieldName
+                                       << sanitizedStorageEngineOpts.getValue()));
+        }
+
         if (spec.hasField("clustered")) {
             invariant(_collectionOptions.clusteredIndex);
             invariant(spec.getBoolField("clustered") == true);
diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp
index adeeeb0afb6..23da9db918a 100644
--- a/src/mongo/db/repl/database_cloner.cpp
+++ b/src/mongo/db/repl/database_cloner.cpp
@@ -29,6 +29,7 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplicationInitialSync
 
+#include "mongo/db/service_context.h"
 #include "mongo/platform/basic.h"
 
 #include "mongo/base/string_data.h"
@@ -69,6 +70,7 @@ BaseCloner::AfterStageBehavior DatabaseCloner::listCollectionsStage() {
     BSONObj res;
     auto collectionInfos =
         getClient()->getCollectionInfos(_dbName, ListCollectionsFilter::makeTypeCollectionFilter());
+    const auto storageEngine = getGlobalServiceContext()->getStorageEngine();
 
     stdx::unordered_set<std::string> seen;
     for (auto&& info : collectionInfos) {
@@ -104,6 +106,13 @@ BaseCloner::AfterStageBehavior DatabaseCloner::listCollectionsStage() {
                               << "'" << result.getName() << "': " << info,
                 isDuplicate);
 
+        // Sanitize storage engine options to remove options which might not apply to this node. See
+        // SERVER-68122.
+        auto sanitizedStorageOptions =
+            uassertStatusOK(storageEngine->getSanitizedStorageOptionsForSecondaryReplication(
+                result.getOptions().storageEngine));
+        result.getOptions().storageEngine = sanitizedStorageOptions;
+
         // While UUID is a member of CollectionOptions, listCollections does not return the
         // collectionUUID there as part of the options, but instead places it in the 'info' field.
         // We need to move it back to CollectionOptions to create the collection properly.
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 4053bc13f91..9c6a3e65ce8 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -62,6 +62,7 @@
 #include "mongo/db/client.h"
 #include "mongo/db/coll_mod_gen.h"
 #include "mongo/db/commands.h"
+#include "mongo/db/commands/create_gen.h"
 #include "mongo/db/commands/feature_compatibility_version_parser.h"
 #include "mongo/db/concurrency/exception_util.h"
 #include "mongo/db/db_raii.h"
@@ -789,6 +790,26 @@ NamespaceString extractNsFromUUIDorNs(OperationContext* opCtx,
     return ui ? extractNsFromUUID(opCtx, ui.get()) : extractNs(ns.db(), cmd);
 }
 
+StatusWith<BSONObj> getObjWithSanitizedStorageEngineOptions(OperationContext* opCtx,
+                                                            const BSONObj& cmd) {
+    static_assert(
+        CreateCommand::kStorageEngineFieldName == IndexDescriptor::kStorageEngineFieldName,
+        "Expected storage engine options field to be the same for collections and indexes.");
+
+    if (auto storageEngineElem = cmd[IndexDescriptor::kStorageEngineFieldName]) {
+        auto storageEngine = opCtx->getServiceContext()->getStorageEngine();
+        auto engineObj = storageEngineElem.embeddedObject();
+        auto sanitizedObj =
+            storageEngine->getSanitizedStorageOptionsForSecondaryReplication(engineObj);
+        if (!sanitizedObj.isOK()) {
+            return sanitizedObj.getStatus();
+        }
+        return cmd.addFields(
+            BSON(IndexDescriptor::kStorageEngineFieldName << sanitizedObj.getValue()));
+    }
+    return cmd;
+}
+
 using OpApplyFn = std::function<Status(
     OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode)>;
 
@@ -813,7 +834,14 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
     {"create",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
           const auto& ui = entry.getUuid();
-          const auto& cmd = entry.getObject();
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          const auto sanitizedCmdOrStatus =
+              getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
+          if (!sanitizedCmdOrStatus.isOK()) {
+              return sanitizedCmdOrStatus.getStatus();
+          }
+          const auto& cmd = sanitizedCmdOrStatus.getValue();
           const NamespaceString nss(extractNs(entry.getNss().db(), cmd));
 
           // Mode SECONDARY steady state replication should not allow create collection to rename an
@@ -857,7 +885,15 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
       {ErrorCodes::NamespaceExists}}},
     {"createIndexes",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
-          const auto& cmd = entry.getObject();
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          const auto sanitizedCmdOrStatus =
+              getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
+          if (!sanitizedCmdOrStatus.isOK()) {
+              return sanitizedCmdOrStatus.getStatus();
+          }
+          const auto& cmd = sanitizedCmdOrStatus.getValue();
+
           if (OplogApplication::Mode::kApplyOpsCmd == mode) {
               return {ErrorCodes::CommandNotSupported,
                       "The createIndexes operation is not supported in applyOps mode"};
@@ -892,6 +928,16 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
                   "Error parsing 'startIndexBuild' oplog entry");
           }
 
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          for (auto& spec : swOplogEntry.getValue().indexSpecs) {
+              auto sanitizedObj = getObjWithSanitizedStorageEngineOptions(opCtx, spec);
+              if (!sanitizedObj.isOK()) {
+                  return swOplogEntry.getStatus();
+              }
+              spec = sanitizedObj.getValue();
+          }
+
           IndexBuildsCoordinator::ApplicationMode applicationMode =
               IndexBuildsCoordinator::ApplicationMode::kNormal;
           if (mode == OplogApplication::Mode::kInitialSync) {
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 304a695308e..ebd81361f70 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -429,6 +429,16 @@ public:
     }
 
     /**
+     * Returns the input storage engine options, sanitized to remove options that may not apply to
+     * this node, such as encryption. Might be called for both collection and index options. See
+     * SERVER-68122.
+     */
+    virtual StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const {
+        return options;
+    }
+
+    /**
      * The destructor will never be called from mongod, but may be called from tests.
      * Engines may assume that this will only be called in the case of clean shutdown, even if
      * cleanShutdown() hasn't been called.
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index bfdb5745ac6..8cea12121f6 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -672,6 +672,13 @@ public:
     virtual void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) = 0;
 
     /**
+     * Returns the input storage engine options, sanitized to remove options that may not apply to
+     * this node, such as encryption. Might be called for both collection and index options. See
+     * SERVER-68122.
+     */
+    virtual StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const = 0;
+    /**
      * Instructs the storage engine to dump its internal state.
      */
     virtual void dump() const = 0;
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 6e9c57de195..2b578944443 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -1378,6 +1378,11 @@ const DurableCatalog* StorageEngineImpl::getCatalog() const {
     return _catalog.get();
 }
 
+StatusWith<BSONObj> StorageEngineImpl::getSanitizedStorageOptionsForSecondaryReplication(
+    const BSONObj& options) const {
+    return _engine->getSanitizedStorageOptionsForSecondaryReplication(options);
+}
+
 void StorageEngineImpl::dump() const {
     _engine->dump();
 }
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index a8e36a1af88..80adc2ce382 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -367,6 +367,9 @@ public:
 
     void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) override;
 
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const override;
+
     void dump() const override;
 
 private:
diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h
index d39b8b248f3..4cbc635eec0 100644
--- a/src/mongo/db/storage/storage_engine_mock.h
+++ b/src/mongo/db/storage/storage_engine_mock.h
@@ -208,6 +208,11 @@ public:
 
     void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) final {}
 
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const final {
+        return options;
+    }
+
     void dump() const final {}
 };
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index a849fa04042..243e84608b8 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -102,7 +102,7 @@ void WiredTigerIndex::getKey(OperationContext* opCtx, WT_CURSOR* cursor, WT_ITEM
 StatusWith<std::string> WiredTigerIndex::parseIndexOptions(const BSONObj& options) {
     StringBuilder ss;
     BSONForEach(elem, options) {
-        if (elem.fieldNameStringData() == "configString") {
+        if (elem.fieldNameStringData() == WiredTigerUtil::kConfigStringField) {
             Status status = WiredTigerUtil::checkTableCreationOptions(elem);
             if (!status.isOK()) {
                 return status;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 048cac044d2..8bbc04c7019 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -2763,4 +2763,30 @@ KeyFormat WiredTigerKVEngine::getKeyFormat(OperationContext* opCtx, StringData i
     return wtTableConfig.find("key_format=u") != string::npos ? KeyFormat::String : KeyFormat::Long;
 }
 
+StatusWith<BSONObj> WiredTigerKVEngine::getSanitizedStorageOptionsForSecondaryReplication(
+    const BSONObj& options) const {
+
+    if (options.isEmpty()) {
+        return options;
+    }
+
+    auto firstElem = options.firstElement();
+    if (firstElem.fieldName() != kWiredTigerEngineName) {
+        return Status(ErrorCodes::InvalidOptions,
+                      str::stream() << "Expected \"" << kWiredTigerEngineName
+                                    << "\" field, but got: " << firstElem.fieldName());
+    }
+
+    BSONObj wtObj = firstElem.Obj();
+    if (auto configStringElem = wtObj.getField(WiredTigerUtil::kConfigStringField)) {
+        auto configString = configStringElem.String();
+        WiredTigerUtil::removeEncryptionFromConfigString(&configString);
+        // Return a new BSONObj with the configString field sanitized.
+        return options.addFields(BSON(kWiredTigerEngineName << wtObj.addFields(BSON(
+                                          WiredTigerUtil::kConfigStringField << configString))));
+    }
+
+    return options;
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index add1d9a9b29..52e1731b6ae 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -385,6 +385,9 @@ public:
 
     KeyFormat getKeyFormat(OperationContext* opCtx, StringData ident) const override;
 
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const override;
+
 private:
     class WiredTigerSessionSweeper;
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index 5e3765d6096..359ac8318b2 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -648,7 +648,7 @@ void WiredTigerRecordStore::OplogStones::adjust(int64_t maxSize) {
 StatusWith<std::string> WiredTigerRecordStore::parseOptionsField(const BSONObj options) {
     StringBuilder ss;
     BSONForEach(elem, options) {
-        if (elem.fieldNameStringData() == "configString") {
+        if (elem.fieldNameStringData() == WiredTigerUtil::kConfigStringField) {
             Status status = WiredTigerUtil::checkTableCreationOptions(elem);
             if (!status.isOK()) {
                 return status;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index 1aa112ec5a7..d374b285d8d 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -37,6 +37,7 @@
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
+#include <pcrecpp.h>
 
 #include "mongo/base/simple_string_data_comparator.h"
 #include "mongo/bson/bsonobjbuilder.h"
@@ -464,7 +465,7 @@ StatusWith<int64_t> WiredTigerUtil::checkApplicationMetadataFormatVersion(Operat
 
 // static
 Status WiredTigerUtil::checkTableCreationOptions(const BSONElement& configElem) {
-    invariant(configElem.fieldNameStringData() == "configString");
+    invariant(configElem.fieldNameStringData() == WiredTigerUtil::kConfigStringField);
 
     if (configElem.type() != String) {
         return {ErrorCodes::TypeMismatch, "'configString' must be a string."};
@@ -1190,5 +1191,9 @@ std::string WiredTigerUtil::generateWTVerboseConfiguration() {
     return cfg;
 }
 
+void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) {
+    static const StaticImmortal<pcrecpp::RE> encryptionOptsRegex(R"re(encryption=\([^\)]*\),?)re");
+    encryptionOptsRegex->GlobalReplace("", configString);
+}
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index dbca93e6bc6..618add61bf8 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -158,6 +158,8 @@ private:
     WiredTigerUtil();
 
 public:
+    static constexpr StringData kConfigStringField = "configString"_sd;
+
     /**
      * Fetch the type and source fields out of the colgroup metadata.  'tableUri' must be a
      * valid table: uri.
@@ -332,6 +334,14 @@ public:
     template <typename T>
     static T castStatisticsValue(uint64_t statisticsValue);
 
+    /**
+     * Removes encryption configuration from a config string. Should only be applied on custom
+     * config strings on secondaries. Fixes an issue where encryption configuration might be
+     * replicated to non-encrypted nodes, or nodes with different encryption options, causing
+     * initial sync or replication to fail. See SERVER-68122.
+     */
+    static void removeEncryptionFromConfigString(std::string* configString);
+
 private:
     /**
      * Casts unsigned 64-bit statistics value to T.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
index 088fb820474..099dc5ce453 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
@@ -452,4 +452,67 @@ TEST(WiredTigerUtilTest, GenerateVerboseConfiguration) {
     }
 }
 
+TEST(WiredTigerUtilTest, RemoveEncryptionFromConfigString) {
+    {  // Found at the middle.
+        std::string input{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),encryption=(name=AES256-CBC,"
+            "keyid="
+            "\".system\"),extensions=[local={entry=mongo_addWiredTigerEncryptors,early_load=true},,"
+            "],"};
+        const std::string expectedOutput{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),extensions=[local={entry=mongo_"
+            "addWiredTigerEncryptors,early_load=true},,],"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Found at start.
+        std::string input{
+            "encryption=(name=AES256-CBC,keyid=\".system\"),extensions=[local={entry=mongo_"
+            "addWiredTigerEncryptors,early_load=true},,],"};
+        const std::string expectedOutput{
+            "extensions=[local={entry=mongo_addWiredTigerEncryptors,early_load=true},,],"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Found at the end.
+        std::string input{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),encryption=(name=AES256-CBC,"
+            "keyid=\".system\")"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4),"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Matches full configString.
+        std::string input{"encryption=(name=AES256-CBC,keyid=\".system\")"};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Matches full configString, trailing comma.
+        std::string input{"encryption=(name=AES256-CBC,keyid=\".system\"),"};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // No match.
+        std::string input{"debug_mode=(table_logging=true,checkpoint_retention=4)"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4)"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // No match, empty.
+        std::string input{""};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Removes multiple instances.
+        std::string input{
+            "encryption=(name=AES256-CBC,keyid=\".system\"),debug_mode=(table_logging=true,"
+            "checkpoint_retention=4),encryption=(name=AES256-CBC,keyid=\".system\")"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4),"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+}
 }  // namespace mongo