SERVER-68122 Remove encryption from storageOptions on secondary replication

(cherry picked from commit ef120ac5552574fb9b84d36d842ead8519588c31)

15 files changed, 209 insertions, 7 deletions

diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp
index 35b1ddf6d7b..100f9ade6ff 100644
--- a/src/mongo/db/repl/collection_cloner.cpp
+++ b/src/mongo/db/repl/collection_cloner.cpp
@@ -29,6 +29,8 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplicationInitialSync
 
+#include "mongo/db/index/index_descriptor_fwd.h"
+#include "mongo/db/service_context.h"
 #include "mongo/platform/basic.h"
 
 #include "mongo/base/string_data.h"
@@ -200,8 +202,20 @@ BaseCloner::AfterStageBehavior CollectionCloner::listIndexesStage() {
                       "source"_attr = getSource());
     }
 
+    const auto storageEngine = getGlobalServiceContext()->getStorageEngine();
     // Parse the index specs into their respective state, ready or unfinished.
     for (auto&& spec : indexSpecs) {
+        // Sanitize storage engine options to remove options which might not apply to this node. See
+        // SERVER-68122.
+        if (auto storageEngineElem = spec.getField(IndexDescriptor::kStorageEngineFieldName)) {
+            auto sanitizedStorageEngineOpts =
+                storageEngine->getSanitizedStorageOptionsForSecondaryReplication(
+                    storageEngineElem.embeddedObject());
+            fassert(6812200, sanitizedStorageEngineOpts);
+            spec = spec.addFields(BSON(IndexDescriptor::kStorageEngineFieldName
+                                       << sanitizedStorageEngineOpts.getValue()));
+        }
+
         if (spec.hasField("buildUUID")) {
             _unfinishedIndexSpecs.push_back(spec.getOwned());
         } else if (spec.hasField("name") && spec.getStringField("name") == "_id_"_sd) {
diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp
index adeeeb0afb6..23da9db918a 100644
--- a/src/mongo/db/repl/database_cloner.cpp
+++ b/src/mongo/db/repl/database_cloner.cpp
@@ -29,6 +29,7 @@
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplicationInitialSync
 
+#include "mongo/db/service_context.h"
 #include "mongo/platform/basic.h"
 
 #include "mongo/base/string_data.h"
@@ -69,6 +70,7 @@ BaseCloner::AfterStageBehavior DatabaseCloner::listCollectionsStage() {
     BSONObj res;
     auto collectionInfos =
         getClient()->getCollectionInfos(_dbName, ListCollectionsFilter::makeTypeCollectionFilter());
+    const auto storageEngine = getGlobalServiceContext()->getStorageEngine();
 
     stdx::unordered_set<std::string> seen;
     for (auto&& info : collectionInfos) {
@@ -104,6 +106,13 @@ BaseCloner::AfterStageBehavior DatabaseCloner::listCollectionsStage() {
                               << "'" << result.getName() << "': " << info,
                 isDuplicate);
 
+        // Sanitize storage engine options to remove options which might not apply to this node. See
+        // SERVER-68122.
+        auto sanitizedStorageOptions =
+            uassertStatusOK(storageEngine->getSanitizedStorageOptionsForSecondaryReplication(
+                result.getOptions().storageEngine));
+        result.getOptions().storageEngine = sanitizedStorageOptions;
+
         // While UUID is a member of CollectionOptions, listCollections does not return the
         // collectionUUID there as part of the options, but instead places it in the 'info' field.
         // We need to move it back to CollectionOptions to create the collection properly.
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 1053b8d275d..e2bb12b3c61 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -759,6 +759,23 @@ NamespaceString extractNsFromUUIDorNs(OperationContext* opCtx,
     return ui ? extractNsFromUUID(opCtx, ui.get()) : extractNs(ns.db(), cmd);
 }
 
+StatusWith<BSONObj> getObjWithSanitizedStorageEngineOptions(OperationContext* opCtx,
+                                                            const BSONObj& cmd) {
+
+    if (auto storageEngineElem = cmd[IndexDescriptor::kStorageEngineFieldName]) {
+        auto storageEngine = opCtx->getServiceContext()->getStorageEngine();
+        auto engineObj = storageEngineElem.embeddedObject();
+        auto sanitizedObj =
+            storageEngine->getSanitizedStorageOptionsForSecondaryReplication(engineObj);
+        if (!sanitizedObj.isOK()) {
+            return sanitizedObj.getStatus();
+        }
+        return cmd.addFields(
+            BSON(IndexDescriptor::kStorageEngineFieldName << sanitizedObj.getValue()));
+    }
+    return cmd;
+}
+
 using OpApplyFn = std::function<Status(
     OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode)>;
 
@@ -783,7 +800,14 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
     {"create",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
           const auto& ui = entry.getUuid();
-          const auto& cmd = entry.getObject();
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          const auto sanitizedCmdOrStatus =
+              getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
+          if (!sanitizedCmdOrStatus.isOK()) {
+              return sanitizedCmdOrStatus.getStatus();
+          }
+          const auto& cmd = sanitizedCmdOrStatus.getValue();
           const NamespaceString nss(extractNs(entry.getNss().db(), cmd));
 
           // Mode SECONDARY steady state replication should not allow create collection to rename an
@@ -827,7 +851,15 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
       {ErrorCodes::NamespaceExists}}},
     {"createIndexes",
      {[](OperationContext* opCtx, const OplogEntry& entry, OplogApplication::Mode mode) -> Status {
-          const auto& cmd = entry.getObject();
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          const auto sanitizedCmdOrStatus =
+              getObjWithSanitizedStorageEngineOptions(opCtx, entry.getObject());
+          if (!sanitizedCmdOrStatus.isOK()) {
+              return sanitizedCmdOrStatus.getStatus();
+          }
+          const auto& cmd = sanitizedCmdOrStatus.getValue();
+
           if (OplogApplication::Mode::kApplyOpsCmd == mode) {
               return {ErrorCodes::CommandNotSupported,
                       "The createIndexes operation is not supported in applyOps mode"};
@@ -862,6 +894,16 @@ const StringMap<ApplyOpMetadata> kOpsMap = {
                   "Error parsing 'startIndexBuild' oplog entry");
           }
 
+          // Sanitize storage engine options to remove options which might not apply to this node.
+          // See SERVER-68122.
+          for (auto& spec : swOplogEntry.getValue().indexSpecs) {
+              auto sanitizedObj = getObjWithSanitizedStorageEngineOptions(opCtx, spec);
+              if (!sanitizedObj.isOK()) {
+                  return swOplogEntry.getStatus();
+              }
+              spec = sanitizedObj.getValue();
+          }
+
           IndexBuildsCoordinator::ApplicationMode applicationMode =
               IndexBuildsCoordinator::ApplicationMode::kNormal;
           if (mode == OplogApplication::Mode::kInitialSync) {
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 89faafe5fd0..601c85f285a 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -391,6 +391,16 @@ public:
     virtual void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) = 0;
 
     /**
+     * Returns the input storage engine options, sanitized to remove options that may not apply to
+     * this node, such as encryption. Might be called for both collection and index options. See
+     * SERVER-68122.
+     */
+    virtual StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const {
+        return options;
+    }
+
+    /**
      * The destructor will never be called from mongod, but may be called from tests.
      * Engines may assume that this will only be called in the case of clean shutdown, even if
      * cleanShutdown() hasn't been called.
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 4d4fb703539..69ff3ec28df 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -694,6 +694,14 @@ public:
      * it.
      */
     virtual void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) = 0;
+
+    /**
+     * Returns the input storage engine options, sanitized to remove options that may not apply to
+     * this node, such as encryption. Might be called for both collection and index options. See
+     * SERVER-68122.
+     */
+    virtual StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const = 0;
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 6ef32e5b376..a5b0926d3c4 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -1280,4 +1280,8 @@ void StorageEngineImpl::setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp
     _engine->setPinnedOplogTimestamp(pinnedTimestamp);
 }
 
+StatusWith<BSONObj> StorageEngineImpl::getSanitizedStorageOptionsForSecondaryReplication(
+    const BSONObj& options) const {
+    return _engine->getSanitizedStorageOptionsForSecondaryReplication(options);
+}
 }  // namespace mongo
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 5731d3c934d..d73de350cde 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -368,9 +368,10 @@ public:
 
     void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) override;
 
-private:
-    using CollIter = std::list<std::string>::iterator;
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const override;
 
+private:
     void _initCollection(OperationContext* opCtx,
                          RecordId catalogId,
                          const NamespaceString& nss,
diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h
index cdeb564a004..cf02ffbf765 100644
--- a/src/mongo/db/storage/storage_engine_mock.h
+++ b/src/mongo/db/storage/storage_engine_mock.h
@@ -202,6 +202,11 @@ public:
     void unpinOldestTimestamp(const std::string& requestingServiceName) final {}
 
     void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) final {}
+
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const final {
+        return options;
+    }
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index ad83073ba04..298e06fe4a1 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -121,7 +121,7 @@ void WiredTigerIndex::getKey(OperationContext* opCtx, WT_CURSOR* cursor, WT_ITEM
 StatusWith<std::string> WiredTigerIndex::parseIndexOptions(const BSONObj& options) {
     StringBuilder ss;
     BSONForEach(elem, options) {
-        if (elem.fieldNameStringData() == "configString") {
+        if (elem.fieldNameStringData() == WiredTigerUtil::kConfigStringField) {
             Status status = WiredTigerUtil::checkTableCreationOptions(elem);
             if (!status.isOK()) {
                 return status;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index f5f69897479..94bce0c7423 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -2513,4 +2513,30 @@ std::uint64_t WiredTigerKVEngine::_getCheckpointTimestamp() const {
     return tmp;
 }
 
+StatusWith<BSONObj> WiredTigerKVEngine::getSanitizedStorageOptionsForSecondaryReplication(
+    const BSONObj& options) const {
+
+    if (options.isEmpty()) {
+        return options;
+    }
+
+    auto firstElem = options.firstElement();
+    if (firstElem.fieldName() != kWiredTigerEngineName) {
+        return Status(ErrorCodes::InvalidOptions,
+                      str::stream() << "Expected \"" << kWiredTigerEngineName
+                                    << "\" field, but got: " << firstElem.fieldName());
+    }
+
+    BSONObj wtObj = firstElem.Obj();
+    if (auto configStringElem = wtObj.getField(WiredTigerUtil::kConfigStringField)) {
+        auto configString = configStringElem.String();
+        WiredTigerUtil::removeEncryptionFromConfigString(&configString);
+        // Return a new BSONObj with the configString field sanitized.
+        return options.addFields(BSON(kWiredTigerEngineName << wtObj.addFields(BSON(
+                                          WiredTigerUtil::kConfigStringField << configString))));
+    }
+
+    return options;
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 11766b6a80a..11a35b341b1 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -354,6 +354,9 @@ public:
 
     void setPinnedOplogTimestamp(const Timestamp& pinnedTimestamp) override;
 
+    StatusWith<BSONObj> getSanitizedStorageOptionsForSecondaryReplication(
+        const BSONObj& options) const override;
+
 private:
     class WiredTigerSessionSweeper;
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
index e84f2a2c14e..c880042a7c9 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp
@@ -645,7 +645,7 @@ void WiredTigerRecordStore::OplogStones::adjust(int64_t maxSize) {
 StatusWith<std::string> WiredTigerRecordStore::parseOptionsField(const BSONObj options) {
     StringBuilder ss;
     BSONForEach(elem, options) {
-        if (elem.fieldNameStringData() == "configString") {
+        if (elem.fieldNameStringData() == WiredTigerUtil::kConfigStringField) {
             Status status = WiredTigerUtil::checkTableCreationOptions(elem);
             if (!status.isOK()) {
                 return status;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index ad1484435bd..296dfb152ee 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -37,6 +37,7 @@
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/path.hpp>
+#include <pcrecpp.h>
 
 #include "mongo/base/simple_string_data_comparator.h"
 #include "mongo/bson/bsonobjbuilder.h"
@@ -51,6 +52,7 @@
 #include "mongo/util/assert_util.h"
 #include "mongo/util/processinfo.h"
 #include "mongo/util/scopeguard.h"
+#include "mongo/util/static_immortal.h"
 #include "mongo/util/str.h"
 
 namespace mongo {
@@ -346,7 +348,7 @@ StatusWith<int64_t> WiredTigerUtil::checkApplicationMetadataFormatVersion(Operat
 
 // static
 Status WiredTigerUtil::checkTableCreationOptions(const BSONElement& configElem) {
-    invariant(configElem.fieldNameStringData() == "configString");
+    invariant(configElem.fieldNameStringData() == WiredTigerUtil::kConfigStringField);
 
     if (configElem.type() != String) {
         return {ErrorCodes::TypeMismatch, "'configString' must be a string."};
@@ -858,4 +860,9 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine,
     settings.append("min pinned timestamp", minPinned);
 }
 
+void WiredTigerUtil::removeEncryptionFromConfigString(std::string* configString) {
+    static const StaticImmortal<pcrecpp::RE> encryptionOptsRegex(R"re(encryption=\([^\)]*\),?)re");
+    encryptionOptsRegex->GlobalReplace("", configString);
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index ff936dd9ee5..ce9578c5cd5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -156,6 +156,8 @@ private:
     WiredTigerUtil();
 
 public:
+    static constexpr StringData kConfigStringField = "configString"_sd;
+
     /**
      * Fetch the type and source fields out of the colgroup metadata.  'tableUri' must be a
      * valid table: uri.
@@ -310,6 +312,14 @@ public:
     template <typename T>
     static T castStatisticsValue(uint64_t statisticsValue);
 
+    /**
+     * Removes encryption configuration from a config string. Should only be applied on custom
+     * config strings on secondaries. Fixes an issue where encryption configuration might be
+     * replicated to non-encrypted nodes, or nodes with different encryption options, causing
+     * initial sync or replication to fail. See SERVER-68122.
+     */
+    static void removeEncryptionFromConfigString(std::string* configString);
+
 private:
     /**
      * Casts unsigned 64-bit statistics value to T.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
index 2fab1cb3e96..9706e7096f9 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util_test.cpp
@@ -348,4 +348,67 @@ TEST(WiredTigerUtilTest, GetStatisticsValueValidKey) {
     ASSERT_EQUALS(0U, result.getValue());
 }
 
+TEST(WiredTigerUtilTest, RemoveEncryptionFromConfigString) {
+    {  // Found at the middle.
+        std::string input{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),encryption=(name=AES256-CBC,"
+            "keyid="
+            "\".system\"),extensions=[local={entry=mongo_addWiredTigerEncryptors,early_load=true},,"
+            "],"};
+        const std::string expectedOutput{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),extensions=[local={entry=mongo_"
+            "addWiredTigerEncryptors,early_load=true},,],"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Found at start.
+        std::string input{
+            "encryption=(name=AES256-CBC,keyid=\".system\"),extensions=[local={entry=mongo_"
+            "addWiredTigerEncryptors,early_load=true},,],"};
+        const std::string expectedOutput{
+            "extensions=[local={entry=mongo_addWiredTigerEncryptors,early_load=true},,],"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Found at the end.
+        std::string input{
+            "debug_mode=(table_logging=true,checkpoint_retention=4),encryption=(name=AES256-CBC,"
+            "keyid=\".system\")"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4),"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Matches full configString.
+        std::string input{"encryption=(name=AES256-CBC,keyid=\".system\")"};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Matches full configString, trailing comma.
+        std::string input{"encryption=(name=AES256-CBC,keyid=\".system\"),"};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // No match.
+        std::string input{"debug_mode=(table_logging=true,checkpoint_retention=4)"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4)"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // No match, empty.
+        std::string input{""};
+        const std::string expectedOutput{""};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+    {  // Removes multiple instances.
+        std::string input{
+            "encryption=(name=AES256-CBC,keyid=\".system\"),debug_mode=(table_logging=true,"
+            "checkpoint_retention=4),encryption=(name=AES256-CBC,keyid=\".system\")"};
+        const std::string expectedOutput{"debug_mode=(table_logging=true,checkpoint_retention=4),"};
+        WiredTigerUtil::removeEncryptionFromConfigString(&input);
+        ASSERT_EQUALS(input, expectedOutput);
+    }
+}
 }  // namespace mongo