SERVER-63199 Add new compact skeleton for mongod

10 files changed, 377 insertions, 2 deletions

diff --git a/jstests/auth/lib/commands_lib.js b/jstests/auth/lib/commands_lib.js
index fa4c349abcd..ef63b600a56 100644
--- a/jstests/auth/lib/commands_lib.js
+++ b/jstests/auth/lib/commands_lib.js
@@ -2718,6 +2718,45 @@ var authCommandsLib = {
           ]
         },
         {
+          testname: "compactStructuredEncryptionData",
+          command: {compactStructuredEncryptionData: "foo", compactionTokens : {}},
+          skipSharded: true,
+          //skipUnlessReplicaSet: true,
+          skipTest:
+              (conn) => !TestData.setParameters.featureFlagFLE2,
+          setup: function(db) {
+              assert.commandWorked(db.createCollection("foo", {
+                encryptedFields: {
+                    "fields": [
+                        {
+                            "path": "firstName",
+                            "keyId": UUID("11d58b8a-0c6c-4d69-a0bd-70c6d9befae9"),
+                            "bsonType": "string",
+                            "queries": {"queryType": "equality"}
+                        },
+                    ]
+                }
+              }));
+          },
+          teardown: function(db) {
+              assert.commandWorked(db.dropDatabase());
+          },
+          testcases: [
+              {
+                runOnDb: firstDbName,
+                roles: { readWrite : 1, readWriteAnyDatabase : 1, dbOwner : 1, root : 1, __system : 1 },
+                privileges:
+                    [{resource: {db: firstDbName, collection: "foo"}, actions: ["compactStructuredEncryptionData"]}]
+              },
+              {
+                runOnDb: secondDbName,
+                roles: { readWriteAnyDatabase : 1, root : 1, __system : 1 },
+                privileges:
+                    [{resource: {db: secondDbName, collection: "foo"}, actions: ["compactStructuredEncryptionData"]}]
+              }
+          ]
+        },
+        {
           testname: "connectionStatus",
           command: {connectionStatus: 1},
           testcases: [
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index f0f800da668..8605f0d95c5 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -223,6 +223,11 @@ let viewsCommandTests = {
     commitReshardCollection: {skip: isUnrelated},
     commitTransaction: {skip: isUnrelated},
     compact: {command: {compact: "view", force: true}, expectFailure: true, skipSharded: true},
+    compactStructuredEncryptionData: {
+        command: {compactStructuredEncryptionData: "view", compactionTokens: {}},
+        expectFailure: true,
+        skipSharded: true
+    },
     configureFailPoint: {skip: isUnrelated},
     configureCollectionAutoSplitter: {
         skip: isUnrelated
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index aabcdf83a5a..b5a80270ad5 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -122,6 +122,7 @@ const allCommands = {
     commitReshardCollection: {skip: isPrimaryOnly},
     commitTransaction: {skip: isPrimaryOnly},
     compact: {skip: isNotAUserDataRead},
+    compactStructuredEncryptionData: {skip: isPrimaryOnly},
     configureFailPoint: {skip: isNotAUserDataRead},
     configureCollectionBalancing: {skip: isPrimaryOnly},
     connPoolStats: {skip: isNotAUserDataRead},
diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js
index caa0f5b1451..1b31046b4b1 100644
--- a/jstests/sharding/read_write_concern_defaults_application.js
+++ b/jstests/sharding/read_write_concern_defaults_application.js
@@ -270,6 +270,7 @@ let testCases = {
         useLogs: true,
     },
     compact: {skip: "does not accept read or write concern"},
+    compactStructuredEncryptionData: {skip: "does not accept read or write concern"},
     configureCollectionAutoSplitter: {
         skip: "does not accept read or write concern"
     },  // TODO SERVER-62374: remove this once 5.3 becomes last continuos release
diff --git a/src/mongo/db/auth/action_type.idl b/src/mongo/db/auth/action_type.idl
index b861ff4d78e..2c48f890b28 100644
--- a/src/mongo/db/auth/action_type.idl
+++ b/src/mongo/db/auth/action_type.idl
@@ -67,6 +67,7 @@ enums:
            collMod :  "collMod"
            collStats :  "collStats"
            compact :  "compact"
+           compactStructuredEncryptionData: "compactStructuredEncryptionData"
            connPoolStats :  "connPoolStats"
            connPoolSync :  "connPoolSync"
            convertToCapped :  "convertToCapped"
@@ -221,6 +222,7 @@ enums:
                         - changeStream
                         - collMod
                         - collStats
+                        - compactStructuredEncryptionData
                         - convertToCapped
                         - createCollection
                         - createIndex
@@ -258,6 +260,7 @@ enums:
                         - changeStream
                         - collMod
                         - collStats
+                        - compactStructuredEncryptionData
                         - convertToCapped
                         - createCollection
                         - createIndex
diff --git a/src/mongo/db/auth/builtin_roles.cpp b/src/mongo/db/auth/builtin_roles.cpp
index 9cc88c20cb3..7faee09e529 100644
--- a/src/mongo/db/auth/builtin_roles.cpp
+++ b/src/mongo/db/auth/builtin_roles.cpp
@@ -123,11 +123,12 @@ MONGO_INITIALIZER(AuthorizationBuiltinRoles)(InitializerContext* context) {
     // Read-write role
     readWriteRoleActions += readRoleActions;
     readWriteRoleActions
+        << ActionType::compactStructuredEncryptionData
         << ActionType::convertToCapped  // db admin gets this also
         << ActionType::createCollection  // db admin gets this also
+        << ActionType::createIndex
         << ActionType::dropCollection
         << ActionType::dropIndex
-        << ActionType::createIndex
         << ActionType::insert
         << ActionType::remove
         << ActionType::renameCollectionSameDB  // db admin gets this also
diff --git a/src/mongo/db/commands/SConscript b/src/mongo/db/commands/SConscript
index 9bf3bcd5956..dd18f866d11 100644
--- a/src/mongo/db/commands/SConscript
+++ b/src/mongo/db/commands/SConscript
@@ -226,6 +226,19 @@ env.Library(
 )
 
 env.Library(
+    target="fle2_compact",
+    source=[
+        'fle2_compact.cpp',
+        'fle2_compact.idl',
+    ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/catalog/catalog_helpers',
+        '$BUILD_DIR/mongo/db/catalog_raii',
+        '$BUILD_DIR/mongo/db/commands',
+    ]
+)
+
+env.Library(
     target="mongod_fsync",
     source=[
         "fsync.cpp",
@@ -557,6 +570,7 @@ env.Library(
         '$BUILD_DIR/mongo/idl/idl_parser',
         '$BUILD_DIR/mongo/util/net/ssl_manager',
         'core',
+        'fle2_compact',
         'kill_common',
         'map_reduce_agg',
         'mongod_fcv',
@@ -773,4 +787,4 @@ env.CppUnitTest(
         "servers",
         "standalone",
     ],
-)
\ No newline at end of file
+)
diff --git a/src/mongo/db/commands/fle2_compact.cpp b/src/mongo/db/commands/fle2_compact.cpp
new file mode 100644
index 00000000000..9b8c21a2b14
--- /dev/null
+++ b/src/mongo/db/commands/fle2_compact.cpp
@@ -0,0 +1,185 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/commands/fle2_compact.h"
+
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/catalog/rename_collection.h"
+#include "mongo/db/catalog_raii.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/views/view_catalog.h"
+#include "mongo/logv2/log.h"
+
+namespace mongo {
+namespace {
+
+class CompactStructuredEncryptionDataCmd final
+    : public TypedCommand<CompactStructuredEncryptionDataCmd> {
+public:
+    using Request = CompactStructuredEncryptionData;
+    using Reply = CompactStructuredEncryptionData::Reply;
+    using TC = TypedCommand<CompactStructuredEncryptionDataCmd>;
+
+    class Invocation final : public TC::InvocationBase {
+    public:
+        using TC::InvocationBase::InvocationBase;
+        using TC::InvocationBase::request;
+
+        Reply typedRun(OperationContext* opCtx) {
+            auto swCompactStats = compactEncryptedCompactionCollection(opCtx, request());
+            uassertStatusOK(swCompactStats);
+            return Reply(swCompactStats.getValue());
+        }
+
+    private:
+        bool supportsWriteConcern() const final {
+            return false;
+        }
+
+        void doCheckAuthorization(OperationContext* opCtx) const final {
+            auto* as = AuthorizationSession::get(opCtx->getClient());
+            uassert(ErrorCodes::Unauthorized,
+                    "Not authorized to compact structured encryption data",
+                    as->isAuthorizedForActionsOnResource(
+                        ResourcePattern::forExactNamespace(request().getNamespace()),
+                        ActionType::compactStructuredEncryptionData));
+        }
+
+        NamespaceString ns() const final {
+            return request().getNamespace();
+        }
+    };
+
+    typename TC::AllowedOnSecondary secondaryAllowed(ServiceContext*) const final {
+        return BasicCommand::AllowedOnSecondary::kNever;
+    }
+
+    bool adminOnly() const final {
+        return false;
+    }
+} compactStructuredEncryptionDataCmd;
+
+}  // namespace
+
+StatusWith<EncryptedStateCollectionsNamespaces>
+EncryptedStateCollectionsNamespaces::createFromDataCollection(const Collection& edc) {
+    if (!edc.getCollectionOptions().encryptedFieldConfig) {
+        return Status(ErrorCodes::BadValue,
+                      str::stream() << "Encrypted data collection " << edc.ns()
+                                    << " is missing encrypted fields metadata");
+    }
+
+    auto& cfg = *(edc.getCollectionOptions().encryptedFieldConfig);
+    auto db = edc.ns().db();
+    StringData missingColl;
+    EncryptedStateCollectionsNamespaces namespaces;
+
+    auto f = [&missingColl](StringData coll) {
+        missingColl = coll;
+        return StringData();
+    };
+
+    namespaces.escNss =
+        NamespaceString(db, cfg.getEscCollection().value_or_eval([&f]() { return f("state"_sd); }));
+    namespaces.eccNss =
+        NamespaceString(db, cfg.getEccCollection().value_or_eval([&f]() { return f("cache"_sd); }));
+    namespaces.ecocNss = NamespaceString(
+        db, cfg.getEcocCollection().value_or_eval([&f]() { return f("compaction"_sd); }));
+
+    if (!missingColl.empty()) {
+        return Status(ErrorCodes::BadValue,
+                      str::stream()
+                          << "Encrypted data collection " << edc.ns()
+                          << " is missing the name of its " << missingColl << " collection");
+    }
+
+    namespaces.ecocRenameNss =
+        NamespaceString(db, namespaces.ecocNss.coll().toString().append(".compact"));
+    return namespaces;
+}
+
+StatusWith<CompactStats> compactEncryptedCompactionCollection(
+    OperationContext* opCtx, const CompactStructuredEncryptionData& request) {
+    mongo::ECOCStats ecocStats(0, 0);
+    mongo::ECStats eccStats(0, 0, 0, 0);
+    mongo::ECStats escStats(0, 0, 0, 0);
+    const auto& edcNss = request.getNamespace();
+
+    LOGV2(6319900, "Compacting the encrypted compaction collection", "namespace"_attr = edcNss);
+
+    AutoGetDb autoDb(opCtx, edcNss.db(), MODE_IX);
+    if (!autoDb.getDb()) {
+        return Status(ErrorCodes::NamespaceNotFound, str::stream() << "database does not exist");
+    }
+
+    auto catalog = CollectionCatalog::get(opCtx);
+    Lock::CollectionLock edcLock(opCtx, edcNss, MODE_IS);
+
+    // Check the data collection exists and is not a view
+    auto edc = catalog->lookupCollectionByNamespace(opCtx, edcNss);
+    if (!edc) {
+        if (ViewCatalog::get(opCtx)->lookup(opCtx, edcNss)) {
+            return Status(ErrorCodes::CommandNotSupportedOnView,
+                          "cannot compact structured encryption data on a view");
+        }
+        return Status(ErrorCodes::NamespaceNotFound, "collection does not exist");
+    }
+
+    auto swNamespaces = EncryptedStateCollectionsNamespaces::createFromDataCollection(*edc.get());
+    if (!swNamespaces.isOK()) {
+        return swNamespaces.getStatus();
+    }
+    auto& namespaces = swNamespaces.getValue();
+
+    auto ecoc = catalog->lookupCollectionByNamespace(opCtx, namespaces.ecocNss);
+    auto ecocRename = catalog->lookupCollectionByNamespace(opCtx, namespaces.ecocRenameNss);
+
+    if (ecoc && !ecocRename) {
+        LOGV2(6319901,
+              "Renaming the encrypted compaction collection",
+              "ecocNss"_attr = namespaces.ecocNss,
+              "ecocRenameNss"_attr = namespaces.ecocRenameNss);
+        RenameCollectionOptions renameOpts;
+        validateAndRunRenameCollection(
+            opCtx, namespaces.ecocNss, namespaces.ecocRenameNss, renameOpts);
+        ecoc.reset();
+    }
+
+
+    LOGV2(6319902,
+          "Done compacting the encrypted compaction collection",
+          "namespace"_attr = request.getNamespace());
+    return CompactStats(ecocStats, eccStats, escStats);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/commands/fle2_compact.h b/src/mongo/db/commands/fle2_compact.h
new file mode 100644
index 00000000000..35308325169
--- /dev/null
+++ b/src/mongo/db/commands/fle2_compact.h
@@ -0,0 +1,52 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/status_with.h"
+#include "mongo/db/commands/fle2_compact_gen.h"
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+
+struct EncryptedStateCollectionsNamespaces {
+
+    static StatusWith<EncryptedStateCollectionsNamespaces> createFromDataCollection(
+        const Collection& edc);
+
+    NamespaceString escNss;
+    NamespaceString eccNss;
+    NamespaceString ecocNss;
+    NamespaceString ecocRenameNss;
+};
+
+StatusWith<CompactStats> compactEncryptedCompactionCollection(
+    OperationContext* opCtx, const CompactStructuredEncryptionData& request);
+
+}  // namespace mongo
diff --git a/src/mongo/db/commands/fle2_compact.idl b/src/mongo/db/commands/fle2_compact.idl
new file mode 100644
index 00000000000..6693ce7aee4
--- /dev/null
+++ b/src/mongo/db/commands/fle2_compact.idl
@@ -0,0 +1,74 @@
+# Copyright (C) 2022-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+global:
+    cpp_namespace: "mongo"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+structs:
+    ECOCStats:
+        description: "Stats about records in ECOC compact touched"
+        fields:
+            read: exactInt64
+            deleted: exactInt64
+
+    ECStats:
+        description: "Stats about records in ECC & ESC compact touched"
+        fields:
+            read: exactInt64
+            inserted: exactInt64
+            updated: exactInt64
+            deleted: exactInt64
+
+    CompactStats:
+        description: "Stats about records in ECC, ECOC, and ESC compact touched"
+        fields:
+           ecoc: ECOCStats
+           ecc: ECStats
+           esc: ECStats
+
+    CompactStructuredEncryptionDataCommandReply:
+        description: "Reply from the {compactStructuredEncryptedData: ...} command"
+        strict: true
+        fields:
+            stats: CompactStats
+
+commands:
+    compactStructuredEncryptionData :
+        description: "Parser for the 'compactStructuredEncryptionData' command"
+        command_name: compactStructuredEncryptionData
+        api_version: ""
+        namespace: concatenate_with_db
+        strict: true
+        reply_type: CompactStructuredEncryptionDataCommandReply
+        fields:
+            compactionTokens:
+                description: "Map of field path to EOCToken"
+                type: object