SERVER-65980: Implement compact(), share isEmpty()

7 files changed, 184 insertions, 141 deletions

diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 5ce8b28f002..68a76639ed5 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -35,6 +35,7 @@ wtEnv.Library(
         'wiredtiger_cursor_helpers.cpp',
         'wiredtiger_global_options.cpp',
         'wiredtiger_index.cpp',
+        'wiredtiger_index_util.cpp',
         'wiredtiger_kv_engine.cpp',
         'wiredtiger_oplog_manager.cpp',
         'wiredtiger_parameters.cpp',
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp
index 25d4c623b14..1f034c86e26 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp
@@ -28,20 +28,13 @@
  */
 
 
-#include "mongo/platform/basic.h"
-
-#include "mongo/db/global_settings.h"
-#include "mongo/db/repl/repl_settings.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_column_store.h"
+#include "mongo/db/global_settings.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_cursor.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_cursor_helpers.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_index_cursor_generic.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h"
 #include "mongo/logv2/log.h"
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage
@@ -411,16 +404,7 @@ std::unique_ptr<ColumnStore::BulkBuilder> WiredTigerColumnStore::makeBulkBuilder
 }
 
 bool WiredTigerColumnStore::isEmpty(OperationContext* opCtx) {
-    // TODO: SERVER-65980, this logic could be shared with WiredTigerIndex::isEmpty().
-    WiredTigerCursor curwrap(_uri, _tableId, false, opCtx);
-    WT_CURSOR* c = curwrap.get();
-    if (!c)
-        return true;
-    int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); });
-    if (ret == WT_NOTFOUND)
-        return true;
-    invariantWTOK(ret, c->session);
-    return false;
+    return WiredTigerIndexUtil::isEmpty(opCtx, _uri, _tableId);
 }
 
 long long WiredTigerColumnStore::getSpaceUsedBytes(OperationContext* opCtx) const {
@@ -443,14 +427,13 @@ long long WiredTigerColumnStore::getFreeStorageBytes(OperationContext* opCtx) co
 }
 
 Status WiredTigerColumnStore::compact(OperationContext* opCtx) {
-    // TODO: SERVER-65980.
-    uasserted(ErrorCodes::NotImplemented, "WiredTigerColumnStore::compact");
+    return WiredTigerIndexUtil::compact(opCtx, _uri);
 }
 
 bool WiredTigerColumnStore::appendCustomStats(OperationContext* opCtx,
                                               BSONObjBuilder* output,
                                               double scale) const {
-    return WiredTigerUtil::appendCustomStats(opCtx, output, scale, _uri);
+    return WiredTigerIndexUtil::appendCustomStats(opCtx, output, scale, _uri);
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
index 147c61a7c84..e194d5af7fa 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp
@@ -28,37 +28,14 @@
  */
 
 
-#include "mongo/platform/basic.h"
-
 #include "mongo/db/storage/wiredtiger/wiredtiger_index.h"
 
-#include <fmt/format.h>
-#include <memory>
-#include <set>
-
-#include "mongo/base/checked_cast.h"
-#include "mongo/db/catalog/index_catalog_entry.h"
-#include "mongo/db/catalog/validate_results.h"
-#include "mongo/db/global_settings.h"
-#include "mongo/db/index/index_descriptor.h"
-#include "mongo/db/json.h"
-#include "mongo/db/repl/repl_settings.h"
-#include "mongo/db/service_context.h"
-#include "mongo/db/stats/resource_consumption_metrics.h"
-#include "mongo/db/storage/index_entry_comparison.h"
-#include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_cursor_helpers.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_index_cursor_generic.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h"
 #include "mongo/logv2/log.h"
-#include "mongo/util/assert_util.h"
-#include "mongo/util/fail_point.h"
-#include "mongo/util/hex.h"
-#include "mongo/util/str.h"
 #include "mongo/util/testing_proctor.h"
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage
@@ -83,7 +60,6 @@
 namespace mongo {
 namespace {
 
-MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY);
 MONGO_FAIL_POINT_DEFINE(WTIndexPauseAfterSearchNear);
 
 static const WiredTigerItem emptyItem(nullptr, 0);
@@ -375,7 +351,7 @@ void WiredTigerIndex::fullValidate(OperationContext* opCtx,
 bool WiredTigerIndex::appendCustomStats(OperationContext* opCtx,
                                         BSONObjBuilder* output,
                                         double scale) const {
-    return WiredTigerUtil::appendCustomStats(opCtx, output, scale, _uri);
+    return WiredTigerIndexUtil::appendCustomStats(opCtx, output, scale, _uri);
 }
 
 Status WiredTigerIndex::dupKeyCheck(OperationContext* opCtx, const KeyString::Value& key) {
@@ -393,15 +369,7 @@ Status WiredTigerIndex::dupKeyCheck(OperationContext* opCtx, const KeyString::Va
 }
 
 bool WiredTigerIndex::isEmpty(OperationContext* opCtx) {
-    WiredTigerCursor curwrap(_uri, _tableId, false, opCtx);
-    WT_CURSOR* c = curwrap.get();
-    if (!c)
-        return true;
-    int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); });
-    if (ret == WT_NOTFOUND)
-        return true;
-    invariantWTOK(ret, c->session);
-    return false;
+    return WiredTigerIndexUtil::isEmpty(opCtx, _uri, _tableId);
 }
 
 void WiredTigerIndex::printIndexEntryMetadata(OperationContext* opCtx,
@@ -497,28 +465,7 @@ Status WiredTigerIndex::initAsEmpty(OperationContext* opCtx) {
 }
 
 Status WiredTigerIndex::compact(OperationContext* opCtx) {
-    dassert(opCtx->lockState()->isWriteLocked());
-    WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache();
-    if (!cache->isEphemeral()) {
-        WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession();
-        opCtx->recoveryUnit()->abandonSnapshot();
-        // WT compact prompts WT to take checkpoints, so we need to take the checkpoint lock around
-        // WT compact calls.
-        auto checkpointLock = opCtx->getServiceContext()->getStorageEngine()->getCheckpointLock(
-            opCtx, StorageEngine::CheckpointLock::Mode::kExclusive);
-        int ret = s->compact(s, uri().c_str(), "timeout=0");
-        if (MONGO_unlikely(WTCompactIndexEBUSY.shouldFail())) {
-            ret = EBUSY;
-        }
-
-        if (ret == EBUSY) {
-            return Status(ErrorCodes::Interrupted,
-                          str::stream() << "Compaction interrupted on " << uri().c_str()
-                                        << " due to cache eviction pressure");
-        }
-        invariantWTOK(ret, s);
-    }
-    return Status::OK();
+    return WiredTigerIndexUtil::compact(opCtx, _uri);
 }
 
 boost::optional<RecordId> WiredTigerIndex::_keyExists(OperationContext* opCtx,
@@ -1543,7 +1490,6 @@ void WiredTigerIndexUnique::insertWithRecordIdInValue_forTest(OperationContext*
     // Now create the table key/value, the actual data record.
     WiredTigerItem keyItem(keyString.getBuffer(), keyString.getSize());
 
-    BufBuilder bufBuilder;
     KeyString::Builder valueBuilder(keyString.getVersion(), rid);
     valueBuilder.appendTypeBits(keyString.getTypeBits());
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp
new file mode 100644
index 00000000000..97278595091
--- /dev/null
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp
@@ -0,0 +1,118 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+
+#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h"
+#include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h"
+
+namespace mongo {
+
+MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY);
+
+bool WiredTigerIndexUtil::appendCustomStats(OperationContext* opCtx,
+                                            BSONObjBuilder* output,
+                                            double scale,
+                                            const std::string& uri) {
+    dassert(opCtx->lockState()->isReadLocked());
+    {
+        BSONObjBuilder metadata(output->subobjStart("metadata"));
+        Status status = WiredTigerUtil::getApplicationMetadata(opCtx, uri, &metadata);
+        if (!status.isOK()) {
+            metadata.append("error", "unable to retrieve metadata");
+            metadata.append("code", static_cast<int>(status.code()));
+            metadata.append("reason", status.reason());
+        }
+    }
+    std::string type, sourceURI;
+    WiredTigerUtil::fetchTypeAndSourceURI(opCtx, uri, &type, &sourceURI);
+    StatusWith<std::string> metadataResult = WiredTigerUtil::getMetadataCreate(opCtx, sourceURI);
+    StringData creationStringName("creationString");
+    if (!metadataResult.isOK()) {
+        BSONObjBuilder creationString(output->subobjStart(creationStringName));
+        creationString.append("error", "unable to retrieve creation config");
+        creationString.append("code", static_cast<int>(metadataResult.getStatus().code()));
+        creationString.append("reason", metadataResult.getStatus().reason());
+    } else {
+        output->append(creationStringName, metadataResult.getValue());
+        // Type can be "lsm" or "file"
+        output->append("type", type);
+    }
+
+    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession();
+    WT_SESSION* s = session->getSession();
+    Status status =
+        WiredTigerUtil::exportTableToBSON(s, "statistics:" + uri, "statistics=(fast)", output);
+    if (!status.isOK()) {
+        output->append("error", "unable to retrieve statistics");
+        output->append("code", static_cast<int>(status.code()));
+        output->append("reason", status.reason());
+    }
+    return true;
+}
+
+Status WiredTigerIndexUtil::compact(OperationContext* opCtx, const std::string& uri) {
+    dassert(opCtx->lockState()->isWriteLocked());
+    WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache();
+    if (!cache->isEphemeral()) {
+        WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession();
+        opCtx->recoveryUnit()->abandonSnapshot();
+        // WT compact prompts WT to take checkpoints, so we need to take the checkpoint lock around
+        // WT compact calls.
+        auto checkpointLock = opCtx->getServiceContext()->getStorageEngine()->getCheckpointLock(
+            opCtx, StorageEngine::CheckpointLock::Mode::kExclusive);
+        int ret = s->compact(s, uri.c_str(), "timeout=0");
+        if (MONGO_unlikely(WTCompactIndexEBUSY.shouldFail())) {
+            ret = EBUSY;
+        }
+
+        if (ret == EBUSY) {
+            return Status(ErrorCodes::Interrupted,
+                          str::stream() << "Compaction interrupted on " << uri.c_str()
+                                        << " due to cache eviction pressure");
+        }
+        invariantWTOK(ret, s);
+    }
+    return Status::OK();
+}
+
+bool WiredTigerIndexUtil::isEmpty(OperationContext* opCtx,
+                                  const std::string& uri,
+                                  uint64_t tableId) {
+    WiredTigerCursor curwrap(uri, tableId, false, opCtx);
+    WT_CURSOR* c = curwrap.get();
+    if (!c)
+        return true;
+    int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); });
+    if (ret == WT_NOTFOUND)
+        return true;
+    invariantWTOK(ret, c->session);
+    return false;
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h
new file mode 100644
index 00000000000..e98c5a7b1dc
--- /dev/null
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h
@@ -0,0 +1,56 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/status.h"
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/db/operation_context.h"
+
+namespace mongo {
+
+class WiredTigerIndexUtil {
+    WiredTigerIndexUtil(const WiredTigerIndexUtil&) = delete;
+    WiredTigerIndexUtil& operator=(const WiredTigerIndexUtil&) = delete;
+
+private:
+    WiredTigerIndexUtil();
+
+public:
+    static bool appendCustomStats(OperationContext* opCtx,
+                                  BSONObjBuilder* output,
+                                  double scale,
+                                  const std::string& uri);
+
+    static Status compact(OperationContext* opCtx, const std::string& uri);
+
+    static bool isEmpty(OperationContext* opCtx, const std::string& uri, uint64_t tableId);
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index 96974706972..321f814a89f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -28,36 +28,22 @@
  */
 
 
-#include "mongo/platform/basic.h"
-
 #include "mongo/db/storage/wiredtiger/wiredtiger_util.h"
 
-#include <limits>
-
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
-#include <boost/filesystem/path.hpp>
 
 #include "mongo/base/simple_string_data_comparator.h"
-#include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/bson/json.h"
 #include "mongo/db/concurrency/exception_util.h"
 #include "mongo/db/concurrency/exception_util_gen.h"
 #include "mongo/db/global_settings.h"
 #include "mongo/db/server_options_general_gen.h"
 #include "mongo/db/snapshot_window_options_gen.h"
-#include "mongo/db/storage/storage_file_util.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_parameters_gen.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h"
-#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h"
 #include "mongo/logv2/log.h"
-#include "mongo/util/assert_util.h"
 #include "mongo/util/processinfo.h"
-#include "mongo/util/scopeguard.h"
-#include "mongo/util/static_immortal.h"
-#include "mongo/util/str.h"
 #include "mongo/util/testing_proctor.h"
 
 #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kWiredTiger
@@ -562,48 +548,6 @@ size_t WiredTigerUtil::getCacheSizeMB(double requestedCacheSizeGB) {
     return static_cast<size_t>(cacheSizeMB);
 }
 
-bool WiredTigerUtil::appendCustomStats(OperationContext* opCtx,
-                                       BSONObjBuilder* output,
-                                       double scale,
-                                       const std::string& uri) {
-    dassert(opCtx->lockState()->isReadLocked());
-    {
-        BSONObjBuilder metadata(output->subobjStart("metadata"));
-        Status status = WiredTigerUtil::getApplicationMetadata(opCtx, uri, &metadata);
-        if (!status.isOK()) {
-            metadata.append("error", "unable to retrieve metadata");
-            metadata.append("code", static_cast<int>(status.code()));
-            metadata.append("reason", status.reason());
-        }
-    }
-    std::string type, sourceURI;
-    WiredTigerUtil::fetchTypeAndSourceURI(opCtx, uri, &type, &sourceURI);
-    StatusWith<std::string> metadataResult = WiredTigerUtil::getMetadataCreate(opCtx, sourceURI);
-    StringData creationStringName("creationString");
-    if (!metadataResult.isOK()) {
-        BSONObjBuilder creationString(output->subobjStart(creationStringName));
-        creationString.append("error", "unable to retrieve creation config");
-        creationString.append("code", static_cast<int>(metadataResult.getStatus().code()));
-        creationString.append("reason", metadataResult.getStatus().reason());
-    } else {
-        output->append(creationStringName, metadataResult.getValue());
-        // Type can be "lsm" or "file"
-        output->append("type", type);
-    }
-
-    WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession();
-    WT_SESSION* s = session->getSession();
-    Status status =
-        WiredTigerUtil::exportTableToBSON(s, "statistics:" + uri, "statistics=(fast)", output);
-    if (!status.isOK()) {
-        output->append("error", "unable to retrieve statistics");
-        output->append("code", static_cast<int>(status.code()));
-        output->append("reason", status.reason());
-    }
-    return true;
-}
-
-
 logv2::LogSeverity getWTLOGV2SeverityLevel(const BSONObj& obj) {
     const std::string field = "verbose_level_id";
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index a042ffd0681..b17dfcb6f85 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -276,11 +276,6 @@ public:
 
     static int64_t getIdentSize(WT_SESSION* s, const std::string& uri);
 
-    static bool appendCustomStats(OperationContext* opCtx,
-                                  BSONObjBuilder* output,
-                                  double scale,
-                                  const std::string& uri);
-
     /**
      * Returns the bytes available for reuse for an ident. This is the amount of allocated space on
      * disk that is not storing any data.