diff options
7 files changed, 184 insertions, 141 deletions
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 5ce8b28f002..68a76639ed5 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -35,6 +35,7 @@ wtEnv.Library( 'wiredtiger_cursor_helpers.cpp', 'wiredtiger_global_options.cpp', 'wiredtiger_index.cpp', + 'wiredtiger_index_util.cpp', 'wiredtiger_kv_engine.cpp', 'wiredtiger_oplog_manager.cpp', 'wiredtiger_parameters.cpp', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp index 25d4c623b14..1f034c86e26 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_column_store.cpp @@ -28,20 +28,13 @@ */ -#include "mongo/platform/basic.h" - -#include "mongo/db/global_settings.h" -#include "mongo/db/repl/repl_settings.h" #include "mongo/db/storage/wiredtiger/wiredtiger_column_store.h" +#include "mongo/db/global_settings.h" #include "mongo/db/storage/wiredtiger/wiredtiger_cursor.h" #include "mongo/db/storage/wiredtiger/wiredtiger_cursor_helpers.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_index_cursor_generic.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h" #include "mongo/logv2/log.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage @@ -411,16 +404,7 @@ std::unique_ptr<ColumnStore::BulkBuilder> WiredTigerColumnStore::makeBulkBuilder } bool WiredTigerColumnStore::isEmpty(OperationContext* opCtx) { - // TODO: SERVER-65980, this logic could be shared with WiredTigerIndex::isEmpty(). - WiredTigerCursor curwrap(_uri, _tableId, false, opCtx); - WT_CURSOR* c = curwrap.get(); - if (!c) - return true; - int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); }); - if (ret == WT_NOTFOUND) - return true; - invariantWTOK(ret, c->session); - return false; + return WiredTigerIndexUtil::isEmpty(opCtx, _uri, _tableId); } long long WiredTigerColumnStore::getSpaceUsedBytes(OperationContext* opCtx) const { @@ -443,14 +427,13 @@ long long WiredTigerColumnStore::getFreeStorageBytes(OperationContext* opCtx) co } Status WiredTigerColumnStore::compact(OperationContext* opCtx) { - // TODO: SERVER-65980. - uasserted(ErrorCodes::NotImplemented, "WiredTigerColumnStore::compact"); + return WiredTigerIndexUtil::compact(opCtx, _uri); } bool WiredTigerColumnStore::appendCustomStats(OperationContext* opCtx, BSONObjBuilder* output, double scale) const { - return WiredTigerUtil::appendCustomStats(opCtx, output, scale, _uri); + return WiredTigerIndexUtil::appendCustomStats(opCtx, output, scale, _uri); } } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 147c61a7c84..e194d5af7fa 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -28,37 +28,14 @@ */ -#include "mongo/platform/basic.h" - #include "mongo/db/storage/wiredtiger/wiredtiger_index.h" -#include <fmt/format.h> -#include <memory> -#include <set> - -#include "mongo/base/checked_cast.h" -#include "mongo/db/catalog/index_catalog_entry.h" -#include "mongo/db/catalog/validate_results.h" -#include "mongo/db/global_settings.h" -#include "mongo/db/index/index_descriptor.h" -#include "mongo/db/json.h" -#include "mongo/db/repl/repl_settings.h" -#include "mongo/db/service_context.h" -#include "mongo/db/stats/resource_consumption_metrics.h" -#include "mongo/db/storage/index_entry_comparison.h" -#include "mongo/db/storage/storage_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_cursor_helpers.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_index_cursor_generic.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_util.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h" #include "mongo/logv2/log.h" -#include "mongo/util/assert_util.h" -#include "mongo/util/fail_point.h" -#include "mongo/util/hex.h" -#include "mongo/util/str.h" #include "mongo/util/testing_proctor.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kStorage @@ -83,7 +60,6 @@ namespace mongo { namespace { -MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY); MONGO_FAIL_POINT_DEFINE(WTIndexPauseAfterSearchNear); static const WiredTigerItem emptyItem(nullptr, 0); @@ -375,7 +351,7 @@ void WiredTigerIndex::fullValidate(OperationContext* opCtx, bool WiredTigerIndex::appendCustomStats(OperationContext* opCtx, BSONObjBuilder* output, double scale) const { - return WiredTigerUtil::appendCustomStats(opCtx, output, scale, _uri); + return WiredTigerIndexUtil::appendCustomStats(opCtx, output, scale, _uri); } Status WiredTigerIndex::dupKeyCheck(OperationContext* opCtx, const KeyString::Value& key) { @@ -393,15 +369,7 @@ Status WiredTigerIndex::dupKeyCheck(OperationContext* opCtx, const KeyString::Va } bool WiredTigerIndex::isEmpty(OperationContext* opCtx) { - WiredTigerCursor curwrap(_uri, _tableId, false, opCtx); - WT_CURSOR* c = curwrap.get(); - if (!c) - return true; - int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); }); - if (ret == WT_NOTFOUND) - return true; - invariantWTOK(ret, c->session); - return false; + return WiredTigerIndexUtil::isEmpty(opCtx, _uri, _tableId); } void WiredTigerIndex::printIndexEntryMetadata(OperationContext* opCtx, @@ -497,28 +465,7 @@ Status WiredTigerIndex::initAsEmpty(OperationContext* opCtx) { } Status WiredTigerIndex::compact(OperationContext* opCtx) { - dassert(opCtx->lockState()->isWriteLocked()); - WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache(); - if (!cache->isEphemeral()) { - WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(); - opCtx->recoveryUnit()->abandonSnapshot(); - // WT compact prompts WT to take checkpoints, so we need to take the checkpoint lock around - // WT compact calls. - auto checkpointLock = opCtx->getServiceContext()->getStorageEngine()->getCheckpointLock( - opCtx, StorageEngine::CheckpointLock::Mode::kExclusive); - int ret = s->compact(s, uri().c_str(), "timeout=0"); - if (MONGO_unlikely(WTCompactIndexEBUSY.shouldFail())) { - ret = EBUSY; - } - - if (ret == EBUSY) { - return Status(ErrorCodes::Interrupted, - str::stream() << "Compaction interrupted on " << uri().c_str() - << " due to cache eviction pressure"); - } - invariantWTOK(ret, s); - } - return Status::OK(); + return WiredTigerIndexUtil::compact(opCtx, _uri); } boost::optional<RecordId> WiredTigerIndex::_keyExists(OperationContext* opCtx, @@ -1543,7 +1490,6 @@ void WiredTigerIndexUnique::insertWithRecordIdInValue_forTest(OperationContext* // Now create the table key/value, the actual data record. WiredTigerItem keyItem(keyString.getBuffer(), keyString.getSize()); - BufBuilder bufBuilder; KeyString::Builder valueBuilder(keyString.getVersion(), rid); valueBuilder.appendTypeBits(keyString.getTypeBits()); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp new file mode 100644 index 00000000000..97278595091 --- /dev/null +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.cpp @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + + +#include "mongo/db/storage/wiredtiger/wiredtiger_index_util.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_prepare_conflict.h" + +namespace mongo { + +MONGO_FAIL_POINT_DEFINE(WTCompactIndexEBUSY); + +bool WiredTigerIndexUtil::appendCustomStats(OperationContext* opCtx, + BSONObjBuilder* output, + double scale, + const std::string& uri) { + dassert(opCtx->lockState()->isReadLocked()); + { + BSONObjBuilder metadata(output->subobjStart("metadata")); + Status status = WiredTigerUtil::getApplicationMetadata(opCtx, uri, &metadata); + if (!status.isOK()) { + metadata.append("error", "unable to retrieve metadata"); + metadata.append("code", static_cast<int>(status.code())); + metadata.append("reason", status.reason()); + } + } + std::string type, sourceURI; + WiredTigerUtil::fetchTypeAndSourceURI(opCtx, uri, &type, &sourceURI); + StatusWith<std::string> metadataResult = WiredTigerUtil::getMetadataCreate(opCtx, sourceURI); + StringData creationStringName("creationString"); + if (!metadataResult.isOK()) { + BSONObjBuilder creationString(output->subobjStart(creationStringName)); + creationString.append("error", "unable to retrieve creation config"); + creationString.append("code", static_cast<int>(metadataResult.getStatus().code())); + creationString.append("reason", metadataResult.getStatus().reason()); + } else { + output->append(creationStringName, metadataResult.getValue()); + // Type can be "lsm" or "file" + output->append("type", type); + } + + WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession(); + WT_SESSION* s = session->getSession(); + Status status = + WiredTigerUtil::exportTableToBSON(s, "statistics:" + uri, "statistics=(fast)", output); + if (!status.isOK()) { + output->append("error", "unable to retrieve statistics"); + output->append("code", static_cast<int>(status.code())); + output->append("reason", status.reason()); + } + return true; +} + +Status WiredTigerIndexUtil::compact(OperationContext* opCtx, const std::string& uri) { + dassert(opCtx->lockState()->isWriteLocked()); + WiredTigerSessionCache* cache = WiredTigerRecoveryUnit::get(opCtx)->getSessionCache(); + if (!cache->isEphemeral()) { + WT_SESSION* s = WiredTigerRecoveryUnit::get(opCtx)->getSession()->getSession(); + opCtx->recoveryUnit()->abandonSnapshot(); + // WT compact prompts WT to take checkpoints, so we need to take the checkpoint lock around + // WT compact calls. + auto checkpointLock = opCtx->getServiceContext()->getStorageEngine()->getCheckpointLock( + opCtx, StorageEngine::CheckpointLock::Mode::kExclusive); + int ret = s->compact(s, uri.c_str(), "timeout=0"); + if (MONGO_unlikely(WTCompactIndexEBUSY.shouldFail())) { + ret = EBUSY; + } + + if (ret == EBUSY) { + return Status(ErrorCodes::Interrupted, + str::stream() << "Compaction interrupted on " << uri.c_str() + << " due to cache eviction pressure"); + } + invariantWTOK(ret, s); + } + return Status::OK(); +} + +bool WiredTigerIndexUtil::isEmpty(OperationContext* opCtx, + const std::string& uri, + uint64_t tableId) { + WiredTigerCursor curwrap(uri, tableId, false, opCtx); + WT_CURSOR* c = curwrap.get(); + if (!c) + return true; + int ret = wiredTigerPrepareConflictRetry(opCtx, [&] { return c->next(c); }); + if (ret == WT_NOTFOUND) + return true; + invariantWTOK(ret, c->session); + return false; +} + +} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h new file mode 100644 index 00000000000..e98c5a7b1dc --- /dev/null +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index_util.h @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/base/status.h" +#include "mongo/bson/bsonobjbuilder.h" +#include "mongo/db/operation_context.h" + +namespace mongo { + +class WiredTigerIndexUtil { + WiredTigerIndexUtil(const WiredTigerIndexUtil&) = delete; + WiredTigerIndexUtil& operator=(const WiredTigerIndexUtil&) = delete; + +private: + WiredTigerIndexUtil(); + +public: + static bool appendCustomStats(OperationContext* opCtx, + BSONObjBuilder* output, + double scale, + const std::string& uri); + + static Status compact(OperationContext* opCtx, const std::string& uri); + + static bool isEmpty(OperationContext* opCtx, const std::string& uri, uint64_t tableId); +}; + +} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 96974706972..321f814a89f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -28,36 +28,22 @@ */ -#include "mongo/platform/basic.h" - #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" -#include <limits> - #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> -#include <boost/filesystem/path.hpp> #include "mongo/base/simple_string_data_comparator.h" -#include "mongo/bson/bsonobjbuilder.h" #include "mongo/bson/json.h" #include "mongo/db/concurrency/exception_util.h" #include "mongo/db/concurrency/exception_util_gen.h" #include "mongo/db/global_settings.h" #include "mongo/db/server_options_general_gen.h" #include "mongo/db/snapshot_window_options_gen.h" -#include "mongo/db/storage/storage_file_util.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h" #include "mongo/db/storage/wiredtiger/wiredtiger_parameters_gen.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/logv2/log.h" -#include "mongo/util/assert_util.h" #include "mongo/util/processinfo.h" -#include "mongo/util/scopeguard.h" -#include "mongo/util/static_immortal.h" -#include "mongo/util/str.h" #include "mongo/util/testing_proctor.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kWiredTiger @@ -562,48 +548,6 @@ size_t WiredTigerUtil::getCacheSizeMB(double requestedCacheSizeGB) { return static_cast<size_t>(cacheSizeMB); } -bool WiredTigerUtil::appendCustomStats(OperationContext* opCtx, - BSONObjBuilder* output, - double scale, - const std::string& uri) { - dassert(opCtx->lockState()->isReadLocked()); - { - BSONObjBuilder metadata(output->subobjStart("metadata")); - Status status = WiredTigerUtil::getApplicationMetadata(opCtx, uri, &metadata); - if (!status.isOK()) { - metadata.append("error", "unable to retrieve metadata"); - metadata.append("code", static_cast<int>(status.code())); - metadata.append("reason", status.reason()); - } - } - std::string type, sourceURI; - WiredTigerUtil::fetchTypeAndSourceURI(opCtx, uri, &type, &sourceURI); - StatusWith<std::string> metadataResult = WiredTigerUtil::getMetadataCreate(opCtx, sourceURI); - StringData creationStringName("creationString"); - if (!metadataResult.isOK()) { - BSONObjBuilder creationString(output->subobjStart(creationStringName)); - creationString.append("error", "unable to retrieve creation config"); - creationString.append("code", static_cast<int>(metadataResult.getStatus().code())); - creationString.append("reason", metadataResult.getStatus().reason()); - } else { - output->append(creationStringName, metadataResult.getValue()); - // Type can be "lsm" or "file" - output->append("type", type); - } - - WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSession(); - WT_SESSION* s = session->getSession(); - Status status = - WiredTigerUtil::exportTableToBSON(s, "statistics:" + uri, "statistics=(fast)", output); - if (!status.isOK()) { - output->append("error", "unable to retrieve statistics"); - output->append("code", static_cast<int>(status.code())); - output->append("reason", status.reason()); - } - return true; -} - - logv2::LogSeverity getWTLOGV2SeverityLevel(const BSONObj& obj) { const std::string field = "verbose_level_id"; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index a042ffd0681..b17dfcb6f85 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -276,11 +276,6 @@ public: static int64_t getIdentSize(WT_SESSION* s, const std::string& uri); - static bool appendCustomStats(OperationContext* opCtx, - BSONObjBuilder* output, - double scale, - const std::string& uri); - /** * Returns the bytes available for reuse for an ident. This is the amount of allocated space on * disk that is not storing any data. |