summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/mongo/db/catalog/multi_index_block.cpp20
-rw-r--r--src/mongo/db/storage/control/storage_control.cpp3
-rw-r--r--src/mongo/db/storage/kv/kv_engine.h22
-rw-r--r--src/mongo/db/storage/storage_engine.h11
-rw-r--r--src/mongo/db/storage/storage_engine_impl.cpp4
-rw-r--r--src/mongo/db/storage/storage_engine_impl.h14
-rw-r--r--src/mongo/db/storage/storage_engine_mock.h9
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp6
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h28
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp6
-rw-r--r--src/mongo/dbtests/validate_tests.cpp88
11 files changed, 186 insertions, 25 deletions
diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp
index 03c41a60991..2df5e30f295 100644
--- a/src/mongo/db/catalog/multi_index_block.cpp
+++ b/src/mongo/db/catalog/multi_index_block.cpp
@@ -49,6 +49,7 @@
#include "mongo/db/repl/repl_set_config.h"
#include "mongo/db/repl/replication_coordinator.h"
#include "mongo/db/repl/tenant_migration_conflict_info.h"
+#include "mongo/db/storage/durable_catalog.h"
#include "mongo/db/storage/storage_options.h"
#include "mongo/db/storage/storage_parameters_gen.h"
#include "mongo/db/storage/write_unit_of_work.h"
@@ -982,6 +983,25 @@ Status MultiIndexBlock::commit(OperationContext* opCtx,
if (bulkBuilder->isMultikey()) {
indexCatalogEntry->setMultikey(opCtx, collection, {}, bulkBuilder->getMultikeyPaths());
}
+
+ if (opCtx->getServiceContext()->getStorageEngine()->supportsCheckpoints()) {
+ // Add the new index ident to a list so that the validate cmd with {background:true}
+ // can ignore the new index until it is regularly checkpoint'ed with the rest of the
+ // storage data.
+ //
+ // Index builds use the bulk loader, which can provoke a checkpoint of just that index.
+ // This makes the checkpoint's PIT view of the collection and indexes inconsistent until
+ // the next storage-wide checkpoint is taken, at which point the list will be reset.
+ //
+ // Note that it is okay if the index commit fails: background validation will never try
+ // to look at the index and the list will be reset by the next periodic storage-wide
+ // checkpoint.
+ auto indexIdent =
+ opCtx->getServiceContext()->getStorageEngine()->getCatalog()->getIndexIdent(
+ opCtx, collection->getCatalogId(), _indexes[i].block->getIndexName());
+ opCtx->getServiceContext()->getStorageEngine()->addIndividuallyCheckpointedIndex(
+ indexIdent);
+ }
}
onCommit();
diff --git a/src/mongo/db/storage/control/storage_control.cpp b/src/mongo/db/storage/control/storage_control.cpp
index fc5534bec69..d47a4030afe 100644
--- a/src/mongo/db/storage/control/storage_control.cpp
+++ b/src/mongo/db/storage/control/storage_control.cpp
@@ -81,7 +81,8 @@ void startStorageControls(ServiceContext* serviceContext, bool forTestOnly) {
JournalFlusher::set(serviceContext, std::move(journalFlusher));
}
- if (!storageEngine->isEphemeral() && !storageGlobalParams.queryableBackupMode) {
+ if (storageEngine->supportsCheckpoints() && !storageEngine->isEphemeral() &&
+ !storageGlobalParams.queryableBackupMode) {
std::unique_ptr<Checkpointer> checkpointer =
std::make_unique<Checkpointer>(storageEngine->getEngine());
checkpointer->go();
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index bca7f550d94..d7a1a66323a 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -239,6 +239,28 @@ public:
"The current storage engine doesn't support backup mode");
}
+ virtual void addIndividuallyCheckpointedIndex(const std::string& ident) {
+ uasserted(ErrorCodes::CommandNotSupported,
+ "The current storage engine does not support checkpoints");
+ }
+
+ virtual void clearIndividuallyCheckpointedIndexes() {
+ uasserted(ErrorCodes::CommandNotSupported,
+ "The current storage engine does not support checkpoints");
+ }
+
+ virtual bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const {
+ uasserted(ErrorCodes::CommandNotSupported,
+ "The current storage engine does not support checkpoints");
+ }
+
+ /**
+ * Returns whether the KVEngine supports checkpoints.
+ */
+ virtual bool supportsCheckpoints() const {
+ return false;
+ }
+
virtual void checkpoint() {}
/**
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 1fe8ed0ee39..d85f19e012f 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -207,6 +207,11 @@ public:
virtual bool supportsCappedCollections() const = 0;
/**
+ * Returns whether the storage engine supports checkpoints.
+ */
+ virtual bool supportsCheckpoints() const = 0;
+
+ /**
* Returns true if the engine does not persist data to disk; false otherwise.
*/
virtual bool isEphemeral() const = 0;
@@ -629,6 +634,12 @@ public:
virtual DurableCatalog* getCatalog() = 0;
virtual const DurableCatalog* getCatalog() const = 0;
+ virtual void addIndividuallyCheckpointedIndex(const std::string& ident) = 0;
+
+ virtual void clearIndividuallyCheckpointedIndexes() = 0;
+
+ virtual bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const = 0;
+
/**
* A service that would like to pin the oldest timestamp registers its request here. If the
* request can be satisfied, OK is returned with the oldest timestamp the caller can
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index a79d22a96b4..19e2f7622f7 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -959,6 +959,10 @@ StatusWith<std::deque<std::string>> StorageEngineImpl::extendBackupCursor(Operat
return _engine->extendBackupCursor(opCtx);
}
+bool StorageEngineImpl::supportsCheckpoints() const {
+ return _engine->supportsCheckpoints();
+}
+
bool StorageEngineImpl::isEphemeral() const {
return _engine->isEphemeral();
}
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 5a318744268..9734f77f10b 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -101,6 +101,8 @@ public:
virtual StatusWith<std::deque<std::string>> extendBackupCursor(
OperationContext* opCtx) override;
+ virtual bool supportsCheckpoints() const override;
+
virtual bool isEphemeral() const override;
virtual Status repairRecordStore(OperationContext* opCtx,
@@ -329,6 +331,18 @@ public:
const DurableCatalog* getCatalog() const override;
+ void addIndividuallyCheckpointedIndex(const std::string& ident) override {
+ return _engine->addIndividuallyCheckpointedIndex(ident);
+ }
+
+ void clearIndividuallyCheckpointedIndexes() override {
+ return _engine->clearIndividuallyCheckpointedIndexes();
+ }
+
+ bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const override {
+ return _engine->isInIndividuallyCheckpointedIndexes(ident);
+ }
+
/**
* When loading after an unclean shutdown, this performs cleanup on the DurableCatalogImpl.
*/
diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h
index a7a7078384a..e180fe7d5e7 100644
--- a/src/mongo/db/storage/storage_engine_mock.h
+++ b/src/mongo/db/storage/storage_engine_mock.h
@@ -47,7 +47,9 @@ public:
bool supportsCappedCollections() const final {
return true;
}
-
+ bool supportsCheckpoints() const final {
+ return false;
+ }
bool isEphemeral() const final {
return true;
}
@@ -194,6 +196,11 @@ public:
const DurableCatalog* getCatalog() const final {
return nullptr;
}
+ void addIndividuallyCheckpointedIndex(const std::string& ident) final {}
+ void clearIndividuallyCheckpointedIndexes() final {}
+ bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const final {
+ return false;
+ }
StatusWith<Timestamp> pinOldestTimestamp(OperationContext* opCtx,
const std::string& requestingServiceName,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index ad6abdca8a2..65c422d2226 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -2013,6 +2013,7 @@ void WiredTigerKVEngine::_checkpoint(WT_SESSION* session) {
// Third, stableTimestamp >= initialDataTimestamp: Take stable checkpoint. Steady state
// case.
if (initialDataTimestamp.asULL() <= 1) {
+ clearIndividuallyCheckpointedIndexes();
invariantWTOK(session->checkpoint(session, "use_timestamp=false"), session);
LOGV2_FOR_RECOVERY(5576602,
2,
@@ -2034,7 +2035,10 @@ void WiredTigerKVEngine::_checkpoint(WT_SESSION* session) {
"stableTimestamp"_attr = stableTimestamp,
"oplogNeededForRollback"_attr = toString(oplogNeededForRollback));
- invariantWTOK(session->checkpoint(session, "use_timestamp=true"), session);
+ {
+ clearIndividuallyCheckpointedIndexes();
+ invariantWTOK(session->checkpoint(session, "use_timestamp=true"), session);
+ }
if (oplogNeededForRollback.isOK()) {
// Now that the checkpoint is durable, publish the oplog needed to recover from it.
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 748ec6c4c1d..10672661629 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -119,6 +119,13 @@ public:
bool supportsDirectoryPerDB() const override;
+ /**
+ * WiredTiger supports checkpoints when it isn't running in memory.
+ */
+ bool supportsCheckpoints() const override {
+ return !isEphemeral();
+ }
+
void checkpoint() override;
bool isEphemeral() const override {
@@ -371,6 +378,18 @@ public:
return _clockSource;
}
+ void addIndividuallyCheckpointedIndex(const std::string& ident) override {
+ _checkpointedIndexes.insert(ident);
+ }
+
+ void clearIndividuallyCheckpointedIndexes() override {
+ _checkpointedIndexes.clear();
+ }
+
+ bool isInIndividuallyCheckpointedIndexes(const std::string& ident) const override {
+ return _checkpointedIndexes.find(ident) != _checkpointedIndexes.end();
+ }
+
StatusWith<Timestamp> pinOldestTimestamp(OperationContext* opCtx,
const std::string& requestingServiceName,
Timestamp requestedTimestamp,
@@ -514,6 +533,15 @@ private:
// timestamp. Provided by replication layer because WT does not persist timestamps.
AtomicWord<std::uint64_t> _initialDataTimestamp;
+ // Required for taking a checkpoint; and can be used to ensure multiple checkpoint cursors
+ // target the same checkpoint.
+ Lock::ResourceMutex _checkpointCursorMutex = Lock::ResourceMutex("checkpointCursorMutex");
+
+ // A set of indexes that were individually checkpoint'ed and are not consistent with the rest
+ // of the checkpoint's PIT view of the storage data. This set is reset when a storage-wide WT
+ // checkpoint is taken that makes the PIT view consistent again.
+ std::set<std::string> _checkpointedIndexes;
+
AtomicWord<std::uint64_t> _oplogNeededForCrashRecovery;
std::unique_ptr<WiredTigerEngineRuntimeConfigParameter> _runTimeConfigParam;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
index a1ad92795a8..a79e5494cdc 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp
@@ -310,7 +310,10 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
auto config = syncType == Fsync::kCheckpointStableTimestamp ? "use_timestamp=true"
: "use_timestamp=false";
- invariantWTOK(s->checkpoint(s, config), s);
+ {
+ _engine->clearIndividuallyCheckpointedIndexes();
+ invariantWTOK(s->checkpoint(s, config), s);
+ }
if (token) {
journalListener->onDurable(token.value());
@@ -362,6 +365,7 @@ void WiredTigerSessionCache::waitUntilDurable(OperationContext* opCtx,
_waitUntilDurableSession);
LOGV2_DEBUG(22419, 4, "flushed journal");
} else {
+ _engine->clearIndividuallyCheckpointedIndexes();
invariantWTOK(_waitUntilDurableSession->checkpoint(_waitUntilDurableSession, nullptr),
_waitUntilDurableSession);
LOGV2_DEBUG(22420, 4, "created checkpoint");
diff --git a/src/mongo/dbtests/validate_tests.cpp b/src/mongo/dbtests/validate_tests.cpp
index d32c99402c2..4674a21de29 100644
--- a/src/mongo/dbtests/validate_tests.cpp
+++ b/src/mongo/dbtests/validate_tests.cpp
@@ -96,6 +96,9 @@ public:
auto coll = db->createCollection(&_opCtx, _nss, options, createIdIndex);
ASSERT_TRUE(coll) << _nss;
wuow.commit();
+
+ _engineSupportsCheckpoints =
+ _opCtx.getServiceContext()->getStorageEngine()->supportsCheckpoints();
}
explicit ValidateBase(bool full, bool background)
@@ -204,6 +207,7 @@ protected:
const NamespaceString _nss;
std::unique_ptr<AutoGetDb> _autoDb;
Database* _db;
+ bool _engineSupportsCheckpoints;
};
template <bool full, bool background>
@@ -212,7 +216,9 @@ public:
ValidateIdIndexCount() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -270,7 +276,9 @@ class ValidateSecondaryIndexCount : public ValidateBase {
public:
ValidateSecondaryIndexCount() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -336,7 +344,9 @@ class ValidateSecondaryIndex : public ValidateBase {
public:
ValidateSecondaryIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -394,7 +404,9 @@ public:
ValidateIdIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -469,7 +481,9 @@ public:
ValidateMultiKeyIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -553,7 +567,9 @@ public:
ValidateSparseIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -612,7 +628,9 @@ public:
ValidatePartialIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -678,7 +696,9 @@ public:
ValidatePartialIndexOnCollectionWithNonIndexableFields() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -738,7 +758,9 @@ public:
ValidateCompoundIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -819,7 +841,9 @@ public:
ValidateIndexEntry() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -938,7 +962,9 @@ public:
ValidateWildCardIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -1062,7 +1088,9 @@ public:
ValidateWildCardIndexWithProjection() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -1162,7 +1190,9 @@ public:
ValidateMissingAndExtraIndexEntryResults() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -1253,7 +1283,9 @@ public:
ValidateMissingIndexEntryResults() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -1371,7 +1403,9 @@ public:
ValidateExtraIndexEntryResults() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -3094,7 +3128,9 @@ public:
ValidateDuplicateKeysUniqueIndex() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -3283,7 +3319,9 @@ public:
ValidateInvalidBSONResults() : ValidateBase(full, background) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -4092,7 +4130,9 @@ public:
: ValidateBase(/*full=*/false, background, /*clustered=*/true) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -4152,7 +4192,9 @@ public:
: ValidateBase(/*full=*/false, background, /*clustered=*/true) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -4252,7 +4294,9 @@ public:
: ValidateBase(/*full=*/false, /*background=*/false, /*clustered=*/true) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
@@ -4394,7 +4438,9 @@ public:
_withSecondaryIndex(withSecondaryIndex) {}
void run() {
- if (_background) {
+ // Cannot run validate with {background:true} if the storage engine does not support
+ // checkpoints.
+ if (_background && !_engineSupportsCheckpoints) {
return;
}
View Lorry Controller Status