diff options
author | Andrew Morrow <acm@mongodb.com> | 2020-02-02 11:05:31 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-03 17:38:53 +0000 |
commit | 93cfea46d961955b795b83e8a3938df7d9746695 (patch) | |
tree | 90e34a77cc09b85bb4bb2198a87a4f890f407701 /src/mongo | |
parent | 7739d3935096e3ec158fba279715f16619989d6f (diff) | |
download | mongo-93cfea46d961955b795b83e8a3938df7d9746695.tar.gz |
SERVER-44549 Remove the mobile storage engine
mode change 100644 => 100755 buildscripts/evergreen_task_timeout.py
delete mode 100644 buildscripts/resmokeconfig/suites/disk_mobile.yml
delete mode 100644 src/mongo/db/storage/mobile/SConscript
delete mode 100644 src/mongo/db/storage/mobile/mobile_index.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_index.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_index_test.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_init.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_kv_engine.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_kv_engine.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_options.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_options.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_options.idl
delete mode 100644 src/mongo/db/storage/mobile/mobile_options_mongod.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_record_store.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_record_store.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_record_store_test.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_recovery_unit.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_recovery_unit.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_session.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_session.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_session_pool.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_session_pool.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_sqlite_statement.h
delete mode 100644 src/mongo/db/storage/mobile/mobile_util.cpp
delete mode 100644 src/mongo/db/storage/mobile/mobile_util.h
delete mode 100755 src/third_party/scripts/sqlite_get_sources.sh
delete mode 100644 src/third_party/shim_sqlite.cpp
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/SConscript
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/patches/gethostuuid.patch
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/sqlite/shell.c
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/sqlite/sqlite3.c
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/sqlite/sqlite3.h
delete mode 100644 src/third_party/sqlite-amalgamation-3260000/sqlite/sqlite3ext.h
Diffstat (limited to 'src/mongo')
33 files changed, 4 insertions, 4781 deletions
diff --git a/src/mongo/db/catalog/multi_index_block.cpp b/src/mongo/db/catalog/multi_index_block.cpp index a9722029d1b..b506c2dc9cc 100644 --- a/src/mongo/db/catalog/multi_index_block.cpp +++ b/src/mongo/db/catalog/multi_index_block.cpp @@ -167,12 +167,6 @@ void MultiIndexBlock::cleanUpAfterBuild(OperationContext* opCtx, } bool MultiIndexBlock::areHybridIndexBuildsEnabled() { - // The mobile storage engine does not suport dupsAllowed mode on bulk builders, which means that - // it does not support hybrid builds. See SERVER-38550 - if (storageGlobalParams.engine == "mobile") { - return false; - } - return enableHybridIndexBuilds.load(); } diff --git a/src/mongo/db/catalog/validate_state.cpp b/src/mongo/db/catalog/validate_state.cpp index c8462c1bc86..4abd966b19e 100644 --- a/src/mongo/db/catalog/validate_state.cpp +++ b/src/mongo/db/catalog/validate_state.cpp @@ -116,11 +116,6 @@ void ValidateState::_yieldLocks(OperationContext* opCtx) { void ValidateState::_yieldCursors(OperationContext* opCtx) { invariant(!_background); - // Mobile does not support saving and restoring cursors, so we skip yielding cursors for it. - if (storageGlobalParams.engine == "mobile") { - return; - } - // Save all the cursors. for (const auto& indexCursor : _indexCursors) { indexCursor.second->save(); diff --git a/src/mongo/db/commands/create_indexes.cpp b/src/mongo/db/commands/create_indexes.cpp index 24162252130..f9788719875 100644 --- a/src/mongo/db/commands/create_indexes.cpp +++ b/src/mongo/db/commands/create_indexes.cpp @@ -375,65 +375,6 @@ void checkCollectionShardingState(OperationContext* opCtx, const NamespaceString } /** - * Opens or creates database for index creation. Only intended for mobile storage engine. - * On database creation, the lock will be made exclusive. - * TODO(SERVER-42513): Remove this function. - */ -Database* getOrCreateDatabase(OperationContext* opCtx, StringData dbName, Lock::DBLock* dbLock) { - auto databaseHolder = DatabaseHolder::get(opCtx); - - if (auto db = databaseHolder->getDb(opCtx, dbName)) { - return db; - } - - // Temporarily release the Database lock while holding a Global IX lock. This prevents - // replication state from changing. Abandon the current snapshot to see changed metadata. - opCtx->recoveryUnit()->abandonSnapshot(); - dbLock->relockWithMode(MODE_X); - - checkDatabaseShardingState(opCtx, dbName); - return databaseHolder->openDb(opCtx, dbName); -} - -/** - * Gets or creates collection to hold indexes. Only intended for mobile storage engine. - * Appends field to command result to indicate if the collection already exists. - * TODO(SERVER-42513): Remove this function. - */ -Collection* getOrCreateCollection(OperationContext* opCtx, - Database* db, - const NamespaceString& ns, - const BSONObj& cmdObj, - std::string* errmsg, - BSONObjBuilder* result) { - if (auto collection = CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, ns)) { - result->appendBool(kCreateCollectionAutomaticallyFieldName, false); - return collection; - } - - result->appendBool(kCreateCollectionAutomaticallyFieldName, true); - - if (ViewCatalog::get(db)->lookup(opCtx, ns.ns())) { - *errmsg = "Cannot create indexes on a view"; - uasserted(ErrorCodes::CommandNotSupportedOnView, *errmsg); - } - - uassertStatusOK(userAllowedCreateNS(ns.db(), ns.coll())); - - CollectionOptions options; - options.uuid = UUID::gen(); - return writeConflictRetry(opCtx, kCommandName, ns.ns(), [&] { - WriteUnitOfWork wunit(opCtx); - auto collection = db->createCollection(opCtx, ns, options); - invariant(collection, - str::stream() << "Failed to create collection " << ns.ns() - << " during index creation: " << redact(cmdObj)); - wunit.commit(); - return collection; - }); -} - -/** * Attempts to create indexes in `specs` on a non-existent collection with namespace `ns`, thereby * implicitly creating the collection. * Returns a BSONObj containing fields to be appended to the result of the calling function. @@ -512,255 +453,6 @@ BSONObj runCreateIndexesOnNewCollection(OperationContext* opCtx, return createResult.obj(); } -/** - * Creates indexes using the given specs for the mobile storage engine. - * TODO(SERVER-42513): Remove this function. - */ -bool runCreateIndexesForMobile(OperationContext* opCtx, - const std::string& dbname, - const BSONObj& cmdObj, - std::string& errmsg, - BSONObjBuilder& result) { - NamespaceString ns(CommandHelpers::parseNsCollectionRequired(dbname, cmdObj)); - uassertStatusOK(userAllowedWriteNS(ns)); - - // Disallow users from creating new indexes on config.transactions since the sessions code - // was optimized to not update indexes - uassert(ErrorCodes::IllegalOperation, - str::stream() << "not allowed to create index on " << ns.ns(), - ns != NamespaceString::kSessionTransactionsTableNamespace); - - auto specs = uassertStatusOK( - parseAndValidateIndexSpecs(opCtx, ns, cmdObj, serverGlobalParams.featureCompatibility)); - - MONGO_COMPILER_VARIABLE_UNUSED auto commitQuorum = parseAndGetCommitQuorum(opCtx, cmdObj); - - Status validateTTL = validateTTLOptions(opCtx, cmdObj); - uassertStatusOK(validateTTL); - - // Do not use AutoGetOrCreateDb because we may relock the database in mode X. - Lock::DBLock dbLock(opCtx, ns.db(), MODE_IX); - checkDatabaseShardingState(opCtx, ns.db()); - if (!repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesFor(opCtx, ns)) { - uasserted(ErrorCodes::NotMaster, - str::stream() << "Not primary while creating indexes in " << ns.ns()); - } - - if (indexesAlreadyExist(opCtx, ns, specs, &result)) { - return true; - } - - auto db = getOrCreateDatabase(opCtx, ns.db(), &dbLock); - - opCtx->recoveryUnit()->abandonSnapshot(); - boost::optional<Lock::CollectionLock> exclusiveCollectionLock( - boost::in_place_init, opCtx, ns, MODE_X); - checkCollectionShardingState(opCtx, ns); - - // Index builds can safely ignore prepare conflicts and perform writes. On primaries, an - // exclusive lock in the final drain phase conflicts with prepared transactions. - opCtx->recoveryUnit()->setPrepareConflictBehavior( - PrepareConflictBehavior::kIgnoreConflictsAllowWrites); - - Collection* collection = getOrCreateCollection(opCtx, db, ns, cmdObj, &errmsg, &result); - // Save the db name and collection uuid so we can correctly relock even across a - // concurrent rename collection operation. We allow rename collection while an - // index is in progress iff the rename is within the same database. - const std::string dbName = ns.db().toString(); - const UUID collectionUUID = collection->uuid(); - - // Use AutoStatsTracker to update Top. - boost::optional<AutoStatsTracker> statsTracker; - const boost::optional<int> dbProfilingLevel = boost::none; - statsTracker.emplace(opCtx, - ns, - Top::LockType::WriteLocked, - AutoStatsTracker::LogMode::kUpdateTopAndCurOp, - dbProfilingLevel); - - MultiIndexBlock indexer; - - const size_t origSpecsSize = specs.size(); - specs = resolveDefaultsAndRemoveExistingIndexes(opCtx, collection, std::move(specs)); - - const int numIndexesBefore = collection->getIndexCatalog()->numIndexesTotal(opCtx); - if (specs.size() == 0) { - fillCommandResultWithIndexesAlreadyExistInfo(numIndexesBefore, &result); - return true; - } - - result.append("numIndexesBefore", numIndexesBefore); - - if (specs.size() != origSpecsSize) { - result.append("note", "index already exists"); - } - - for (size_t i = 0; i < specs.size(); i++) { - const BSONObj& spec = specs[i]; - if (spec["unique"].trueValue()) { - checkUniqueIndexConstraints(opCtx, ns, spec["key"].Obj()); - } - } - - // The 'indexer' can throw, so ensure the build cleanup occurs. - ON_BLOCK_EXIT([&] { - opCtx->recoveryUnit()->abandonSnapshot(); - if (MONGO_unlikely(leaveIndexBuildUnfinishedForShutdown.shouldFail())) { - // Set a flag to leave the persisted index build state intact when cleanUpAfterBuild() - // is called below. The index build will be found on server startup. - // - // Note: this failpoint has two parts, the first to make the index build error and the - // second to catch it here: the index build must error before commit(), otherwise - // commit() clears the state. - indexer.abortWithoutCleanup(opCtx); - } - - if (!indexer.isCommitted()) { - opCtx->recoveryUnit()->abandonSnapshot(); - exclusiveCollectionLock.reset(); - UninterruptibleLockGuard noInterrupt(opCtx->lockState()); - Lock::DBLock dbLock(opCtx, ns.db(), MODE_IX); - Lock::CollectionLock colLock(opCtx, {dbName, collectionUUID}, MODE_X); - indexer.cleanUpAfterBuild(opCtx, collection, MultiIndexBlock::kNoopOnCleanUpFn); - } else { - indexer.cleanUpAfterBuild(opCtx, collection, MultiIndexBlock::kNoopOnCleanUpFn); - } - }); - - std::vector<BSONObj> indexInfoObjs = - writeConflictRetry(opCtx, kCommandName, ns.ns(), [opCtx, collection, &indexer, &specs] { - return uassertStatusOK( - indexer.init(opCtx, - collection, - specs, - MultiIndexBlock::makeTimestampedIndexOnInitFn(opCtx, collection))); - }); - - // Don't hold an exclusive collection lock during background indexing, so that other readers - // and writers can proceed during this phase. A BackgroundOperation has been registered on the - // namespace, so the collection cannot be removed after yielding the lock. - if (indexer.isBackgroundBuilding()) { - invariant(BackgroundOperation::inProgForNs(ns)); - opCtx->recoveryUnit()->abandonSnapshot(); - exclusiveCollectionLock.reset(); - } - - // Collection scan and insert into index, followed by a drain of writes received in the - // background. - { - Lock::CollectionLock colLock(opCtx, {dbName, collectionUUID}, MODE_IS); - - // Reaquire the collection pointer because we momentarily released the collection lock. - collection = CollectionCatalog::get(opCtx).lookupCollectionByUUID(opCtx, collectionUUID); - invariant(collection); - - // Reaquire the 'ns' string in case the collection was renamed while we momentarily released - // the collection lock. - ns = collection->ns(); - - uassertStatusOK(indexer.insertAllDocumentsInCollection(opCtx, collection)); - } - - if (MONGO_unlikely(hangAfterIndexBuildDumpsInsertsFromBulk.shouldFail())) { - log() << "Hanging after dumping inserts from bulk builder"; - hangAfterIndexBuildDumpsInsertsFromBulk.pauseWhileSet(); - } - - // Perform the first drain while holding an intent lock. - { - opCtx->recoveryUnit()->abandonSnapshot(); - Lock::CollectionLock colLock(opCtx, {dbName, collectionUUID}, MODE_IS); - - // Reaquire the collection pointer because we momentarily released the collection lock. - collection = CollectionCatalog::get(opCtx).lookupCollectionByUUID(opCtx, collectionUUID); - invariant(collection); - - // Reaquire the 'ns' string in case the collection was renamed while we momentarily released - // the collection lock. - ns = collection->ns(); - - uassertStatusOK( - indexer.drainBackgroundWrites(opCtx, - RecoveryUnit::ReadSource::kUnset, - IndexBuildInterceptor::DrainYieldPolicy::kYield)); - } - - if (MONGO_unlikely(hangAfterIndexBuildFirstDrain.shouldFail())) { - log() << "Hanging after index build first drain"; - hangAfterIndexBuildFirstDrain.pauseWhileSet(); - } - - // Perform the second drain while stopping writes on the collection. - { - opCtx->recoveryUnit()->abandonSnapshot(); - Lock::CollectionLock colLock(opCtx, {dbName, collectionUUID}, MODE_S); - - // Reaquire the collection pointer because we momentarily released the collection lock. - collection = CollectionCatalog::get(opCtx).lookupCollectionByUUID(opCtx, collectionUUID); - invariant(collection); - - // Reaquire the 'ns' string in case the collection was renamed while we momentarily released - // the collection lock. - ns = collection->ns(); - - uassertStatusOK( - indexer.drainBackgroundWrites(opCtx, - RecoveryUnit::ReadSource::kUnset, - IndexBuildInterceptor::DrainYieldPolicy::kNoYield)); - } - - if (MONGO_unlikely(hangAfterIndexBuildSecondDrain.shouldFail())) { - log() << "Hanging after index build second drain"; - hangAfterIndexBuildSecondDrain.pauseWhileSet(); - } - - // Need to get exclusive collection lock back to complete the index build. - if (indexer.isBackgroundBuilding()) { - opCtx->recoveryUnit()->abandonSnapshot(); - exclusiveCollectionLock.emplace( - opCtx, NamespaceStringOrUUID(dbName, collectionUUID), MODE_X); - - // Reaquire the collection pointer because we momentarily released the collection lock. - collection = CollectionCatalog::get(opCtx).lookupCollectionByUUID(opCtx, collectionUUID); - invariant(collection); - - // Reaquire the 'ns' string in case the collection was renamed while we momentarily released - // the collection lock. - ns = collection->ns(); - } - - auto databaseHolder = DatabaseHolder::get(opCtx); - db = databaseHolder->getDb(opCtx, ns.db()); - invariant(CollectionCatalog::get(opCtx).lookupCollectionByNamespace(opCtx, ns)); - - // Perform the third and final drain while holding the exclusive collection lock. - uassertStatusOK( - indexer.drainBackgroundWrites(opCtx, - RecoveryUnit::ReadSource::kUnset, - IndexBuildInterceptor::DrainYieldPolicy::kNoYield)); - - // This is required before completion. - uassertStatusOK(indexer.checkConstraints(opCtx)); - - writeConflictRetry(opCtx, kCommandName, ns.ns(), [&] { - WriteUnitOfWork wunit(opCtx); - - uassertStatusOK( - indexer.commit(opCtx, - collection, - [opCtx, &ns, collection](const BSONObj& spec) { - opCtx->getServiceContext()->getOpObserver()->onCreateIndex( - opCtx, ns, collection->uuid(), spec, false); - }, - MultiIndexBlock::kNoopOnCommitFn)); - - wunit.commit(); - }); - - result.append("numIndexesAfter", collection->getIndexCatalog()->numIndexesTotal(opCtx)); - - return true; -} bool runCreateIndexesWithCoordinator(OperationContext* opCtx, const std::string& dbname, @@ -1014,11 +706,6 @@ public: bool shouldLogMessageOnAlreadyBuildingError = true; while (true) { try { - // TODO(SERVER-42513): Remove runCreateIndexesForMobile() when the mobile storage - // engine is supported by runCreateIndexesWithCoordinator(). - if (storageGlobalParams.engine == "mobile") { - return runCreateIndexesForMobile(opCtx, dbname, cmdObj, errmsg, result); - } return runCreateIndexesWithCoordinator(opCtx, dbname, cmdObj, errmsg, result); } catch (const DBException& ex) { // We can only wait for an existing index build to finish if we are able to release diff --git a/src/mongo/db/index_builds_coordinator.cpp b/src/mongo/db/index_builds_coordinator.cpp index 26165e90611..c7bff176eee 100644 --- a/src/mongo/db/index_builds_coordinator.cpp +++ b/src/mongo/db/index_builds_coordinator.cpp @@ -1699,10 +1699,8 @@ void IndexBuildsCoordinator::_scanCollectionAndInsertKeysIntoSorter( } // Rebuilding system indexes during startup using the IndexBuildsCoordinator is done by all - // storage engines if they're missing. This includes the mobile storage engine which builds - // its indexes in the foreground. - invariant(_indexBuildsManager.isBackgroundBuilding(replState->buildUUID) || - storageGlobalParams.engine == "mobile"); + // storage engines if they're missing. + invariant(_indexBuildsManager.isBackgroundBuilding(replState->buildUUID)); // Index builds can safely ignore prepare conflicts and perform writes. On secondaries, prepare // operations wait for index builds to complete. diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index 84ef52957c3..fe5f4a40296 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -9,7 +9,6 @@ env.SConscript( 'devnull', 'ephemeral_for_test', 'kv', - 'mobile', 'wiredtiger', ], exports=[ diff --git a/src/mongo/db/storage/mobile/SConscript b/src/mongo/db/storage/mobile/SConscript deleted file mode 100644 index 4679b558ac5..00000000000 --- a/src/mongo/db/storage/mobile/SConscript +++ /dev/null @@ -1,110 +0,0 @@ -Import("env") -Import("mobile_se") -env = env.Clone() - - -env.InjectThirdParty(libraries=['sqlite']) - -env.Library( - target='storage_mobile_core', - source=[ - 'mobile_index.cpp', - 'mobile_kv_engine.cpp', - 'mobile_record_store.cpp', - 'mobile_recovery_unit.cpp', - 'mobile_session.cpp', - 'mobile_session_pool.cpp', - 'mobile_sqlite_statement.cpp', - 'mobile_util.cpp', - 'mobile_options.cpp', - env.Idlc('mobile_options.idl')[0], - ], - LIBDEPS=[ - '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/db/catalog/collection_options', - '$BUILD_DIR/mongo/db/concurrency/lock_manager', - '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', - '$BUILD_DIR/mongo/db/index/index_descriptor', - '$BUILD_DIR/mongo/db/namespace_string', - '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', - '$BUILD_DIR/mongo/db/storage/index_entry_comparison', - '$BUILD_DIR/mongo/db/storage/journal_listener', - '$BUILD_DIR/mongo/db/storage/key_string', - '$BUILD_DIR/mongo/db/storage/recovery_unit_base', - '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', - '$BUILD_DIR/mongo/db/storage/oplog_hack', - '$BUILD_DIR/third_party/shim_sqlite', - ], - LIBDEPS_PRIVATE= [ - '$BUILD_DIR/mongo/util/options_parser/options_parser', - ], - ) - -serveronlyProgDepsDependents = [] -if mobile_se: - serveronlyProgDepsDependents = [ - '$BUILD_DIR/mongo/mongod', - '$BUILD_DIR/mongo/dbtests/dbtest', - ] - - env.Library( - target='mobile_options_mongod', - source=[ - 'mobile_options_mongod.cpp', - ], - PROGDEPS_DEPENDENTS=serveronlyProgDepsDependents, - LIBDEPS=[ - '$BUILD_DIR/mongo/util/options_parser/options_parser', - ], - LIBDEPS_PRIVATE=[ - 'storage_mobile_core', - ], - ) - -env.Library( - target='storage_mobile', - source=[ - 'mobile_init.cpp', - ], - PROGDEPS_DEPENDENTS=serveronlyProgDepsDependents, - LIBDEPS=[ - 'storage_mobile_core', - '$BUILD_DIR/mongo/db/storage/durable_catalog_impl', - '$BUILD_DIR/mongo/db/storage/storage_engine_impl', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/storage/storage_engine_common', - ], -) - -''' -env.CppUnitTest( - target='storage_mobile_index_test', - source=[ - 'mobile_index_test.cpp' - ], - LIBDEPS=[ - 'storage_mobile_core', - '$BUILD_DIR/mongo/db/storage/sorted_data_interface_test_harness' - ] - ) -''' - -env.CppUnitTest( - target='storage_mobile_test', - source=[ - 'mobile_kv_engine_test.cpp', - 'mobile_record_store_test.cpp', - ], - LIBDEPS=[ - 'storage_mobile_core', - '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', - '$BUILD_DIR/mongo/db/storage/kv/kv_engine_test_harness', - '$BUILD_DIR/mongo/db/storage/record_store_test_harness', - '$BUILD_DIR/mongo/db/storage/storage_options', - '$BUILD_DIR/mongo/util/options_parser/options_parser', - ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/auth/authmocks', - ], -) diff --git a/src/mongo/db/storage/mobile/mobile_index.cpp b/src/mongo/db/storage/mobile/mobile_index.cpp deleted file mode 100644 index 0c0808b5dc3..00000000000 --- a/src/mongo/db/storage/mobile/mobile_index.cpp +++ /dev/null @@ -1,795 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include <sqlite3.h> - -#include "mongo/bson/bsonobj.h" -#include "mongo/db/index/index_descriptor.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/key_string.h" -#include "mongo/db/storage/mobile/mobile_index.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" - -namespace mongo { -namespace { - -using std::shared_ptr; -using std::string; -using std::vector; -} // namespace - -MobileIndex::MobileIndex(OperationContext* opCtx, - const IndexDescriptor* desc, - const std::string& ident) - : SortedDataInterface(KeyString::Version::kLatestVersion, Ordering::make(desc->keyPattern())), - _isUnique(desc->unique()), - _ordering(Ordering::make(desc->keyPattern())), - _ident(ident), - _collectionNamespace(desc->parentNS()), - _indexName(desc->indexName()), - _keyPattern(desc->keyPattern()) {} - -Status MobileIndex::insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - return _insert(opCtx, keyString, dupsAllowed); -} - -template <typename ValueType> -Status MobileIndex::doInsert(OperationContext* opCtx, - const char* keyBuffer, - size_t keySize, - const KeyString::TypeBits& typeBits, - const ValueType& value, - bool isTransactional) { - MobileSession* session; - if (isTransactional) { - session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - } else { - session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - } - - SqliteStatement insertStmt( - *session, "INSERT INTO \"", _ident, "\" (key, value) VALUES (?, ?);"); - - insertStmt.bindBlob(0, keyBuffer, keySize); - insertStmt.bindBlob(1, value.getBuffer(), value.getSize()); - - int status = insertStmt.step(); - if (status == SQLITE_CONSTRAINT) { - insertStmt.setExceptionStatus(status); - if (isUnique()) { - // Return error if duplicate key inserted in a unique index. - BSONObj bson = KeyString::toBson(keyBuffer, keySize, _ordering, typeBits); - return buildDupKeyErrorStatus(bson, _collectionNamespace, _indexName, _keyPattern); - } else { - // A record with same key could already be present in a standard index, that is OK. This - // can happen when building a background index while documents are being written in - // parallel. - return Status::OK(); - } - } - embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); - - return Status::OK(); -} - -void MobileIndex::unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - _unindex(opCtx, keyString, dupsAllowed); -} - -void MobileIndex::_doDelete(OperationContext* opCtx, - const char* keyBuffer, - size_t keySize, - KeyString::Builder* value) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - - SqliteStatement deleteStmt( - *session, "DELETE FROM \"", _ident, "\" WHERE key = ?", value ? " AND value = ?" : "", ";"); - - deleteStmt.bindBlob(0, keyBuffer, keySize); - if (value) { - deleteStmt.bindBlob(1, value->getBuffer(), value->getSize()); - } - deleteStmt.step(SQLITE_DONE); -} - -/** - * Note: this validates the entire database file, not just the table used by this index. - */ -void MobileIndex::fullValidate(OperationContext* opCtx, - long long* numKeysOut, - ValidateResults* fullResults) const { - if (fullResults) { - embedded::doValidate(opCtx, fullResults); - if (!fullResults->valid) { - return; - } - } - if (numKeysOut) { - *numKeysOut = numEntries(opCtx); - } -} - -bool MobileIndex::appendCustomStats(OperationContext* opCtx, - BSONObjBuilder* output, - double scale) const { - return false; -} - -long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - // Sum the number of bytes in each column. - // SQLite aggregate functions return null if the column is empty or has only nulls, so return 0 - // bytes if there is no data in the column. - SqliteStatement sizeStmt(*session, - "SELECT IFNULL(SUM(LENGTH(key)), 0) + ", - "IFNULL(SUM(LENGTH(value)), 0) FROM \"", - _ident, - "\";"); - - sizeStmt.step(SQLITE_ROW); - - long long dataSize = sizeStmt.getColInt(0); - return dataSize; -} - -long long MobileIndex::numEntries(OperationContext* opCtx) const { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - SqliteStatement countStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); - - countStmt.step(SQLITE_ROW); - long long numRecs = countStmt.getColInt(0); - return numRecs; -} - -bool MobileIndex::isEmpty(OperationContext* opCtx) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - SqliteStatement emptyCheckStmt(*session, "SELECT * FROM \"", _ident, "\" LIMIT 1;"); - - int status = emptyCheckStmt.step(); - if (status == SQLITE_DONE) { - return true; - } - embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); - return false; -} - -Status MobileIndex::initAsEmpty(OperationContext* opCtx) { - // No-op. - return Status::OK(); -} - -Status MobileIndex::create(OperationContext* opCtx, const std::string& ident) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - - SqliteStatement createTableStmt( - *session, "CREATE TABLE \"", ident, "\"(key BLOB PRIMARY KEY, value BLOB);"); - - createTableStmt.step(SQLITE_DONE); - return Status::OK(); -} - -Status MobileIndex::dupKeyCheck(OperationContext* opCtx, const KeyString::Value& key) { - invariant(_isUnique); - - if (_isDup(opCtx, key)) - return buildDupKeyErrorStatus( - key, _collectionNamespace, _indexName, _keyPattern, _ordering); - return Status::OK(); -} - -bool MobileIndex::_isDup(OperationContext* opCtx, const KeyString::Value& key) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - SqliteStatement dupCheckStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\" WHERE key = ?;"); - - dupCheckStmt.bindBlob(0, key.getBuffer(), key.getSize()); - - dupCheckStmt.step(SQLITE_ROW); - long long records = dupCheckStmt.getColInt(0); - return records > 1; -} - -class MobileIndex::BulkBuilderBase : public SortedDataBuilderInterface { -public: - BulkBuilderBase(MobileIndex* index, - OperationContext* opCtx, - bool dupsAllowed, - const NamespaceString& collectionNamespace, - const std::string& indexName, - const BSONObj& keyPattern) - : _index(index), - _opCtx(opCtx), - _lastKeyString(index->getKeyStringVersion()), - _dupsAllowed(dupsAllowed), - _collectionNamespace(collectionNamespace), - _indexName(indexName), - _keyPattern(keyPattern) {} - - virtual ~BulkBuilderBase() {} - - Status addKey(const KeyString::Value& keyString) override { - Status status = _checkNextKey(keyString); - if (!status.isOK()) { - return status; - } - - _lastKeyString.resetFromBuffer(keyString.getBuffer(), keyString.getSize()); - - return _addKey(keyString); - } - - void commit(bool mayInterrupt) override {} - -protected: - /** - * Checks whether the new key to be inserted is > or >= the previous one depending - * on _dupsAllowed. - */ - Status _checkNextKey(const KeyString::Value& keyString) { - const int cmp = _lastKeyString.compare(keyString); - if (!_dupsAllowed && cmp == 0) { - auto key = KeyString::toBson(keyString, _index->getOrdering()); - return buildDupKeyErrorStatus(key, _collectionNamespace, _indexName, _keyPattern); - } else if (cmp > 0) { - return Status(ErrorCodes::InternalError, "expected higher RecordId in bulk builder"); - } - return Status::OK(); - } - - virtual Status _addKey(const KeyString::Value& keyString) = 0; - - MobileIndex* _index; - OperationContext* const _opCtx; - KeyString::Builder _lastKeyString; - const bool _dupsAllowed; - const NamespaceString _collectionNamespace; - const std::string _indexName; - const BSONObj _keyPattern; -}; - -/** - * Bulk builds a non-unique index. - */ -class MobileIndex::BulkBuilderStandard final : public BulkBuilderBase { -public: - BulkBuilderStandard(MobileIndex* index, - OperationContext* opCtx, - bool dupsAllowed, - const NamespaceString& collectionNamespace, - const std::string& indexName, - const BSONObj& keyPattern) - : BulkBuilderBase(index, opCtx, dupsAllowed, collectionNamespace, indexName, keyPattern) {} - -protected: - Status _addKey(const KeyString::Value& keyString) override { - KeyString::TypeBits typeBits = keyString.getTypeBits(); - return _index->doInsert( - _opCtx, keyString.getBuffer(), keyString.getSize(), typeBits, typeBits, false); - } -}; - -/** - * Bulk builds a unique index. - */ -class MobileIndex::BulkBuilderUnique : public BulkBuilderBase { -public: - BulkBuilderUnique(MobileIndex* index, - OperationContext* opCtx, - bool dupsAllowed, - const NamespaceString& collectionNamespace, - const std::string& indexName, - const BSONObj& keyPattern) - : BulkBuilderBase(index, opCtx, dupsAllowed, collectionNamespace, indexName, keyPattern) { - // Replication is not supported so dups are not allowed. - invariant(!dupsAllowed); - } - -protected: - Status _addKey(const KeyString::Value& keyString) override { - RecordId recId = KeyString::decodeRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - invariant(recId.isValid()); - - KeyString::Builder value(_index->getKeyStringVersion(), recId); - KeyString::TypeBits typeBits = keyString.getTypeBits(); - if (!typeBits.isAllZeros()) { - value.appendTypeBits(typeBits); - } - auto sizeWithoutRecordId = - KeyString::sizeWithoutRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - - return _index->doInsert(_opCtx, - keyString.getBuffer(), - sizeWithoutRecordId, - keyString.getTypeBits(), - value, - false); - } -}; - -namespace { - -/** - * Implements basic cursor functionality used by standard and unique indexes. - */ -class CursorBase : public SortedDataInterface::Cursor { -public: - CursorBase(const MobileIndex& index, OperationContext* opCtx, bool isForward) - : _index(index), - _opCtx(opCtx), - _isForward(isForward), - _savedKey(index.getKeyStringVersion()), - _savedRecId(0), - _savedTypeBits(index.getKeyStringVersion()), - _startPosition(index.getKeyStringVersion()) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - _stmt = std::make_unique<SqliteStatement>(*session, - "SELECT key, value FROM \"", - _index.getIdent(), - "\" WHERE key ", - (_isForward ? ">=" : "<="), - " ? ORDER BY key ", - (_isForward ? "ASC" : "DESC"), - ";"); - } - - virtual ~CursorBase() {} - - boost::optional<IndexKeyEntry> next(RequestedInfo parts) override { - if (!_advanceNext()) { - return {}; - } - return getCurrentEntry(parts); - } - - boost::optional<KeyStringEntry> nextKeyString() override { - if (!_advanceNext()) { - return {}; - } - if (_isEOF) { - return {}; - } - return _getKeyStringEntry(); - } - - void setEndPosition(const BSONObj& key, bool inclusive) override { - // Scan to end of index. - if (key.isEmpty()) { - _endPosition.reset(); - return; - } - - // This uses the opposite rules as a normal seek because a forward scan should end after the - // key if inclusive and before if exclusive. - const auto discriminator = _isForward == inclusive - ? KeyString::Discriminator::kExclusiveAfter - : KeyString::Discriminator::kExclusiveBefore; - _endPosition = std::make_unique<KeyString::Builder>(_index.getKeyStringVersion()); - _endPosition->resetToKey( - BSONObj::stripFieldNames(key), _index.getOrdering(), discriminator); - } - - boost::optional<IndexKeyEntry> seek(const KeyString::Value& keyString, - RequestedInfo parts) override { - seekForKeyString(keyString); - if (_isEOF) { - return {}; - } - return getCurrentEntry(parts); - } - - boost::optional<KeyStringEntry> seekForKeyString( - const KeyString::Value& keyStringValue) override { - _startPosition.resetFromBuffer(keyStringValue.getBuffer(), keyStringValue.getSize()); - _doSeek(); - _updatePosition(); - if (_isEOF) { - return {}; - } - return _getKeyStringEntry(); - } - - boost::optional<IndexKeyEntry> seekExact(const KeyString::Value& keyStringValue, - RequestedInfo parts) override { - auto ksEntry = seekExactForKeyString(keyStringValue); - if (ksEntry) { - auto kv = getCurrentEntry(parts); - invariant(kv); - return kv; - } - return {}; - } - - boost::optional<KeyStringEntry> seekExactForKeyString( - const KeyString::Value& keyStringValue) override { - auto ksEntry = [&]() { - if (_isForward) { - return seekForKeyString(keyStringValue); - } - - // Append a kExclusiveAfter discriminator if it's a reverse cursor to ensure that the - // KeyString we construct will always be greater than the KeyString that we retrieve - // (even when it has a RecordId). - KeyString::Builder keyCopy(_index.getKeyStringVersion(), _index.getOrdering()); - - // Reset by copying all but the last byte, the kEnd byte. - keyCopy.resetFromBuffer(keyStringValue.getBuffer(), keyStringValue.getSize() - 1); - - // Append a different discriminator and new end byte. - keyCopy.appendDiscriminator(KeyString::Discriminator::kExclusiveAfter); - return seekForKeyString(keyCopy.getValueCopy()); - }(); - - if (!ksEntry) { - return {}; - } - - if (KeyString::compare(ksEntry->keyString.getBuffer(), - keyStringValue.getBuffer(), - KeyString::sizeWithoutRecordIdAtEnd(ksEntry->keyString.getBuffer(), - ksEntry->keyString.getSize()), - keyStringValue.getSize()) == 0) { - return KeyStringEntry(ksEntry->keyString, ksEntry->loc); - } - return {}; - } - - // All work is done in restore(). - void save() override { - // SQLite acquires implicit locks over the snapshot this cursor is using. It is important - // to finalize the corresponding statement to release these locks. - _stmt->finalize(); - } - - void saveUnpositioned() override { - save(); - } - - void restore() override { - if (_isEOF) { - return; - } - - // Obtaining a session starts a read transaction if not done already. - MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx); - // save() finalized this cursor's SQLite statement. We need to prepare a new statement, - // before re-positioning it at the saved state. - _stmt->prepare(*session); - - _startPosition.resetFromBuffer(_savedKey.getBuffer(), _savedKey.getSize()); - bool isExactMatch = _doSeek(); - - if (!isExactMatch) { - _isEOF = false; - _resetStatement(); - } - } - - void detachFromOperationContext() override { - _opCtx = nullptr; - } - - void reattachToOperationContext(OperationContext* opCtx) override { - _opCtx = opCtx; - } - -protected: - bool _advanceNext() { - if (_isEOF) { - return false; - } - - _advance(); - _updatePosition(); - return true; - } - - KeyStringEntry _getKeyStringEntry() { - auto sizeWithoutRecordId = KeyString::getKeySize(_savedKey.getBuffer(), - _savedKey.getSize(), - _index.getOrdering(), - _savedKey.getTypeBits()); - if (_savedKey.getSize() == sizeWithoutRecordId) { - // Create a copy of _key with a RecordId. Because _key is used during cursor restore(), - // appending the RecordId would cause the cursor to be repositioned incorrectly. - KeyString::Builder keyWithRecordId(_savedKey); - keyWithRecordId.appendRecordId(_savedRecId); - keyWithRecordId.setTypeBits(_savedTypeBits); - return KeyStringEntry(keyWithRecordId.getValueCopy(), _savedRecId); - } - - _savedKey.setTypeBits(_savedTypeBits); - return KeyStringEntry(_savedKey.getValueCopy(), _savedRecId); - } - - /** - * Advances the cursor and determines if end reached. - */ - void _advance() { - int status = _stmt->step(); - if (status == SQLITE_DONE) { - _isEOF = true; - return; - } - embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); - - _isEOF = false; - - const void* key = _stmt->getColBlob(0); - const long long size = _stmt->getColBytes(0); - - KeyString::Builder currKey(_index.getKeyStringVersion()); - currKey.resetFromBuffer(key, size); - - // The cursor has reached EOF if the current row passes the end position. - _isEOF = (_endPosition && _isForward && currKey > *_endPosition) || - (_endPosition && !_isForward && currKey < *_endPosition); - } - - /** - * Updates the cursor state to reflect current position by setting the current key value, - * record id, and type bits. - */ - void _updatePosition() { - if (_isEOF) { - return; - } - const void* key = _stmt->getColBlob(0); - const long long size = _stmt->getColBytes(0); - - _savedKey.resetFromBuffer(key, size); - _updateRecIdAndTypeBits(); - } - - /** - * Returns the requested parts of the entry at the cursor's current position. - */ - boost::optional<IndexKeyEntry> getCurrentEntry(RequestedInfo parts) const { - if (_isEOF) { - return {}; - } - - BSONObj bson; - if (parts & kWantKey) { - bson = KeyString::toBson( - _savedKey.getBuffer(), _savedKey.getSize(), _index.getOrdering(), _savedTypeBits); - } - - return {{std::move(bson), _savedRecId}}; - } - - /** - * Moves the cursor to begin at the given position. Returns true if the new position matches - * the saved position; returns false otherwise. - */ - bool _doSeek() { - _resetStatement(); - _bindStartPoint(); - - _isEOF = false; - - _advance(); - - if (_isEOF) { - return false; - } - - const void* key = _stmt->getColBlob(0); - const long long size = _stmt->getColBytes(0); - - KeyString::Builder nearestKey(_index.getKeyStringVersion()); - nearestKey.resetFromBuffer(key, size); - - return nearestKey == _startPosition; - } - - /** - * Binds the start point for the cursor. - */ - void _bindStartPoint() { - _stmt->bindBlob(0, _startPosition.getBuffer(), _startPosition.getSize()); - } - - /** - * Resets the prepared statement on the SQLite statement that performs the iteration. - */ - void _resetStatement() { - _stmt->reset(); - } - - virtual void _updateRecIdAndTypeBits() = 0; - - const MobileIndex& _index; - OperationContext* _opCtx; // Not owned. - - bool _isForward; - bool _isEOF = true; - - KeyString::Builder _savedKey; - RecordId _savedRecId; - KeyString::TypeBits _savedTypeBits; - - // The statement executed to fetch rows from SQLite. - std::unique_ptr<SqliteStatement> _stmt; - - KeyString::Builder _startPosition; - std::unique_ptr<KeyString::Builder> _endPosition; -}; - -/** - * Cursor for a non-unique index. - */ -class CursorStandard final : public CursorBase { -public: - CursorStandard(const MobileIndex& index, OperationContext* opCtx, bool isForward) - : CursorBase(index, opCtx, isForward) {} - -protected: - void _updateRecIdAndTypeBits() override { - _savedRecId = KeyString::decodeRecordIdAtEnd(_savedKey.getBuffer(), _savedKey.getSize()); - - const void* value = _stmt->getColBlob(1); - const long long size = _stmt->getColBytes(1); - BufReader br(value, size); - _savedTypeBits.resetFromBuffer(&br); - } -}; - -/** - * Cursor for a unique index. - */ -class CursorUnique final : public CursorBase { -public: - CursorUnique(const MobileIndex& index, OperationContext* opCtx, bool isForward) - : CursorBase(index, opCtx, isForward) {} - -protected: - void _updateRecIdAndTypeBits() override { - const void* value = _stmt->getColBlob(1); - const long long size = _stmt->getColBytes(1); - BufReader br(value, size); - _savedRecId = KeyString::decodeRecordId(&br); - _savedTypeBits.resetFromBuffer(&br); - } -}; -} // namespace - -MobileIndexStandard::MobileIndexStandard(OperationContext* opCtx, - const IndexDescriptor* desc, - const std::string& ident) - : MobileIndex(opCtx, desc, ident) {} - -SortedDataBuilderInterface* MobileIndexStandard::getBulkBuilder(OperationContext* opCtx, - bool dupsAllowed) { - invariant(dupsAllowed); - return new BulkBuilderStandard( - this, opCtx, dupsAllowed, _collectionNamespace, _indexName, _keyPattern); -} - -std::unique_ptr<SortedDataInterface::Cursor> MobileIndexStandard::newCursor(OperationContext* opCtx, - bool isForward) const { - return std::make_unique<CursorStandard>(*this, opCtx, isForward); -} - -Status MobileIndexStandard::_insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - invariant(dupsAllowed); - dassert(KeyString::decodeRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()).isValid()); - - const KeyString::TypeBits typeBits = keyString.getTypeBits(); - return doInsert(opCtx, keyString.getBuffer(), keyString.getSize(), typeBits, typeBits); -} - -void MobileIndexStandard::_unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - invariant(dupsAllowed); - dassert(KeyString::decodeRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()).isValid()); - - _doDelete(opCtx, keyString.getBuffer(), keyString.getSize()); -} - -MobileIndexUnique::MobileIndexUnique(OperationContext* opCtx, - const IndexDescriptor* desc, - const std::string& ident) - : MobileIndex(opCtx, desc, ident), _isPartial(desc->isPartial()) {} - -SortedDataBuilderInterface* MobileIndexUnique::getBulkBuilder(OperationContext* opCtx, - bool dupsAllowed) { - // Replication is not supported so dups are not allowed. - invariant(!dupsAllowed); - return new BulkBuilderUnique( - this, opCtx, dupsAllowed, _collectionNamespace, _indexName, _keyPattern); -} - -std::unique_ptr<SortedDataInterface::Cursor> MobileIndexUnique::newCursor(OperationContext* opCtx, - bool isForward) const { - return std::make_unique<CursorUnique>(*this, opCtx, isForward); -} - -Status MobileIndexUnique::_insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - // Replication is not supported so dups are not allowed. - invariant(!dupsAllowed); - - RecordId recId = KeyString::decodeRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - invariant(recId.isValid()); - - KeyString::Builder value(_keyStringVersion, recId); - KeyString::TypeBits typeBits = keyString.getTypeBits(); - if (!typeBits.isAllZeros()) { - value.appendTypeBits(typeBits); - } - auto sizeWithoutRecordId = - KeyString::sizeWithoutRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - - return doInsert( - opCtx, keyString.getBuffer(), sizeWithoutRecordId, keyString.getTypeBits(), value); -} - -void MobileIndexUnique::_unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) { - // Replication is not supported so dups are not allowed. - invariant(!dupsAllowed); - - // A partial index may attempt to delete a non-existent record id. If it is a partial index, it - // must delete a row that matches both key and value. - auto sizeWithoutRecordId = - KeyString::sizeWithoutRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - if (_isPartial) { - RecordId recId = KeyString::decodeRecordIdAtEnd(keyString.getBuffer(), keyString.getSize()); - invariant(recId.isValid()); - - KeyString::Builder value(_keyStringVersion, recId); - KeyString::TypeBits typeBits = keyString.getTypeBits(); - if (!typeBits.isAllZeros()) { - value.appendTypeBits(typeBits); - } - - _doDelete(opCtx, keyString.getBuffer(), sizeWithoutRecordId, &value); - } else { - _doDelete(opCtx, keyString.getBuffer(), sizeWithoutRecordId); - } -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_index.h b/src/mongo/db/storage/mobile/mobile_index.h deleted file mode 100644 index a660ea232f5..00000000000 --- a/src/mongo/db/storage/mobile/mobile_index.h +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <set> - -#include "mongo/db/catalog/index_catalog_entry.h" -#include "mongo/db/storage/index_entry_comparison.h" -#include "mongo/db/storage/key_string.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/sorted_data_interface.h" -#include "mongo/platform/basic.h" - -namespace mongo { - -class MobileIndex : public SortedDataInterface { -public: - MobileIndex(OperationContext* opCtx, const IndexDescriptor* desc, const std::string& ident); - - MobileIndex(bool isUnique, - const Ordering& ordering, - const std::string& ident, - const std::string& collectionNamespace, - const std::string& indexName); - - virtual ~MobileIndex() {} - - Status insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; - - void unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; - - void fullValidate(OperationContext* opCtx, - long long* numKeysOut, - ValidateResults* fullResults) const override; - - bool appendCustomStats(OperationContext* opCtx, - BSONObjBuilder* output, - double scale) const override; - - long long getSpaceUsedBytes(OperationContext* opCtx) const override; - - long long numEntries(OperationContext* opCtx) const override; - - bool isEmpty(OperationContext* opCtx) override; - - Status initAsEmpty(OperationContext* opCtx) override; - - Status dupKeyCheck(OperationContext* opCtx, const KeyString::Value& key) override; - - // Beginning of MobileIndex-specific methods - - /** - * Creates a SQLite table suitable for a new Mobile index. - */ - static Status create(OperationContext* opCtx, const std::string& ident); - - /** - * Performs the insert into the table with the given key and value. - */ - template <typename ValueType> - Status doInsert(OperationContext* opCtx, - const char* keyBuffer, - size_t keySize, - const KeyString::TypeBits& typeBits, - const ValueType& value, - bool isTransactional = true); - - bool isUnique() const { - return _isUnique; - } - - std::string getIdent() const { - return _ident; - } - -protected: - bool _advanceNext(); - - KeyStringEntry _getKeyStringEntry(); - - bool _isDup(OperationContext* opCtx, const KeyString::Value& key); - - /** - * Performs the deletion from the table matching the given key. - */ - void _doDelete(OperationContext* opCtx, - const char* keyBuffer, - size_t keySize, - KeyString::Builder* value = nullptr); - - virtual Status _insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) = 0; - - virtual void _unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) = 0; - - class BulkBuilderBase; - class BulkBuilderStandard; - class BulkBuilderUnique; - - const bool _isUnique; - const Ordering _ordering; - const std::string _ident; - const NamespaceString _collectionNamespace; - const std::string _indexName; - const BSONObj _keyPattern; -}; - -class MobileIndexStandard final : public MobileIndex { -public: - MobileIndexStandard(OperationContext* opCtx, - const IndexDescriptor* desc, - const std::string& ident); - - SortedDataBuilderInterface* getBulkBuilder(OperationContext* opCtx, bool dupsAllowed) override; - - std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, - bool isForward) const override; - -protected: - Status _insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; - - void _unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; -}; - -class MobileIndexUnique final : public MobileIndex { -public: - MobileIndexUnique(OperationContext* opCtx, - const IndexDescriptor* desc, - const std::string& ident); - - SortedDataBuilderInterface* getBulkBuilder(OperationContext* opCtx, bool dupsAllowed) override; - - std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, - bool isForward) const override; - -protected: - Status _insert(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; - - void _unindex(OperationContext* opCtx, - const KeyString::Value& keyString, - bool dupsAllowed) override; - - const bool _isPartial = false; -}; -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_index_test.cpp b/src/mongo/db/storage/mobile/mobile_index_test.cpp deleted file mode 100644 index b3ccfda87bb..00000000000 --- a/src/mongo/db/storage/mobile/mobile_index_test.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include <boost/filesystem.hpp> -#include <boost/system/error_code.hpp> -#include <memory> - -#include "mongo/base/init.h" -#include "mongo/db/operation_context_noop.h" -#include "mongo/db/storage/mobile/mobile_index.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" -#include "mongo/db/storage/sorted_data_interface_test_harness.h" -#include "mongo/platform/basic.h" -#include "mongo/unittest/temp_dir.h" -#include "mongo/unittest/unittest.h" - -namespace mongo { -int inc = 0; - -class MobileIndexTestHarnessHelper final : public virtual SortedDataInterfaceHarnessHelper { -public: - MobileIndexTestHarnessHelper() - : _dbPath("mobile_index_harness"), _ordering(Ordering::make(BSONObj())) { - boost::filesystem::path fullPath(_dbPath.path()); - fullPath /= "mobile.sqlite"; - _fullPath = fullPath.string(); - _sessionPool.reset(new MobileSessionPool(_fullPath)); - } - - std::unique_ptr<SortedDataInterface> newIdIndexSortedDataInterface() final { - return newSortedDataInterface(true /* unique */, false /* partial */); - } - - std::unique_ptr<SortedDataInterface> newSortedDataInterface(bool isUnique, bool isPartial) { - std::string ident("index_" + std::to_string(inc++)); - OperationContextNoop opCtx(newRecoveryUnit().release()); - Status status = MobileIndex::create(&opCtx, ident); - fassert(37052, status); - - if (isUnique) { - return std::make_unique<MobileIndexUnique>( - _ordering, ident, "test.mobile", "indexName"); - } - return std::make_unique<MobileIndexStandard>(_ordering, ident, "test.mobile", "indexName"); - } - - std::unique_ptr<RecoveryUnit> newRecoveryUnit() { - return std::make_unique<MobileRecoveryUnit>(_sessionPool.get()); - } - -private: - unittest::TempDir _dbPath; - std::string _fullPath; - std::unique_ptr<MobileSessionPool> _sessionPool; - const Ordering _ordering; -}; - -std::unique_ptr<SortedDataInterfaceHarnessHelper> makeMobileIndexHarnessHelper() { - return std::make_unique<MobileIndexTestHarnessHelper>(); -} - -MONGO_INITIALIZER(RegisterSortedDataInterfaceHarnessFactory)(InitializerContext* const) { - mongo::registerSortedDataInterfaceHarnessHelperFactory(makeMobileIndexHarnessHelper); - return Status::OK(); -} -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_init.cpp b/src/mongo/db/storage/mobile/mobile_init.cpp deleted file mode 100644 index 87818f6df58..00000000000 --- a/src/mongo/db/storage/mobile/mobile_init.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include <sqlite3.h> - -#include "mongo/base/init.h" -#include "mongo/db/service_context.h" -#include "mongo/db/storage/mobile/mobile_kv_engine.h" -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/storage_engine_impl.h" -#include "mongo/db/storage/storage_engine_init.h" -#include "mongo/db/storage/storage_options.h" - -namespace mongo { - -namespace { -class MobileFactory : public StorageEngine::Factory { -public: - StorageEngine* create(const StorageGlobalParams& params, - const StorageEngineLockFile* lockFile) const override { - uassert(ErrorCodes::InvalidOptions, - "mobile does not support --groupCollections", - !params.groupCollections); - - StorageEngineOptions options; - options.directoryPerDB = params.directoryperdb; - options.forRepair = params.repair; - - MobileKVEngine* kvEngine = new MobileKVEngine( - params.dbpath, embedded::mobileGlobalOptions, getGlobalServiceContext()); - - return new StorageEngineImpl(kvEngine, options); - } - - StringData getCanonicalName() const override { - return "mobile"; - } - - Status validateMetadata(const StorageEngineMetadata& metadata, - const StorageGlobalParams& params) const override { - return Status::OK(); - } - - BSONObj createMetadataOptions(const StorageGlobalParams& params) const override { - return BSONObj(); - } -}; - -ServiceContext::ConstructorActionRegisterer mobileKVEngineInitializer{ - "MobileKVEngineInit", [](ServiceContext* service) { - registerStorageEngine(service, std::make_unique<MobileFactory>()); - }}; -} // namespace -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp deleted file mode 100644 index 495dce80070..00000000000 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include <boost/filesystem/operations.hpp> -#include <boost/filesystem/path.hpp> -#include <boost/system/error_code.hpp> -#include <memory> -#include <vector> - -#include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/lock_state.h" -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/index/index_descriptor.h" -#include "mongo/db/storage/mobile/mobile_index.h" -#include "mongo/db/storage/mobile/mobile_kv_engine.h" -#include "mongo/db/storage/mobile/mobile_record_store.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/util/log.h" -#include "mongo/util/scopeguard.h" - -namespace mongo { -namespace { -int64_t queryPragmaInt(const MobileSession& session, StringData pragma) { - SqliteStatement stmt(session, "PRAGMA ", pragma, ";"); - stmt.step(SQLITE_ROW); - return stmt.getColInt(0); -} - -std::string queryPragmaStr(const MobileSession& session, StringData pragma) { - SqliteStatement stmt(session, "PRAGMA ", pragma, ";"); - stmt.step(SQLITE_ROW); - return stmt.getColText(0); -} -} // namespace - -MobileKVEngine::MobileKVEngine(const std::string& path, - const embedded::MobileOptions& options, - ServiceContext* serviceContext) - : _options(options) { - _initDBPath(path); - - _sessionPool.reset(new MobileSessionPool(_path, _options)); - - // getSession only needs a valid opCtx if the pool has no available sessions and we need to wait - // for one. But as this is init code and we've just initialized the pool that should not happen. - // Passing nullptr as opCtx so we avoid creating one here. - auto session = _sessionPool->getSession(nullptr); - - fassert(37001, queryPragmaStr(*session, "journal_mode"_sd) == "wal"); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database opened in WAL mode"; - - fassert(50869, queryPragmaInt(*session, "synchronous"_sd) == _options.durabilityLevel); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database has synchronous set to: " - << _options.durabilityLevel; - - fassert(50868, queryPragmaInt(*session, "fullfsync"_sd) == 1); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database is set to fsync with " - "F_FULLFSYNC if the platform supports it (currently only darwin " - "kernels). Value: 1"; - - if (!_options.disableVacuumJob) { - _vacuumJob = serviceContext->getPeriodicRunner()->makeJob( - PeriodicRunner::PeriodicJob("SQLiteVacuumJob", - [this](Client* client) { - if (!client->getServiceContext()->getStorageEngine()) - return; - maybeVacuum(client, Date_t::max()); - }, - Minutes(options.vacuumCheckIntervalMinutes))); - _vacuumJob.start(); - } -} - -void MobileKVEngine::cleanShutdown() { - try { - if (!_options.disableVacuumJob) - maybeVacuum(Client::getCurrent(), Date_t()); - } catch (const std::exception& e) { - LOG(MOBILE_LOG_LEVEL_LOW) - << "MobileSE: Exception while doing vacuum at shutdown, surpressing. " << e.what(); - } -} - -void MobileKVEngine::maybeVacuum(Client* client, Date_t deadline) { - ServiceContext::UniqueOperationContext opCtxUPtr; - OperationContext* opCtx = client->getOperationContext(); - if (!opCtx) { - opCtxUPtr = client->makeOperationContext(); - opCtx = opCtxUPtr.get(); - } - - std::unique_ptr<MobileSession> session; - int64_t pageCount; - int64_t freelistCount; - { - // There may be other threads doing write operations that has locked the database in - // exclusive mode. Grab an S lock here to ensure that isn't the case while we get the - // session and query the pragmas. - Lock::GlobalLock lk(opCtx, MODE_S, deadline, Lock::InterruptBehavior::kThrow); - if (!lk.isLocked()) - return; - - session = _sessionPool->getSession(opCtx); - pageCount = queryPragmaInt(*session, "page_count"_sd); - freelistCount = queryPragmaInt(*session, "freelist_count"_sd); - } - - constexpr int kPageSize = 4096; // SQLite default - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Evaluating if we need to vacuum. page_count = " - << pageCount << ", freelist_count = " << freelistCount; - if ((pageCount > 0 && (float)freelistCount / pageCount >= _options.vacuumFreePageRatio) || - (freelistCount * kPageSize >= _options.vacuumFreeSizeMB * 1024 * 1024)) { - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Performing incremental vacuum"; - // Data will we moved on the file system, take an exclusive lock - Lock::GlobalLock lk(opCtx, MODE_X, deadline, Lock::InterruptBehavior::kThrow); - if (!lk.isLocked()) - return; - - SqliteStatement::execQuery(session.get(), "PRAGMA incremental_vacuum;"); - } -} - -void MobileKVEngine::_initDBPath(const std::string& path) { - boost::system::error_code err; - boost::filesystem::path dbPath(path); - - if (!boost::filesystem::exists(dbPath, err)) { - if (err) { - uasserted(4085, err.message()); - } - std::string errMsg("DB path not found: "); - errMsg += dbPath.generic_string(); - uasserted(4086, errMsg); - - } else if (!boost::filesystem::is_directory(dbPath, err)) { - if (err) { - uasserted(4087, err.message()); - } - std::string errMsg("DB path is not a valid directory: "); - errMsg += dbPath.generic_string(); - uasserted(4088, errMsg); - } - - dbPath /= "mobile.sqlite"; - - if (boost::filesystem::exists(dbPath, err)) { - if (err) { - uasserted(4089, err.message()); - } else if (!boost::filesystem::is_regular_file(dbPath)) { - std::string errMsg("Failed to open " + dbPath.generic_string() + - ": not a regular file"); - uasserted(4090, errMsg); - } - } - _path = dbPath.generic_string(); -} - -RecoveryUnit* MobileKVEngine::newRecoveryUnit() { - return new MobileRecoveryUnit(_sessionPool.get()); -} - -Status MobileKVEngine::createRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) { - // TODO: eventually will support file renaming but otherwise do not use collection options. - - // Mobile SE doesn't support creating an oplog - if (NamespaceString::oplog(ns)) { - return Status(ErrorCodes::InvalidOptions, - "Replication is not supported by the mobile storage engine"); - } - - // Mobile doesn't support capped collections - if (options.capped) { - return Status(ErrorCodes::InvalidOptions, - "Capped collections are not supported by the mobile storage engine"); - } - - MobileRecordStore::create(opCtx, ident.toString()); - return Status::OK(); -} - -std::unique_ptr<RecordStore> MobileKVEngine::getRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) { - return std::make_unique<MobileRecordStore>(opCtx, ns, _path, ident.toString(), options); -} - -std::unique_ptr<RecordStore> MobileKVEngine::makeTemporaryRecordStore(OperationContext* opCtx, - StringData ident) { - MobileRecordStore::create(opCtx, ident.toString()); - return std::make_unique<MobileRecordStore>( - opCtx, "", _path, ident.toString(), CollectionOptions()); -} - - -Status MobileKVEngine::createSortedDataInterface(OperationContext* opCtx, - const CollectionOptions& collOptions, - StringData ident, - const IndexDescriptor* desc) { - return MobileIndex::create(opCtx, ident.toString()); -} - -std::unique_ptr<SortedDataInterface> MobileKVEngine::getSortedDataInterface( - OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) { - if (desc->unique()) { - return std::make_unique<MobileIndexUnique>(opCtx, desc, ident.toString()); - } - return std::make_unique<MobileIndexStandard>(opCtx, desc, ident.toString()); -} - -Status MobileKVEngine::dropIdent(OperationContext* opCtx, RecoveryUnit* ru, StringData ident) { - auto mRu = checked_cast<MobileRecoveryUnit*>(ru); - MobileSession* session = mRu->getSessionNoTxn(opCtx); - std::string dropQuery = "DROP TABLE IF EXISTS \"" + ident + "\";"; - - try { - SqliteStatement::execQuery(session, dropQuery); - } catch (const WriteConflictException&) { - // It is possible that this drop fails because of transaction running in parallel. - // We pretend that it succeeded, queue it for now and keep retrying later. - LOG(MOBILE_LOG_LEVEL_LOW) - << "MobileSE: Caught WriteConflictException while dropping table, " - "queuing to retry later"; - mRu->enqueueFailedDrop(dropQuery); - } - return Status::OK(); -} - -/** - * Note: this counts the total number of bytes in the key and value columns, not the actual - * number of bytes on disk used by this ident. - */ -int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - // Get key-value column names. - SqliteStatement colNameStmt(*session, "PRAGMA table_info(\"", ident, "\")"); - - colNameStmt.step(SQLITE_ROW); - std::string keyColName(colNameStmt.getColText(1)); - colNameStmt.step(SQLITE_ROW); - std::string valueColName(colNameStmt.getColText(1)); - colNameStmt.step(SQLITE_DONE); - - // Get total data size of key-value columns. - SqliteStatement dataSizeStmt(*session, - "SELECT IFNULL(SUM(LENGTH(", - keyColName, - ")), 0) + ", - "IFNULL(SUM(LENGTH(", - valueColName, - ")), 0) FROM \"", - ident, - "\";"); - - dataSizeStmt.step(SQLITE_ROW); - return dataSizeStmt.getColInt(0); -} - -bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - SqliteStatement findTableStmt(*session, - "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;"); - findTableStmt.bindText(0, ident.rawData(), ident.size()); - - int status = findTableStmt.step(); - if (status == SQLITE_DONE) { - return false; - } - embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); - - return true; -} - -std::vector<std::string> MobileKVEngine::getAllIdents(OperationContext* opCtx) const { - std::vector<std::string> idents; - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - - SqliteStatement getTablesStmt(*session, "SELECT name FROM sqlite_master WHERE type='table';"); - - int status; - while ((status = getTablesStmt.step()) == SQLITE_ROW) { - std::string tableName(getTablesStmt.getColText(0)); - idents.push_back(tableName); - } - embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); - return idents; -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h deleted file mode 100644 index 662f2898b24..00000000000 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.h +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <memory> - -#include "mongo/db/storage/journal_listener.h" -#include "mongo/db/storage/kv/kv_engine.h" -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" -#include "mongo/platform/mutex.h" -#include "mongo/util/periodic_runner.h" -#include "mongo/util/string_map.h" - -namespace mongo { - -class JournalListener; - -class MobileKVEngine : public KVEngine { -public: - MobileKVEngine(const std::string& path, - const embedded::MobileOptions& options, - ServiceContext* serviceContext); - - RecoveryUnit* newRecoveryUnit() override; - - Status createRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) override; - - std::unique_ptr<RecordStore> getRecordStore(OperationContext* opCtx, - StringData ns, - StringData ident, - const CollectionOptions& options) override; - - std::unique_ptr<RecordStore> makeTemporaryRecordStore(OperationContext* opCtx, - StringData ident) override; - - Status createSortedDataInterface(OperationContext* opCtx, - const CollectionOptions& collOptions, - StringData ident, - const IndexDescriptor* desc) override; - - std::unique_ptr<SortedDataInterface> getSortedDataInterface( - OperationContext* opCtx, StringData ident, const IndexDescriptor* desc) override; - - Status beginBackup(OperationContext* opCtx) override { - return Status::OK(); - } - - void endBackup(OperationContext* opCtx) override {} - - Status dropIdent(OperationContext* opCtx, RecoveryUnit* ru, StringData ident) override; - - bool supportsDocLocking() const override { - return false; - } - - bool supportsDBLocking() const override { - return false; - } - - bool supportsCappedCollections() const override { - return false; - } - - bool supportsDirectoryPerDB() const override { - return false; - } - - bool isDurable() const override { - return true; - } - - /** - * Flush is a no-op since SQLite transactions are durable by default after each commit. - */ - int flushAllFiles(OperationContext* opCtx, bool sync) override { - return 0; - } - - bool isEphemeral() const override { - return false; - } - - int64_t getIdentSize(OperationContext* opCtx, StringData ident) override; - - Status repairIdent(OperationContext* opCtx, StringData ident) override { - return Status::OK(); - } - - void cleanShutdown() override; - - bool hasIdent(OperationContext* opCtx, StringData ident) const override; - - std::vector<std::string> getAllIdents(OperationContext* opCtx) const override; - - void setJournalListener(JournalListener* jl) override { - stdx::unique_lock<Latch> lk(_mutex); - _journalListener = jl; - } - - virtual Timestamp getAllDurableTimestamp() const override { - MONGO_UNREACHABLE; - } - - virtual Timestamp getOldestOpenReadTimestamp() const override { - return Timestamp(); - } - - boost::optional<Timestamp> getOplogNeededForCrashRecovery() const final { - return boost::none; - } - -private: - void maybeVacuum(Client* client, Date_t deadline); - - mutable Mutex _mutex = MONGO_MAKE_LATCH("MobileKVEngine::_mutex"); - void _initDBPath(const std::string& path); - std::int32_t _setSQLitePragma(const std::string& pragma, sqlite3* session); - - std::unique_ptr<MobileSessionPool> _sessionPool; - - // Notified when we write as everything is considered "journalled" since repl depends on it. - JournalListener* _journalListener = &NoOpJournalListener::instance; - - std::string _path; - embedded::MobileOptions _options; - - PeriodicJobAnchor _vacuumJob; -}; - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp deleted file mode 100644 index dfa93f18c19..00000000000 --- a/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/base/init.h" - -#include <memory> - -#include "mongo/db/storage/kv/kv_engine_test_harness.h" -#include "mongo/db/storage/mobile/mobile_kv_engine.h" -#include "mongo/db/storage/mobile/mobile_options_gen.h" -#include "mongo/unittest/temp_dir.h" -#include "mongo/util/options_parser/options_parser.h" -#include "mongo/util/options_parser/startup_options.h" - -namespace mongo { -namespace { - -class MobileKVHarnessHelper : public KVHarnessHelper { -public: - MobileKVHarnessHelper() : _dbPath("mobile_kv_engine_harness") { - addMobileStorageOptionDefinitions(&optionenvironment::startupOptions).ignore(); - optionenvironment::OptionsParser parser; - std::vector<std::string> args; - std::map<std::string, std::string> env; - parser - .run(optionenvironment::startupOptions, - args, - env, - &optionenvironment::startupOptionsParsed) - .ignore(); - storeMobileStorageOptionDefinitions(optionenvironment::startupOptionsParsed).ignore(); - - embedded::mobileGlobalOptions.disableVacuumJob = true; - _engine = std::make_unique<MobileKVEngine>( - _dbPath.path(), embedded::mobileGlobalOptions, nullptr); - } - - virtual KVEngine* restartEngine() { - _engine.reset(new MobileKVEngine( - _dbPath.path(), embedded::mobileGlobalOptions, _serviceContext.get())); - return _engine.get(); - } - - virtual KVEngine* getEngine() { - return _engine.get(); - } - -private: - unittest::TempDir _dbPath; - std::unique_ptr<MobileKVEngine> _engine; - ServiceContext::UniqueServiceContext _serviceContext; -}; - -std::unique_ptr<KVHarnessHelper> makeHelper() { - return std::make_unique<MobileKVHarnessHelper>(); -} - -MONGO_INITIALIZER(RegisterKVHarnessFactory)(InitializerContext*) { - KVHarnessHelper::registerFactory(makeHelper); - return Status::OK(); -} - -} // namespace -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_options.cpp b/src/mongo/db/storage/mobile/mobile_options.cpp deleted file mode 100644 index ac5ff02aed9..00000000000 --- a/src/mongo/db/storage/mobile/mobile_options.cpp +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include "mongo/db/storage/mobile/mobile_options.h" - -namespace mongo { -namespace embedded { - -MobileOptions mobileGlobalOptions; - -} // namespace embedded -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_options.h b/src/mongo/db/storage/mobile/mobile_options.h deleted file mode 100644 index f5dd4ec71ac..00000000000 --- a/src/mongo/db/storage/mobile/mobile_options.h +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <string> - -#include "mongo/base/status.h" -#include "mongo/util/options_parser/environment.h" - -namespace mongo { -namespace embedded { - -struct MobileOptions { - // Initialize to broken nonsense defaults, the real ones are in IDL - uint32_t durabilityLevel = 0; - uint32_t cacheSizeKB = 0; - uint32_t mmapSizeKB = 0; - uint32_t journalSizeLimitKB = 0; - - double vacuumFreePageRatio = 0.0; - uint32_t vacuumFreeSizeMB = 0; - uint32_t vacuumCheckIntervalMinutes = 0; - - // This setting is not available for users to configure. Just meant to be able to disable this - // feature in certain unit tests. - bool disableVacuumJob = false; -}; - -extern MobileOptions mobileGlobalOptions; - -} // namespace embedded -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_options.idl b/src/mongo/db/storage/mobile/mobile_options.idl deleted file mode 100644 index c4ed0292208..00000000000 --- a/src/mongo/db/storage/mobile/mobile_options.idl +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright (C) 2019-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# <http://www.mongodb.com/licensing/server-side-public-license>. -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. -# - -global: - cpp_namespace: "mongo" - cpp_includes: - - "mongo/db/storage/mobile/mobile_options.h" - configs: - section: 'Mobile Storage Engine options' - source: [ cli, ini, yaml ] - initializer: - register: addMobileStorageOptionDefinitions - store: storeMobileStorageOptionDefinitions - -configs: - # Mobile storage engine options - "storage.mobile.durabilityLevel": - description: >- - How often you want fsync data to disk. This is a trade-off between - durability guarantees and performance. This value maps directly to - SQLite's synchronous PRAGMA. The default value is 1 or NORMAL - durability, where it's possible to have a write rolled back if the - device crashes or loses power (you are always safe from - application crashes). If write guarantees are more important than - performance, set this to 2 or 3. Conversely, if you are fine with - somewhat ephemeral local data then you can set this to 0. - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.durabilityLevel' - short_name: mobileDurabilityLevel - default: 1 - validator: {gte: 0, lte: 3} - - "storage.mobile.cacheSizeKB": - description: 'Maximum size in kilobytes of the database to be cached in memory.' - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.cacheSizeKB' - short_name: mobileCacheSizeKB - default: 10240 - validator: {gte: 0, lte: {expr: 'std::numeric_limits<int>::max()'}} - - "storage.mobile.mmapSizeKB": - description: 'Maximum size in kilobytes that can be used for memory mapped I/O.' - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.mmapSizeKB' - short_name: mobileMmapSizeKB - default: 51200 - validator: {gte: 0} - - "storage.mobile.journalSizeLimitKB": - description: 'Maximum size of the rollback journal in kilobytes that is stored on the file system.' - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.journalSizeLimitKB' - short_name: mobileJournalSizeLimitKB - default: 5120 - validator: {gte: 0} - - "storage.mobile.vacuumFreePageRatio": - description: 'Ratio of free pages to total pages that triggers vacuuming, if above, of the database files on the file system.' - arg_vartype: Double - cpp_varname: 'embedded::mobileGlobalOptions.vacuumFreePageRatio' - short_name: mobileVacuumFreePageRatio - default: 0.25 - validator: {gt: 0.0, lte: 1.0} - - "storage.mobile.vacuumFreeSizeMB": - description: 'Number of megabytes of free space in the database files that trigger vaccuming.' - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.vacuumFreeSizeMB' - short_name: mobileVacuumFreeSizeMB - default: 50 - validator: {gt: 0} - - "storage.mobile.vacuumCheckIntervalMinutes": - description: 'Interval in minutes to check if vacuum needs to be triggered.' - arg_vartype: Int - cpp_varname: 'embedded::mobileGlobalOptions.vacuumCheckIntervalMinutes' - short_name: mobileVacuumCheckIntervalMinutes - default: 10 - validator: {gt: 0} - diff --git a/src/mongo/db/storage/mobile/mobile_options_mongod.cpp b/src/mongo/db/storage/mobile/mobile_options_mongod.cpp deleted file mode 100644 index 58a1d95485e..00000000000 --- a/src/mongo/db/storage/mobile/mobile_options_mongod.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright (C) 2019-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/storage/mobile/mobile_options_gen.h" -#include "mongo/util/options_parser/startup_option_init.h" -#include "mongo/util/options_parser/startup_options.h" - -// When the mobile storage engine is used in embedded we don't need to do this. But when mongod we -// need to inject the mobile specific options. -namespace mongo { -MONGO_MODULE_STARTUP_OPTIONS_REGISTER(mobile_options_mongod_register)(InitializerContext*) { - return addMobileStorageOptionDefinitions(&optionenvironment::startupOptions); -} -MONGO_STARTUP_OPTIONS_STORE(mobile_options_mongod_store)(InitializerContext*) { - return storeMobileStorageOptionDefinitions(optionenvironment::startupOptionsParsed); -} -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_record_store.cpp b/src/mongo/db/storage/mobile/mobile_record_store.cpp deleted file mode 100644 index e07dd4ecb7b..00000000000 --- a/src/mongo/db/storage/mobile/mobile_record_store.cpp +++ /dev/null @@ -1,508 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include "mongo/db/storage/mobile/mobile_record_store.h" - -#include <memory> -#include <sqlite3.h> - -#include "mongo/base/static_assert.h" -#include "mongo/db/catalog/collection_options.h" -#include "mongo/db/jsobj.h" -#include "mongo/db/namespace_string.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/db/storage/oplog_hack.h" -#include "mongo/db/storage/recovery_unit.h" -#include "mongo/util/assert_util.h" -#include "mongo/util/log.h" -#include "mongo/util/str.h" - -namespace mongo { - -class MobileRecordStore::Cursor final : public SeekableRecordCursor { -public: - Cursor(OperationContext* opCtx, - const MobileRecordStore& rs, - const std::string& path, - const std::string& ident, - bool forward) - : _opCtx(opCtx), _forward(forward) { - - MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx); - _stmt = std::make_unique<SqliteStatement>(*session, - "SELECT rec_id, data from \"", - ident, - "\" ", - "WHERE rec_id ", - (forward ? ">" : "<"), - " ? ", - "ORDER BY rec_id ", - (forward ? "ASC" : "DESC"), - ";"); - - _startIdNum = (forward ? RecordId::min().repr() : RecordId::max().repr()); - _savedId = RecordId(_startIdNum); - - _stmt->bindInt(0, _savedId.repr()); - } - - boost::optional<Record> next() final { - if (_eof) { - return {}; - } - - int status = _stmt->step(); - // Reached end of result rows. - if (status == SQLITE_DONE) { - _eof = true; - _savedId = RecordId(_startIdNum); - return {}; - } - - // Checks no error was thrown and that step retrieved a row. - embedded::checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next"); - - long long recId = _stmt->getColInt(0); - const void* data = _stmt->getColBlob(1); - int64_t dataSize = _stmt->getColBytes(1); - - _savedId = RecordId(recId); - // The data returned from sqlite3_column_blob is only valid until the next call to - // sqlite3_step. Using getOwned copies the buffer so the data is not invalidated. - return {{_savedId, RecordData(static_cast<const char*>(data), dataSize).getOwned()}}; - } - - boost::optional<Record> seekExact(const RecordId& id) final { - // Set the saved position and use save/restore to reprepare the SQL statement so that - // the cursor restarts at the parameter id. - int decr = (_forward ? -1 : 1); - _savedId = RecordId(id.repr() + decr); - _eof = false; - - save(); - restore(); - - boost::optional<Record> rec = next(); - if (rec && rec->id != id) { - // The record we found isn't the one the caller asked for. - return boost::none; - } - - return rec; - } - - void save() final { - // SQLite acquires implicit locks over the snapshot this cursor is using. It is important - // to finalize the corresponding statement to release these locks. - _stmt->finalize(); - } - - void saveUnpositioned() final { - save(); - _savedId = RecordId(_startIdNum); - } - - bool restore() final { - if (_eof) { - return true; - } - - // Obtaining a session starts a read transaction if not done already. - MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx); - // save() finalized this cursor's SQLite statement. We need to prepare a new statement, - // before re-positioning it at the saved state. - _stmt->prepare(*session); - - _stmt->bindInt(0, _savedId.repr()); - return true; - } - - void detachFromOperationContext() final { - _opCtx = nullptr; - } - - void reattachToOperationContext(OperationContext* opCtx) final { - _opCtx = opCtx; - } - -private: - /** - * Resets the prepared statement. - */ - void _resetStatement() { - _stmt->reset(); - } - - OperationContext* _opCtx; - std::unique_ptr<SqliteStatement> _stmt; - - bool _eof = false; - - // Saved location for restoring. RecordId(0) means EOF. - RecordId _savedId; - - // Default start ID number that is specific to cursor direction. - int64_t _startIdNum; - - const bool _forward; -}; - -MobileRecordStore::MobileRecordStore(OperationContext* opCtx, - StringData ns, - const std::string& path, - const std::string& ident, - const CollectionOptions& options) - : RecordStore(ns), _path(path), _ident(ident) { - - // Mobile SE doesn't support creating an oplog, assert now - massert(ErrorCodes::IllegalOperation, - "Replication is not supported by the mobile storage engine", - !NamespaceString::oplog(ns)); - - // Mobile SE doesn't support creating a capped collection, assert now - massert(ErrorCodes::IllegalOperation, - "Capped Collections are not supported by the mobile storage engine", - !options.capped); - - // Determines the nextId to be used for a new record. - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - SqliteStatement maxRecIdStmt(*session, "SELECT IFNULL(MAX(rec_id), 0) FROM \"", _ident, "\";"); - - maxRecIdStmt.step(SQLITE_ROW); - - long long nextId = maxRecIdStmt.getColInt(0); - _nextIdNum.store(nextId + 1); -} - -const char* MobileRecordStore::name() const { - return "Mobile"; -} - -const std::string& MobileRecordStore::getIdent() const { - return _ident; -} - -void MobileRecordStore::_initDataSizeIfNeeded_inlock(OperationContext* opCtx) const { - if (_isDataSizeInitialized) { - return; - } - - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - SqliteStatement dataSizeStmt( - *session, "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"", _ident, "\";"); - - dataSizeStmt.step(SQLITE_ROW); - int64_t dataSize = dataSizeStmt.getColInt(0); - - _dataSize = dataSize; - _isDataSizeInitialized = true; -} - -long long MobileRecordStore::dataSize(OperationContext* opCtx) const { - stdx::lock_guard<Latch> lock(_dataSizeMutex); - _initDataSizeIfNeeded_inlock(opCtx); - return _dataSize; -} - -void MobileRecordStore::_initNumRecsIfNeeded_inlock(OperationContext* opCtx) const { - if (_isNumRecsInitialized) { - return; - } - - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - SqliteStatement numRecordsStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); - - numRecordsStmt.step(SQLITE_ROW); - - int64_t numRecs = numRecordsStmt.getColInt(0); - - _numRecs = numRecs; - _isNumRecsInitialized = true; -} - -long long MobileRecordStore::numRecords(OperationContext* opCtx) const { - stdx::lock_guard<Latch> lock(_numRecsMutex); - _initNumRecsIfNeeded_inlock(opCtx); - return _numRecs; -} - -bool MobileRecordStore::findRecord(OperationContext* opCtx, - const RecordId& recId, - RecordData* rd) const { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - SqliteStatement stmt(*session, "SELECT data FROM \"", _ident, "\" WHERE rec_id = ?;"); - - stmt.bindInt(0, recId.repr()); - - int status = stmt.step(); - if (status == SQLITE_DONE) { - return false; - } - embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); - - const void* recData = stmt.getColBlob(0); - int nBytes = stmt.getColBytes(0); - *rd = RecordData(static_cast<const char*>(recData), nBytes).getOwned(); - return true; -} - -void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& recId) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - - SqliteStatement dataSizeStmt( - *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); - dataSizeStmt.bindInt(0, recId.repr()); - dataSizeStmt.step(SQLITE_ROW); - - int64_t dataSizeBefore = dataSizeStmt.getColInt(0); - _changeNumRecs(opCtx, -1); - _changeDataSize(opCtx, -dataSizeBefore); - - SqliteStatement deleteStmt(*session, "DELETE FROM \"", _ident, "\" WHERE rec_id = ?;"); - deleteStmt.bindInt(0, recId.repr()); - deleteStmt.step(SQLITE_DONE); -} - -Status MobileRecordStore::insertRecords(OperationContext* opCtx, - std::vector<Record>* inOutRecords, - const std::vector<Timestamp>& timestamps) { - // Inserts record into SQLite table (or replaces if duplicate record id). - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - - SqliteStatement insertStmt( - *session, "INSERT OR REPLACE INTO \"", _ident, "\"(rec_id, data) VALUES(?, ?);"); - - for (auto& record : *inOutRecords) { - const auto data = record.data.data(); - const auto len = record.data.size(); - - _changeNumRecs(opCtx, 1); - _changeDataSize(opCtx, len); - - RecordId recId = _nextId(); - insertStmt.bindInt(0, recId.repr()); - insertStmt.bindBlob(1, data, len); - insertStmt.step(SQLITE_DONE); - - record.id = recId; - insertStmt.reset(); - } - - return Status::OK(); -} - -Status MobileRecordStore::updateRecord(OperationContext* opCtx, - const RecordId& recId, - const char* data, - int len) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - - SqliteStatement dataSizeStmt( - *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); - dataSizeStmt.bindInt(0, recId.repr()); - dataSizeStmt.step(SQLITE_ROW); - - int64_t dataSizeBefore = dataSizeStmt.getColInt(0); - _changeDataSize(opCtx, -dataSizeBefore + len); - - SqliteStatement updateStmt( - *session, "UPDATE \"", _ident, "\" SET data = ? ", "WHERE rec_id = ?;"); - updateStmt.bindBlob(0, data, len); - updateStmt.bindInt(1, recId.repr()); - updateStmt.step(SQLITE_DONE); - - return Status::OK(); -} - -bool MobileRecordStore::updateWithDamagesSupported() const { - return false; -} - -StatusWith<RecordData> MobileRecordStore::updateWithDamages( - OperationContext* opCtx, - const RecordId& recId, - const RecordData& oldRec, - const char* damageSource, - const mutablebson::DamageVector& damages) { - return RecordData(); -} - -std::unique_ptr<SeekableRecordCursor> MobileRecordStore::getCursor(OperationContext* opCtx, - bool forward) const { - return std::make_unique<Cursor>(opCtx, *this, _path, _ident, forward); -} - -/** - * SQLite does not directly support truncate. The SQLite documentation recommends a DELETE - * statement without a WHERE clause. A Truncate Optimizer deletes all of the table content - * without having to visit each row of the table individually. - */ -Status MobileRecordStore::truncate(OperationContext* opCtx) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - - int64_t numRecsBefore = numRecords(opCtx); - _changeNumRecs(opCtx, -numRecsBefore); - int64_t dataSizeBefore = dataSize(opCtx); - _changeDataSize(opCtx, -dataSizeBefore); - - SqliteStatement::execQuery(session, "DELETE FROM \"", _ident, "\";"); - - return Status::OK(); -} - -void MobileRecordStore::validate(OperationContext* opCtx, - ValidateResults* results, - BSONObjBuilder* output) { - embedded::doValidate(opCtx, results); -} - -/** - * Note: does not accurately return the size of the table on disk. Instead, it returns the number of - * bytes used to store the BSON documents. - */ -int64_t MobileRecordStore::storageSize(OperationContext* opCtx, - BSONObjBuilder* extraInfo, - int infoLevel) const { - return dataSize(opCtx); -} - -RecordId MobileRecordStore::_nextId() { - RecordId out = RecordId(_nextIdNum.fetchAndAdd(1)); - invariant(out.isNormal()); - return out; -} - -/** - * Keeps track of the changes to the number of records. - */ -class MobileRecordStore::NumRecsChange final : public RecoveryUnit::Change { -public: - NumRecsChange(MobileRecordStore* rs, int64_t diff) : _rs(rs), _diff(diff) {} - - void commit(boost::optional<Timestamp>) override {} - - void rollback() override { - stdx::lock_guard<Latch> lock(_rs->_numRecsMutex); - _rs->_numRecs -= _diff; - } - -private: - MobileRecordStore* _rs; - int64_t _diff; -}; - -void MobileRecordStore::_changeNumRecs(OperationContext* opCtx, int64_t diff) { - stdx::lock_guard<Latch> lock(_numRecsMutex); - opCtx->recoveryUnit()->registerChange(std::make_unique<NumRecsChange>(this, diff)); - _initNumRecsIfNeeded_inlock(opCtx); - _numRecs += diff; -} - -bool MobileRecordStore::_resetNumRecsIfNeeded(OperationContext* opCtx, int64_t newNumRecs) { - bool wasReset = false; - int64_t currNumRecs = numRecords(opCtx); - if (currNumRecs != newNumRecs) { - wasReset = true; - stdx::lock_guard<Latch> lock(_numRecsMutex); - _numRecs = newNumRecs; - } - return wasReset; -} - -/** - * Keeps track of the total data size. - */ -class MobileRecordStore::DataSizeChange final : public RecoveryUnit::Change { -public: - DataSizeChange(MobileRecordStore* rs, int64_t diff) : _rs(rs), _diff(diff) {} - - void commit(boost::optional<Timestamp>) override {} - - void rollback() override { - stdx::lock_guard<Latch> lock(_rs->_dataSizeMutex); - _rs->_dataSize -= _diff; - } - -private: - MobileRecordStore* _rs; - int64_t _diff; -}; - -void MobileRecordStore::_changeDataSize(OperationContext* opCtx, int64_t diff) { - stdx::lock_guard<Latch> lock(_dataSizeMutex); - opCtx->recoveryUnit()->registerChange(std::make_unique<DataSizeChange>(this, diff)); - _initDataSizeIfNeeded_inlock(opCtx); - _dataSize += diff; -} - -bool MobileRecordStore::_resetDataSizeIfNeeded(OperationContext* opCtx, int64_t newDataSize) { - bool wasReset = false; - int64_t currDataSize = dataSize(opCtx); - - if (currDataSize != _dataSize) { - wasReset = true; - stdx::lock_guard<Latch> lock(_dataSizeMutex); - _dataSize = newDataSize; - } - return wasReset; -} - -boost::optional<RecordId> MobileRecordStore::oplogStartHack( - OperationContext* opCtx, const RecordId& startingPosition) const { - return {}; -} - -/** - * Creates a new record store within SQLite. - * The method is not transactional. Callers are responsible for handling transactional semantics. - */ -void MobileRecordStore::create(OperationContext* opCtx, const std::string& ident) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - SqliteStatement::execQuery(session, - "CREATE TABLE IF NOT EXISTS \"", - ident, - "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));"); -} - -void MobileRecordStore::updateStatsAfterRepair(OperationContext* opCtx, - long long numRecords, - long long dataSize) { - _resetDataSizeIfNeeded(opCtx, (int64_t)dataSize); - _resetNumRecsIfNeeded(opCtx, (int64_t)numRecords); -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_record_store.h b/src/mongo/db/storage/mobile/mobile_record_store.h deleted file mode 100644 index d595ec980e0..00000000000 --- a/src/mongo/db/storage/mobile/mobile_record_store.h +++ /dev/null @@ -1,193 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <memory> -#include <string> - -#include "mongo/db/catalog/collection_options.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/record_store.h" -#include "mongo/platform/atomic_word.h" - -namespace mongo { - -/** - * A RecordStore that stores all data in SQLite. - */ -class MobileRecordStore : public RecordStore { -public: - explicit MobileRecordStore(OperationContext* opCtx, - StringData ns, - const std::string& path, - const std::string& ident, - const CollectionOptions& options); - - const char* name() const override; - - const std::string& getIdent() const override; - - bool findRecord(OperationContext* opCtx, const RecordId& recId, RecordData* rd) const override; - - void deleteRecord(OperationContext* opCtx, const RecordId& dl) override; - - Status insertRecords(OperationContext* opCtx, - std::vector<Record>* inOutRecords, - const std::vector<Timestamp>& timestamps) override; - - Status updateRecord(OperationContext* opCtx, - const RecordId& oldLocation, - const char* data, - int len) override; - - bool updateWithDamagesSupported() const override; - - StatusWith<RecordData> updateWithDamages(OperationContext* opCtx, - const RecordId& recId, - const RecordData& oldRec, - const char* damageSource, - const mutablebson::DamageVector& damages) override; - - std::unique_ptr<SeekableRecordCursor> getCursor(OperationContext* opCtx, - bool forward) const override; - - Status truncate(OperationContext* opCtx) override; - - void cappedTruncateAfter(OperationContext* opCtx, RecordId end, bool inclusive) override { - // Capped Collections are not supported, do nothing - } - - /** - * Validates the entire database file, not just the table used by this record store. - */ - void validate(OperationContext* opCtx, - ValidateResults* results, - BSONObjBuilder* output) override; - - void appendCustomStats(OperationContext* opCtx, - BSONObjBuilder* result, - double scale) const override { - // No custom stats to add - } - - int64_t storageSize(OperationContext* opCtx, - BSONObjBuilder* extraInfo = nullptr, - int infoLevel = 0) const override; - - long long dataSize(OperationContext* opCtx) const override; - - long long numRecords(OperationContext* opCtx) const override; - - boost::optional<RecordId> oplogStartHack(OperationContext* opCtx, - const RecordId& startingPosition) const override; - - void waitForAllEarlierOplogWritesToBeVisible(OperationContext* opCtx) const override {} - - void updateStatsAfterRepair(OperationContext* opCtx, long long numRecords, long long dataSize); - - bool isCapped() const override { - // Capped Collections are not supported - return false; - } - - void setCappedCallback(CappedCallback* cb) override { - // Capped Collections are not supported, do nothing - } - - // Not in record store API. - - /** - * Creates a new record store inside SQLite. - * Transational semantics are handled by the caller. - */ - static void create(OperationContext* opCtx, const std::string& ident); - -private: - class InsertChange; - class RemoveChange; - class TruncateChange; - - class NumRecsChange; - class DataSizeChange; - - class Cursor; - - RecordId _nextId(); - - const std::string _path; - const std::string _ident; - - AtomicWord<long long> _nextIdNum; - - /** - * Fetches the number of records from the database. _numRecsMutex should be locked before this - * is called. - */ - void _initNumRecsIfNeeded_inlock(OperationContext* opCtx) const; - - /** - * Updates _numRecords. This must be called before the actual change is made to the database. - */ - void _changeNumRecs(OperationContext* opCtx, int64_t diff); - - /** - * Resets _numRecords to the new value. Returns true if _numRecs was reset; returns false - * otherwise. - */ - bool _resetNumRecsIfNeeded(OperationContext* opCtx, int64_t newNumRecs); - - mutable int64_t _numRecs; - mutable Mutex _numRecsMutex = MONGO_MAKE_LATCH("MobileRecordStore::_numRecsMutex"); - mutable bool _isNumRecsInitialized = false; - - /** - * Fetches the data size from the database. _dataSizeMutex should be locked before this is - * called. - */ - void _initDataSizeIfNeeded_inlock(OperationContext* opCtx) const; - - /** - * Updates _dataSize. This must be called before the actual change is made to the database. - */ - void _changeDataSize(OperationContext* opCtx, int64_t diff); - - /** - * Resets _dataSize to the new value. Returns true if _dataSize was reset; returns false - * otherwise. - */ - bool _resetDataSizeIfNeeded(OperationContext* opCtx, int64_t newDataSize); - - mutable int64_t _dataSize; - mutable Mutex _dataSizeMutex = MONGO_MAKE_LATCH("MobileRecordStore::_dataSizeMutex"); - mutable bool _isDataSizeInitialized = false; -}; - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_record_store_test.cpp b/src/mongo/db/storage/mobile/mobile_record_store_test.cpp deleted file mode 100644 index 081f5d8990d..00000000000 --- a/src/mongo/db/storage/mobile/mobile_record_store_test.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include <boost/filesystem.hpp> -#include <boost/system/error_code.hpp> -#include <memory> - -#include "mongo/base/init.h" -#include "mongo/db/catalog/collection_options.h" -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/storage/mobile/mobile_options_gen.h" -#include "mongo/db/storage/mobile/mobile_record_store.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/db/storage/record_store_test_harness.h" -#include "mongo/unittest/temp_dir.h" -#include "mongo/unittest/unittest.h" -#include "mongo/util/options_parser/options_parser.h" -#include "mongo/util/options_parser/startup_options.h" - - -namespace mongo { - -namespace { - -static int inc = 0; - -class MobileRecordStoreHarnessHelper final : public RecordStoreHarnessHelper { -public: - MobileRecordStoreHarnessHelper() : _dbPath("mobile_record_store_harness") { - // TODO: Determine if this should be util function. - boost::system::error_code err; - boost::filesystem::path dir(_dbPath.path()); - - if (!boost::filesystem::exists(dir, err)) { - if (err) { - uasserted(ErrorCodes::UnknownError, err.message()); - } - - boost::filesystem::create_directory(dir, err); - if (err) { - uasserted(ErrorCodes::UnknownError, err.message()); - } - } - - boost::filesystem::path file("mobile.sqlite"); - boost::filesystem::path fullPath = dir / file; - - if (boost::filesystem::exists(fullPath, err)) { - if (err) { - uasserted(ErrorCodes::UnknownError, err.message()); - } else if (!boost::filesystem::is_regular_file(fullPath)) { - std::string errMsg("Failed to open " + dir.generic_string() + - ": not a regular file"); - uasserted(ErrorCodes::BadValue, errMsg); - } - } - - _fullPath = fullPath.string(); - - addMobileStorageOptionDefinitions(&optionenvironment::startupOptions).ignore(); - optionenvironment::OptionsParser parser; - std::vector<std::string> args; - std::map<std::string, std::string> env; - parser - .run(optionenvironment::startupOptions, - args, - env, - &optionenvironment::startupOptionsParsed) - .ignore(); - storeMobileStorageOptionDefinitions(optionenvironment::startupOptionsParsed).ignore(); - - _sessionPool.reset(new MobileSessionPool(_fullPath, embedded::mobileGlobalOptions)); - } - - std::unique_ptr<RecordStore> newNonCappedRecordStore() override { - inc++; - return newNonCappedRecordStore("table_" + std::to_string(inc)); - } - - std::unique_ptr<RecordStore> newNonCappedRecordStore(const std::string& ns) override { - ServiceContext::UniqueOperationContext opCtx(this->newOperationContext()); - MobileRecordStore::create(opCtx.get(), ns); - return std::make_unique<MobileRecordStore>( - opCtx.get(), ns, _fullPath, ns, CollectionOptions()); - } - - std::unique_ptr<RecordStore> newCappedRecordStore(int64_t cappedMaxSize, - int64_t cappedMaxDocs) override { - inc++; - return newCappedRecordStore("table_" + std::to_string(inc), cappedMaxSize, cappedMaxDocs); - } - - std::unique_ptr<RecordStore> newCappedRecordStore(const std::string& ns, - int64_t cappedMaxSize, - int64_t cappedMaxDocs) override { - ServiceContext::UniqueOperationContext opCtx(this->newOperationContext()); - MobileRecordStore::create(opCtx.get(), ns); - CollectionOptions options; - options.capped = true; - options.cappedSize = cappedMaxSize; - options.cappedMaxDocs = cappedMaxDocs; - return std::make_unique<MobileRecordStore>(opCtx.get(), ns, _fullPath, ns, options); - } - - std::unique_ptr<RecoveryUnit> newRecoveryUnit() final { - return std::make_unique<MobileRecoveryUnit>(_sessionPool.get()); - } - - bool supportsDocLocking() final { - return false; - } - -private: - unittest::TempDir _dbPath; - std::string _fullPath; - std::unique_ptr<MobileSessionPool> _sessionPool; -}; - -std::unique_ptr<RecordStoreHarnessHelper> makeMobileRecordStoreHarnessHelper() { - return std::make_unique<MobileRecordStoreHarnessHelper>(); -} - -MONGO_INITIALIZER(RegisterHarnessFactory)(InitializerContext* const) { - mongo::registerRecordStoreHarnessHelperFactory(makeMobileRecordStoreHarnessHelper); - return Status::OK(); -} -} // namespace -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp deleted file mode 100644 index 3cc2433870b..00000000000 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include <string> - -#include "mongo/db/concurrency/d_concurrency.h" -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/db/storage/sorted_data_interface.h" -#include "mongo/util/log.h" - -#define RECOVERY_UNIT_TRACE() LOG(MOBILE_TRACE_LEVEL) << "MobileSE: RecoveryUnit ID:" << _id << " " - -namespace mongo { - -AtomicWord<long long> MobileRecoveryUnit::_nextID(0); - -MobileRecoveryUnit::MobileRecoveryUnit(MobileSessionPool* sessionPool) - : _isReadOnly(true), _sessionPool(sessionPool) { - // Increment the global instance count and assign this instance an id. - _id = _nextID.addAndFetch(1); - - RECOVERY_UNIT_TRACE() << "Created."; -} - -MobileRecoveryUnit::~MobileRecoveryUnit() { - invariant(!_inUnitOfWork(), toString(_getState())); - _abort(); - RECOVERY_UNIT_TRACE() << "Destroyed."; -} - -void MobileRecoveryUnit::_commit() { - if (_session && _isActive()) { - _txnClose(true); - } - _setState(State::kCommitting); - commitRegisteredChanges(boost::none); - _setState(State::kInactive); -} - -void MobileRecoveryUnit::_abort() { - if (_session && _isActive()) { - _txnClose(false); - } - _setState(State::kAborting); - abortRegisteredChanges(); - - invariant(!_isActive(), toString(_getState())); - _setState(State::kInactive); -} - -void MobileRecoveryUnit::beginUnitOfWork(OperationContext* opCtx) { - invariant(!_inUnitOfWork(), toString(_getState())); - - RECOVERY_UNIT_TRACE() << "Unit of work Active."; - - if (_isActive()) { - // Confirm a write transaction is not running - invariant(_isReadOnly); - - // Rollback read transaction running outside wuow - _txnClose(false); - } - _setState(State::kInactiveInUnitOfWork); - _txnOpen(opCtx, false); -} - -void MobileRecoveryUnit::doCommitUnitOfWork() { - invariant(_inUnitOfWork(), toString(_getState())); - - RECOVERY_UNIT_TRACE() << "Unit of work commited, marked inactive."; - - _commit(); -} - -void MobileRecoveryUnit::doAbortUnitOfWork() { - invariant(_inUnitOfWork(), toString(_getState())); - - RECOVERY_UNIT_TRACE() << "Unit of work aborted, marked inactive."; - - _abort(); -} - -bool MobileRecoveryUnit::waitUntilDurable(OperationContext* opCtx) { - // This is going to be slow as we're taking a global X lock and doing a full checkpoint. This - // should not be needed to do on Android or iOS if we are on WAL and synchronous=NORMAL which - // are our default settings. The system will make sure any non-flushed writes will not be lost - // before going down but our powercycle test bench require it. Therefore make sure embedded does - // not call this (by disabling writeConcern j:true) but allow it when this is used inside - // mongod. - if (_sessionPool->getOptions().durabilityLevel < 2) { - _ensureSession(opCtx); - RECOVERY_UNIT_TRACE() << "waitUntilDurable called, attempting to perform a checkpoint"; - int framesInWAL = 0; - int checkpointedFrames = 0; - int ret; - { - Lock::GlobalLock lk(opCtx, MODE_X); - // Use FULL mode to guarantee durability - ret = sqlite3_wal_checkpoint_v2(_session.get()->getSession(), - nullptr, - SQLITE_CHECKPOINT_FULL, - &framesInWAL, - &checkpointedFrames); - } - embedded::checkStatus(ret, SQLITE_OK, "sqlite3_wal_checkpoint_v2"); - fassert(51164, - framesInWAL != -1 && checkpointedFrames != -1 && framesInWAL == checkpointedFrames); - RECOVERY_UNIT_TRACE() << "Checkpointed " << checkpointedFrames << " of the " << framesInWAL - << " total frames in the WAL"; - } else { - RECOVERY_UNIT_TRACE() << "No checkpoint attempted -- in full synchronous mode"; - } - - return true; -} - -void MobileRecoveryUnit::doAbandonSnapshot() { - invariant(!_inUnitOfWork(), toString(_getState())); - if (_isActive()) { - // We can't be in a WriteUnitOfWork, so it is safe to rollback. - _txnClose(false); - } - _setState(State::kInactive); -} - -MobileSession* MobileRecoveryUnit::getSession(OperationContext* opCtx, bool readOnly) { - RECOVERY_UNIT_TRACE() << "getSession called with readOnly:" << (readOnly ? "TRUE" : "FALSE"); - - invariant(_inUnitOfWork() || readOnly); - if (!_isActive()) { - _txnOpen(opCtx, readOnly); - } - - return _session.get(); -} - -MobileSession* MobileRecoveryUnit::getSessionNoTxn(OperationContext* opCtx) { - _ensureSession(opCtx); - return _session.get(); -} - -void MobileRecoveryUnit::assertInActiveTxn() const { - fassert(37050, _isActive()); -} - -void MobileRecoveryUnit::_ensureSession(OperationContext* opCtx) { - RECOVERY_UNIT_TRACE() << "Creating new session:" << (_session ? "NO" : "YES"); - if (!_session) { - _session = _sessionPool->getSession(opCtx); - } -} - -void MobileRecoveryUnit::_txnOpen(OperationContext* opCtx, bool readOnly) { - invariant(!_isActive(), toString(_getState())); - RECOVERY_UNIT_TRACE() << "_txnOpen called with readOnly:" << (readOnly ? "TRUE" : "FALSE"); - _ensureSession(opCtx); - - /* - * Starting a transaction with the "BEGIN" statement doesn't take an immediate lock. - * SQLite defers taking any locks until the database is first accessed. This creates the - * possibility of having multiple transactions opened in parallel. All sessions except the - * first to request the access get a database locked error. - * However, "BEGIN IMMEDIATE" forces SQLite to take a lock immediately. If another session - * tries to create a transaction in parallel, it receives a busy error and then retries. - * Reads outside these explicit transactions proceed unaffected. - */ - - // Check for correct locking at higher levels - if (readOnly) { - // Confirm that this reader has taken a shared lock - if (!opCtx->lockState()->isLockHeldForMode(resourceIdGlobal, MODE_S)) { - opCtx->lockState()->dump(); - invariant(!"Reading without a shared lock"); - } - SqliteStatement::execQuery(_session.get(), "BEGIN"); - } else { - // Single writer allowed at a time, confirm a global write lock has been taken - if (!opCtx->lockState()->isLockHeldForMode(resourceIdGlobal, MODE_X)) { - opCtx->lockState()->dump(); - invariant(!"Writing without an exclusive lock"); - } - SqliteStatement::execQuery(_session.get(), "BEGIN EXCLUSIVE"); - } - - _isReadOnly = readOnly; - _setState(_inUnitOfWork() ? State::kActive : State::kActiveNotInUnitOfWork); -} - -void MobileRecoveryUnit::_txnClose(bool commit) { - invariant(_isActive(), toString(_getState())); - RECOVERY_UNIT_TRACE() << "_txnClose called with " << (commit ? "commit " : "rollback "); - - if (commit) { - SqliteStatement::execQuery(_session.get(), "COMMIT"); - } else { - SqliteStatement::execQuery(_session.get(), "ROLLBACK"); - } - - _isReadOnly = true; // I don't suppose we need this, but no harm in doing so -} - -void MobileRecoveryUnit::enqueueFailedDrop(std::string& dropQuery) { - _sessionPool->failedDropsQueue.enqueueOp(dropQuery); -} -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.h b/src/mongo/db/storage/mobile/mobile_recovery_unit.h deleted file mode 100644 index 6304cde8a5f..00000000000 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.h +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <memory> -#include <string> -#include <vector> - -#include "mongo/base/checked_cast.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/record_id.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" -#include "mongo/db/storage/recovery_unit.h" -#include "mongo/db/storage/snapshot.h" -#include "mongo/platform/atomic_word.h" - -namespace mongo { - -class SortedDataInterface; - -class MobileRecoveryUnit final : public RecoveryUnit { -public: - MobileRecoveryUnit(MobileSessionPool* sessionPool); - virtual ~MobileRecoveryUnit(); - - void beginUnitOfWork(OperationContext* opCtx) override; - bool waitUntilDurable(OperationContext* opCtx) override; - - MobileSession* getSession(OperationContext* opCtx, bool readOnly = true); - - MobileSession* getSessionNoTxn(OperationContext* opCtx); - - bool inActiveTxn() const { - return _isActive(); - } - - void assertInActiveTxn() const; - - void enqueueFailedDrop(std::string& dropQuery); - - static MobileRecoveryUnit* get(OperationContext* opCtx) { - return checked_cast<MobileRecoveryUnit*>(opCtx->recoveryUnit()); - } - - void setOrderedCommit(bool orderedCommit) override {} - -private: - void doCommitUnitOfWork() override; - void doAbortUnitOfWork() override; - - void doAbandonSnapshot() override; - - void _abort(); - void _commit(); - - void _ensureSession(OperationContext* opCtx); - void _txnClose(bool commit); - void _txnOpen(OperationContext* opCtx, bool readOnly); - void _upgradeToWriteSession(OperationContext* opCtx); - - static AtomicWord<long long> _nextID; - uint64_t _id; - bool _isReadOnly; - - std::string _path; - MobileSessionPool* _sessionPool; - std::unique_ptr<MobileSession> _session; -}; - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_session.cpp b/src/mongo/db/storage/mobile/mobile_session.cpp deleted file mode 100644 index 1b32da9b80e..00000000000 --- a/src/mongo/db/storage/mobile/mobile_session.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/platform/basic.h" - -#include <sqlite3.h> - -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" - -namespace mongo { - -MobileSession::MobileSession(sqlite3* session, MobileSessionPool* sessionPool) - : _session(session), _sessionPool(sessionPool) {} - -MobileSession::~MobileSession() { - // Releases this session back to the session pool. - _sessionPool->releaseSession(this); -} - -sqlite3* MobileSession::getSession() const { - return _session; -} -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_session.h b/src/mongo/db/storage/mobile/mobile_session.h deleted file mode 100644 index 6e48caed880..00000000000 --- a/src/mongo/db/storage/mobile/mobile_session.h +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <sqlite3.h> -#include <string> - -#include "mongo/db/storage/mobile/mobile_session_pool.h" - -namespace mongo { -class MobileSessionPool; - -/** - * This class manages a SQLite database connection object. - */ -class MobileSession final { - MobileSession(const MobileSession&) = delete; - MobileSession& operator=(const MobileSession&) = delete; - -public: - MobileSession(sqlite3* session, MobileSessionPool* sessionPool); - - ~MobileSession(); - - /** - * Returns a pointer to the underlying SQLite connection object. - */ - sqlite3* getSession() const; - -private: - sqlite3* _session; - MobileSessionPool* _sessionPool; -}; -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_session_pool.cpp b/src/mongo/db/storage/mobile/mobile_session_pool.cpp deleted file mode 100644 index a8a211bcc6b..00000000000 --- a/src/mongo/db/storage/mobile/mobile_session_pool.cpp +++ /dev/null @@ -1,191 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include <queue> -#include <sqlite3.h> -#include <string> -#include <vector> - -#include "mongo/db/client.h" -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/db/storage/mobile/mobile_session_pool.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/platform/mutex.h" -#include "mongo/util/log.h" - -namespace mongo { - -MobileDelayedOpQueue::MobileDelayedOpQueue() : _isEmpty(true) {} - -void MobileDelayedOpQueue::enqueueOp(std::string& opQuery) { - _queueMutex.lock(); - // If the queue is empty till now, update the cached atomic to reflect the new state. - if (_opQueryQueue.empty()) - _isEmpty.store(false); - _opQueryQueue.push(opQuery); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Enqueued operation for delayed execution: " << opQuery; - _queueMutex.unlock(); -} - -void MobileDelayedOpQueue::execAndDequeueOp(MobileSession* session) { - std::string opQuery; - - _queueMutex.lock(); - if (!_opQueryQueue.empty()) { - opQuery = _opQueryQueue.front(); - _opQueryQueue.pop(); - // If the queue is empty now, set the cached atomic to reflect the new state. - if (_opQueryQueue.empty()) - _isEmpty.store(true); - } - _queueMutex.unlock(); - - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Retrying previously enqueued operation: " << opQuery; - try { - SqliteStatement::execQuery(session, opQuery); - } catch (const WriteConflictException&) { - // It is possible that this operation fails because of a transaction running in parallel. - // We re-enqueue it for now and keep retrying later. - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Caught WriteConflictException while executing " - " previously enqueued operation, re-enquing it"; - enqueueOp(opQuery); - } -} - -void MobileDelayedOpQueue::execAndDequeueAllOps(MobileSession* session) { - // Keep trying till the queue empties - while (!_isEmpty.load()) - execAndDequeueOp(session); -} - -bool MobileDelayedOpQueue::isEmpty() { - return (_isEmpty.load()); -} - -MobileSessionPool::MobileSessionPool(const std::string& path, - const embedded::MobileOptions& options, - std::uint64_t maxPoolSize) - : _path(path), _options(options), _maxPoolSize(maxPoolSize) {} - -MobileSessionPool::~MobileSessionPool() { - shutDown(); -} - -std::unique_ptr<MobileSession> MobileSessionPool::getSession(OperationContext* opCtx) { - stdx::unique_lock<Latch> lk(_mutex); - - // We should never be able to get here after _shuttingDown is set, because no new operations - // should be allowed to start. - invariant(!_shuttingDown); - - // Checks if there is an open session available. - if (!_sessions.empty()) { - sqlite3* session = _popSession_inlock(); - return std::make_unique<MobileSession>(session, this); - } - - // Checks if a new session can be opened. - if (_curPoolSize < _maxPoolSize) { - sqlite3* session; - int status = sqlite3_open(_path.c_str(), &session); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); - embedded::configureSession(session, _options); - _curPoolSize++; - return std::make_unique<MobileSession>(session, this); - } - - // There are no open sessions available and the maxPoolSize has been reached. - // waitForConditionOrInterrupt is notified when an open session is released and available. - opCtx->waitForConditionOrInterrupt( - _releasedSessionNotifier, lk, [&] { return !_sessions.empty(); }); - - sqlite3* session = _popSession_inlock(); - return std::make_unique<MobileSession>(session, this); -} - -void MobileSessionPool::releaseSession(MobileSession* session) { - // Retry drop that have been queued on failure - if (!failedDropsQueue.isEmpty()) - failedDropsQueue.execAndDequeueOp(session); - - stdx::lock_guard<Latch> lk(_mutex); - _sessions.push_back(session->getSession()); - _releasedSessionNotifier.notify_one(); -} - -void MobileSessionPool::shutDown() { - stdx::unique_lock<Latch> lk(_mutex); - _shuttingDown = true; - - // Retrieve the operation context from the thread's client if the client exists. - if (haveClient()) { - OperationContext* opCtx = cc().getOperationContext(); - - // Locks if the operation context still exists. - if (opCtx) { - opCtx->waitForConditionOrInterrupt( - _releasedSessionNotifier, lk, [&] { return _sessions.size() == _curPoolSize; }); - } - } else { - _releasedSessionNotifier.wait(lk, [&] { return _sessions.size() == _curPoolSize; }); - } - - // Retry all the drops that have been queued on failure. - // Create a new sqlite session to do so, all other sessions might have been closed already. - if (!failedDropsQueue.isEmpty()) { - sqlite3* session; - - int status = sqlite3_open(_path.c_str(), &session); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); - std::unique_ptr<MobileSession> mobSession = std::make_unique<MobileSession>(session, this); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Executing queued drops at shutdown"; - failedDropsQueue.execAndDequeueAllOps(mobSession.get()); - sqlite3_close(session); - } - - for (auto&& session : _sessions) { - sqlite3_close(session); - } -} - -// This method should only be called when _sessions is locked. -sqlite3* MobileSessionPool::_popSession_inlock() { - sqlite3* session = _sessions.back(); - _sessions.pop_back(); - return session; -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_session_pool.h b/src/mongo/db/storage/mobile/mobile_session_pool.h deleted file mode 100644 index 031953cdfb3..00000000000 --- a/src/mongo/db/storage/mobile/mobile_session_pool.h +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <queue> -#include <sqlite3.h> -#include <string> -#include <vector> - -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/platform/mutex.h" - -namespace mongo { -class MobileSession; - -/** - * This class manages a queue of operations delayed for some reason - */ -class MobileDelayedOpQueue final { - MobileDelayedOpQueue(const MobileDelayedOpQueue&) = delete; - MobileDelayedOpQueue& operator=(const MobileDelayedOpQueue&) = delete; - -public: - MobileDelayedOpQueue(); - void enqueueOp(std::string& opQuery); - void execAndDequeueOp(MobileSession* session); - void execAndDequeueAllOps(MobileSession* session); - bool isEmpty(); - -private: - AtomicWord<bool> _isEmpty; - Mutex _queueMutex = MONGO_MAKE_LATCH("MobileDelayedOpQueue::_queueMutex"); - std::queue<std::string> _opQueryQueue; -}; - -/** - * This class manages a pool of open sqlite3* objects. - */ -class MobileSessionPool final { - MobileSessionPool(const MobileSessionPool&) = delete; - MobileSessionPool& operator=(const MobileSessionPool&) = delete; - -public: - MobileSessionPool(const std::string& path, - const embedded::MobileOptions& options, - std::uint64_t maxPoolSize = 80); - - ~MobileSessionPool(); - - /** - * Returns a smart pointer to a previously released session for reuse, or creates a new session. - */ - std::unique_ptr<MobileSession> getSession(OperationContext* opCtx); - - /** - * Returns a session to the pool for later reuse. - */ - void releaseSession(MobileSession* session); - - /** - * Transitions the pool to shutting down mode. It waits until all sessions are released back - * into the pool and closes all open sessions. - */ - void shutDown(); - - // Failed drops get queued here and get re-tried periodically - MobileDelayedOpQueue failedDropsQueue; - - // Returns the mobile options associated with this storage engine instance - const embedded::MobileOptions& getOptions() const { - return _options; - } - -private: - /** - * Gets the front element from _sessions and then pops it off the queue. - */ - sqlite3* _popSession_inlock(); - - // This is used to lock the _sessions vector. - Mutex _mutex; - stdx::condition_variable _releasedSessionNotifier; - - std::string _path; - const embedded::MobileOptions& _options; - - /** - * PoolSize is the number of open sessions associated with the session pool. - */ - std::uint64_t _maxPoolSize = 80; - std::uint64_t _curPoolSize = 0; - bool _shuttingDown = false; - - using SessionPool = std::vector<sqlite3*>; - SessionPool _sessions; -}; -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp deleted file mode 100644 index 947746277cb..00000000000 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/platform/basic.h" - -#include <sqlite3.h> -#include <string> - -#include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" -#include "mongo/util/assert_util.h" -#include "mongo/util/log.h" -#include "mongo/util/scopeguard.h" - -#define SQLITE_STMT_TRACE() LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite Stmt ID:" << _id << " " -#define SQLITE_STMT_TRACE_ENABLED() \ - (shouldLog(MongoLogDefaultComponent_component, \ - ::mongo::LogstreamBuilder::severityCast(MOBILE_TRACE_LEVEL))) - -namespace mongo { - -AtomicWord<long long> SqliteStatement::_nextID(0); - -void SqliteStatement::finalize() { - if (!_stmt) { - return; - } - SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery.data(); - - int status = sqlite3_finalize(_stmt); - fassert(37053, status == _exceptionStatus); - _stmt = nullptr; -} - -void SqliteStatement::prepare(const MobileSession& session) { - SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery.data(); - - int status = sqlite3_prepare_v2( - session.getSession(), _sqlQuery.data(), _sqlQuery.size(), &_stmt, nullptr); - if (status == SQLITE_BUSY) { - SQLITE_STMT_TRACE() << "Throwing writeConflictException, " - << "SQLITE_BUSY while preparing: " << _sqlQuery.data(); - throw WriteConflictException(); - } else if (status != SQLITE_OK) { - SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery.data(); - std::string errMsg = "sqlite3_prepare_v2 failed: "; - errMsg += sqlite3_errstr(status); - uasserted(ErrorCodes::UnknownError, errMsg); - } -} - -SqliteStatement::~SqliteStatement() { - finalize(); - - static_assert( - sizeof(SqliteStatement) == - sizeof(std::aligned_storage_t<sizeof(SqliteStatement), alignof(SqliteStatement)>), - "expected size to be exactly its aligned storage size to not waste memory, " - "adjust kMaxFixedSize to make this true"); -} - -void SqliteStatement::bindInt(int paramIndex, int64_t intValue) { - // SQLite bind methods begin paramater indexes at 1 rather than 0. - int status = sqlite3_bind_int64(_stmt, paramIndex + 1, intValue); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); -} - -void SqliteStatement::bindBlob(int paramIndex, const void* data, int len) { - // SQLite bind methods begin paramater indexes at 1 rather than 0. - int status = sqlite3_bind_blob(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); -} - -void SqliteStatement::bindText(int paramIndex, const char* data, int len) { - // SQLite bind methods begin paramater indexes at 1 rather than 0. - int status = sqlite3_bind_text(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); -} - -void SqliteStatement::clearBindings() { - int status = sqlite3_clear_bindings(_stmt); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings"); -} - -int SqliteStatement::step(int desiredStatus) { - int status = sqlite3_step(_stmt); - - // A non-negative desiredStatus indicates that checkStatus should assert that the returned - // status is equivalent to the desired status. - if (desiredStatus >= 0) { - embedded::checkStatus(status, desiredStatus, "sqlite3_step"); - } - - if (SQLITE_STMT_TRACE_ENABLED()) { - char* full_stmt = sqlite3_expanded_sql(_stmt); - SQLITE_STMT_TRACE() << embedded::sqliteStatusToStr(status) - << " - on stepping: " << full_stmt; - sqlite3_free(full_stmt); - } - - return status; -} - -int64_t SqliteStatement::getColInt(int colIndex) { - return sqlite3_column_int64(_stmt, colIndex); -} - -const void* SqliteStatement::getColBlob(int colIndex) { - return sqlite3_column_blob(_stmt, colIndex); -} - -int64_t SqliteStatement::getColBytes(int colIndex) { - return sqlite3_column_bytes(_stmt, colIndex); -} - -const char* SqliteStatement::getColText(int colIndex) { - return reinterpret_cast<const char*>(sqlite3_column_text(_stmt, colIndex)); -} - -void SqliteStatement::_execQuery(sqlite3* session, const char* query) { - LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite sqlite3_exec: " << query; - - char* errMsg = nullptr; - int status = sqlite3_exec(session, query, nullptr, nullptr, &errMsg); - - if (status == SQLITE_BUSY || status == SQLITE_LOCKED) { - LOG(MOBILE_TRACE_LEVEL) << "MobileSE: " << (status == SQLITE_BUSY ? "Busy" : "Locked") - << " - Throwing WriteConflictException on sqlite3_exec: " << query; - throw WriteConflictException(); - } - - // The only return value from sqlite3_exec in a success case is SQLITE_OK. - embedded::checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - - // When the error message is not NULL, it is allocated through sqlite3_malloc and must be freed - // before exiting the method. If the error message is NULL, sqlite3_free is a no-op. - sqlite3_free(errMsg); -} - -void SqliteStatement::reset() { - int status = sqlite3_reset(_stmt); - embedded::checkStatus(status, SQLITE_OK, "sqlite3_reset"); -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h deleted file mode 100644 index 31a61f7ffa8..00000000000 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h +++ /dev/null @@ -1,235 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include <sqlite3.h> -#include <string> - -#include "mongo/db/storage/mobile/mobile_session.h" -#include "mongo/platform/atomic_word.h" - -#include <boost/container/small_vector.hpp> - -namespace mongo { - -/** - * SqliteStatement is a wrapper around the sqlite3_stmt object. All calls to the SQLite API that - * involve a sqlite_stmt object are made in this class. - */ -class SqliteStatement final { -public: - /** - * Creates and prepares a SQLite statement. - */ - template <class... Args> - SqliteStatement(const MobileSession& session, Args&&... args); - - /** - * Finalizes the prepared statement. - */ - ~SqliteStatement(); - - /** - * The various bind methods bind a value to the query parameter specified by paramIndex. - * - * @param paramIndex - zero-based index of a query parameter. - */ - void bindInt(int paramIndex, int64_t intValue); - - void bindBlob(int paramIndex, const void* data, int len); - - void bindText(int paramIndex, const char* data, int len); - - void clearBindings(); - - /** - * Wraps sqlite3_step and returns the resulting status. - * - * @param desiredStatus - the desired return status of sqlite3_step. When desiredStatus is - * non-negative, checkStatus compares desiredStatus with the returned status from sqlite3_step. - * By default, checkStatus is ignored. - */ - int step(int desiredStatus = -1); - - /** - * The getCol methods wrap sqlite3_column methods and return the correctly typed values - * stored in a retrieved query row[colIndex]. - * - * @param colIndex - zero-based index of a column retrieved from a query row. - */ - int64_t getColInt(int colIndex); - - const void* getColBlob(int colIndex); - - /** - * Returns the number of bytes in a corresponding blob or string. - */ - int64_t getColBytes(int colIndex); - - /** - * Wraps sqlite3_column_text method and returns the text from the retrieved query row[colIndex]. - * - * @param colIndex - zero-based index of a column retrieved from a query row. - */ - const char* getColText(int colIndex); - - /** - * Resets the statement to the first of the query result rows. - */ - void reset(); - - /** - * Sets the last status on the prepared statement. - */ - void setExceptionStatus(int status) { - _exceptionStatus = status; - } - - /** - * A one step query execution that wraps sqlite3_prepare_v2(), sqlite3_step(), and - * sqlite3_finalize(). - * None of the rows retrieved, if any, are saved before the query is finalized. Thus, this - * method should not be used for read operations. - */ - template <class... Args> - static void execQuery(sqlite3* session, const char* first, Args&&... args) { - // If we just have a single char*, we're good to go, no need to build a new string. - constexpr std::size_t num = sizeof...(args); - if (!num) { - _execQuery(session, first); - } else { - _execQueryBuilder(session, first, std::forward<Args>(args)...); - } - } - template <class... Args> - static void execQuery(sqlite3* session, const std::string& first, Args&&... args) { - execQuery(session, first.c_str(), std::forward<Args>(args)...); - } - template <class... Args> - static void execQuery(sqlite3* session, StringData first, Args&&... args) { - // StringData may not be null terminated, so build a new string - _execQueryBuilder(session, first, std::forward<Args>(args)...); - } - template <class... Args> - static void execQuery(MobileSession* session, Args&&... args) { - execQuery(session->getSession(), std::forward<Args>(args)...); - } - - /** - * Finalizes a prepared statement. - */ - void finalize(); - - /** - * Prepare a statement with the given mobile session. - */ - void prepare(const MobileSession& session); - - uint64_t _id; - -private: - static void _execQuery(sqlite3* session, const char* query); - template <class... Args> - static void _execQueryBuilder(sqlite3* session, Args&&... args); - - static AtomicWord<long long> _nextID; - sqlite3_stmt* _stmt; - - // If the most recent call to sqlite3_step on this statement returned an error, the error is - // returned again when the statement is finalized. This is used to verify that the last error - // code returned matches the finalize error code, if there is any. - int _exceptionStatus = SQLITE_OK; - - // Static memory that fits short SQL statements to avoid a temporary memory allocation - static constexpr size_t kMaxFixedSize = 96; - using SqlQuery_t = boost::container::small_vector<char, kMaxFixedSize>; - SqlQuery_t _sqlQuery; - - template <std::size_t N> - static void queryAppend(SqlQuery_t& dest, char const (&str)[N], std::true_type) { - dest.insert(dest.end(), str, str + N - 1); - } - static void queryAppend(SqlQuery_t& dest, StringData sd, std::false_type) { - dest.insert(dest.end(), sd.begin(), sd.end()); - } -}; - -namespace detail { -// Most of the strings we build statements with are static strings so we can calculate their length -// during compile time. -// Arrays decay to pointer in overload resolution, force that to not happen by providing true_type -// as second argument if array -template <std::size_t N> -constexpr std::size_t stringLength(char const (&)[N], std::true_type) { - // Omit the null terminator, will added back when we call reserve later - return N - 1; -} -inline std::size_t stringLength(StringData sd, std::false_type) { - return sd.size(); -} - -} // namespace detail - -template <class... Args> -SqliteStatement::SqliteStatement(const MobileSession& session, Args&&... args) { - // Increment the global instance count and assign this instance an id. - _id = _nextID.addAndFetch(1); - - // Reserve the size we need once to avoid any additional allocations - _sqlQuery.reserve((detail::stringLength(std::forward<Args>(args), - std::is_array<std::remove_reference_t<Args>>()) + - ...) + - 1); - - // Copy all substrings into buffer for SQL statement - (queryAppend( - _sqlQuery, std::forward<Args>(args), std::is_array<std::remove_reference_t<Args>>()), - ...); - _sqlQuery.push_back('\0'); - - prepare(session); -} - -template <class... Args> -void SqliteStatement::_execQueryBuilder(sqlite3* session, Args&&... args) { - SqlQuery_t sqlQuery; - - sqlQuery.reserve((detail::stringLength(std::forward<Args>(args), - std::is_array<std::remove_reference_t<Args>>()) + - ...) + - 1); - (queryAppend( - sqlQuery, std::forward<Args>(args), std::is_array<std::remove_reference_t<Args>>()), - ...); - sqlQuery.push_back('\0'); - _execQuery(session, sqlQuery.data()); -} - -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.cpp b/src/mongo/db/storage/mobile/mobile_util.cpp deleted file mode 100644 index 3159dd169f8..00000000000 --- a/src/mongo/db/storage/mobile/mobile_util.cpp +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage - -#include "mongo/util/log.h" -#include "mongo/util/str.h" - -#include <sqlite3.h> - -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/mobile/mobile_recovery_unit.h" -#include "mongo/db/storage/mobile/mobile_sqlite_statement.h" -#include "mongo/db/storage/mobile/mobile_util.h" - -namespace mongo { -namespace embedded { - -using std::string; - -Status sqliteRCToStatus(int retCode, const char* prefix) { - str::stream s; - if (prefix) - s << prefix << " "; - s << retCode << ": " << sqlite3_errstr(retCode); - - switch (retCode) { - case SQLITE_OK: - return Status::OK(); - case SQLITE_INTERNAL: - return Status(ErrorCodes::InternalError, s); - case SQLITE_PERM: - return Status(ErrorCodes::Unauthorized, s); - case SQLITE_BUSY: - return Status(ErrorCodes::LockBusy, s); - case SQLITE_LOCKED: - return Status(ErrorCodes::LockBusy, s); - case SQLITE_NOMEM: - return Status(ErrorCodes::ExceededMemoryLimit, s); - case SQLITE_READONLY: - return Status(ErrorCodes::Unauthorized, s); - case SQLITE_INTERRUPT: - return Status(ErrorCodes::Interrupted, s); - case SQLITE_CANTOPEN: - return Status(ErrorCodes::FileOpenFailed, s); - case SQLITE_PROTOCOL: - return Status(ErrorCodes::ProtocolError, s); - case SQLITE_MISMATCH: - return Status(ErrorCodes::TypeMismatch, s); - case SQLITE_MISUSE: - return Status(ErrorCodes::BadValue, s); - case SQLITE_NOLFS: - return Status(ErrorCodes::CommandNotSupported, s); - case SQLITE_AUTH: - return Status(ErrorCodes::AuthenticationFailed, s); - case SQLITE_FORMAT: - return Status(ErrorCodes::UnsupportedFormat, s); - case SQLITE_RANGE: - return Status(ErrorCodes::BadValue, s); - case SQLITE_NOTADB: - return Status(ErrorCodes::FileOpenFailed, s); - default: - return Status(ErrorCodes::UnknownError, s); - } -} - -const char* sqliteStatusToStr(int retStatus) { - const char* msg = nullptr; - - switch (retStatus) { - case SQLITE_OK: - msg = "SQLITE_OK"; - break; - case SQLITE_ERROR: - msg = "SQLITE_ERROR"; - break; - case SQLITE_BUSY: - msg = "SQLITE_BUSY"; - break; - case SQLITE_LOCKED: - msg = "SQLITE_LOCKED"; - break; - case SQLITE_NOTFOUND: - msg = "SQLITE_NOTFOUND"; - break; - case SQLITE_FULL: - msg = "SQLITE_FULL"; - break; - case SQLITE_MISUSE: - msg = "SQLITE_MISUSE"; - break; - case SQLITE_ROW: - msg = "SQLITE_ROW"; - break; - case SQLITE_DONE: - msg = "SQLITE_DONE"; - break; - default: - msg = "Status not converted"; - break; - } - return msg; -} - -void checkStatus(int retStatus, int desiredStatus, const char* fnName, const char* errMsg) { - if (retStatus != desiredStatus) { - std::stringstream s; - s << fnName << " failed with return status " << sqlite3_errstr(retStatus); - - if (errMsg) { - s << "------ Error Message: " << errMsg; - } - - severe() << s.str(); - fassertFailed(37000); - } -} - -/** - * Helper to add and log errors for validate. - */ -void validateLogAndAppendError(ValidateResults* results, const std::string& errMsg) { - error() << "validate found error: " << errMsg; - results->errors.push_back(errMsg); - results->valid = false; -} - -void doValidate(OperationContext* opCtx, ValidateResults* results) { - MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - try { - SqliteStatement validateStmt(*session, "PRAGMA integrity_check;"); - - int status; - // By default, the integrity check returns the first 100 errors found. - while ((status = validateStmt.step()) == SQLITE_ROW) { - std::string errMsg(validateStmt.getColText(0)); - - if (errMsg == "ok") { - // If the first message returned is "ok", the integrity check passed without - // finding any corruption. - continue; - } - - validateLogAndAppendError(results, errMsg); - } - - if (status == SQLITE_CORRUPT) { - uasserted(ErrorCodes::UnknownError, sqlite3_errstr(status)); - } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); - - } catch (const DBException& e) { - // SQLite statement may fail to prepare or execute correctly if the file is corrupted. - std::string errMsg = "database file is corrupt - " + e.toString(); - validateLogAndAppendError(results, errMsg); - } -} - -void configureSession(sqlite3* session, const MobileOptions& options) { - auto executePragma = [session](auto pragma, auto value) { - SqliteStatement::execQuery(session, "PRAGMA ", pragma, " = ", value, ";"); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE session configuration: " << pragma << " = " << value; - }; - // We don't manually use VACUUM so set incremental(2) mode to reclaim space - // This need to be set the first thing we do, before any internal tables are created. - executePragma("auto_vacuum"_sd, "incremental"_sd); - - // Set SQLite in Write-Ahead Logging mode. https://sqlite.org/wal.html - executePragma("journal_mode"_sd, "WAL"_sd); - - // synchronous = NORMAL(1) is recommended with WAL, but we allow it to be overriden - executePragma("synchronous"_sd, std::to_string(options.durabilityLevel)); - - // Set full fsync on OSX (only supported there) to ensure durability - executePragma("fullfsync"_sd, "1"_sd); - - // We just use SQLite as key-value store, so disable foreign keys - executePragma("foreign_keys"_sd, "0"_sd); - - // Set some additional internal sizes for this session - // Cache size described as KB should be set as negative number - // https://sqlite.org/pragma.html#pragma_cache_size - executePragma("cache_size"_sd, std::to_string(-static_cast<int32_t>(options.cacheSizeKB))); - executePragma("mmap_size"_sd, std::to_string(options.mmapSizeKB * 1024)); - executePragma("journal_size_limit"_sd, std::to_string(options.journalSizeLimitKB * 1024)); -} - -} // namespace embedded -} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.h b/src/mongo/db/storage/mobile/mobile_util.h deleted file mode 100644 index 9cdb8bf0e15..00000000000 --- a/src/mongo/db/storage/mobile/mobile_util.h +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#pragma once - -#include "mongo/base/status.h" -#include "mongo/db/operation_context.h" -#include "mongo/db/storage/mobile/mobile_options.h" -#include "mongo/db/storage/record_store.h" - -#define MOBILE_LOG_LEVEL_LOW 2 -#define MOBILE_LOG_LEVEL_HIGH 5 -#define MOBILE_TRACE_LEVEL MOBILE_LOG_LEVEL_HIGH - -namespace mongo { -namespace embedded { - -/** - * Converts SQLite return codes to MongoDB statuses. - */ -Status sqliteRCToStatus(int retCode, const char* prefix = nullptr); - -/** - * Converts SQLite return codes to string equivalents. - */ -const char* sqliteStatusToStr(int retStatus); - -/** - * Checks if retStatus == desiredStatus; else calls fassert. - */ -void checkStatus(int retStatus, - int desiredStatus, - const char* fnName, - const char* errMsg = nullptr); - -/** - * Validate helper function to log an error and append the error to the results. - */ -void validateLogAndAppendError(ValidateResults* results, const std::string& errMsg); - -/** - * Checks if the database file is corrupt. - */ -void doValidate(OperationContext* opCtx, ValidateResults* results); - -/** - * Sets the SQLite Pragmas that we want (https://www.sqlite.org/pragma.html) - * These should generally improve behavior, performance, and resource usage - */ -void configureSession(sqlite3* session, const MobileOptions& options); - -} // namespace embedded -} // namespace mongo diff --git a/src/mongo/dbtests/indexupdatetests.cpp b/src/mongo/dbtests/indexupdatetests.cpp index 21e45f0adbd..f474310f530 100644 --- a/src/mongo/dbtests/indexupdatetests.cpp +++ b/src/mongo/dbtests/indexupdatetests.cpp @@ -658,17 +658,9 @@ class IndexUpdateTests : public OldStyleSuiteSpecification { public: IndexUpdateTests() : OldStyleSuiteSpecification("indexupdate") {} - // Must be evaluated at test run() time, not static-init time. - static bool shouldSkip() { - return mongo::storageGlobalParams.engine == "mobile"; - } - template <typename T> void addIf() { - addNameCallback(nameForTestClass<T>(), [] { - if (!shouldSkip()) - T().run(); - }); + addNameCallback(nameForTestClass<T>(), [] { T().run(); }); } void setupTests() { diff --git a/src/mongo/dbtests/repltests.cpp b/src/mongo/dbtests/repltests.cpp index 9b38d9d863d..f6c1a6e2445 100644 --- a/src/mongo/dbtests/repltests.cpp +++ b/src/mongo/dbtests/repltests.cpp @@ -106,11 +106,6 @@ public: : _client(&_opCtx), _defaultReplSettings( ReplicationCoordinator::get(getGlobalServiceContext())->getSettings()) { - // Replication is not supported by mobile SE. - if (mongo::storageGlobalParams.engine == "mobile") { - return; - } - transport::TransportLayerASIO::Options opts; opts.mode = transport::TransportLayerASIO::Options::kEgress; auto sc = getGlobalServiceContext(); @@ -155,10 +150,6 @@ public: deleteAll(cllNS()); } ~Base() { - // Replication is not supported by mobile SE. - if (mongo::storageGlobalParams.engine == "mobile") { - return; - } try { deleteAll(ns()); deleteAll(cllNS()); @@ -336,10 +327,6 @@ protected: class LogBasic : public Base { public: void run() { - // Replication is not supported by mobile SE. - if (mongo::storageGlobalParams.engine == "mobile") { - return; - } ASSERT_EQUALS(0, opCount()); _client.insert(ns(), fromjson("{\"a\":\"b\"}")); ASSERT_EQUALS(1, opCount()); @@ -352,10 +339,6 @@ class Base : public ReplTests::Base { public: virtual ~Base() {} void run() { - // Replication is not supported by mobile SE. - if (mongo::storageGlobalParams.engine == "mobile") { - return; - } reset(); doIt(); int nOps = opCount(); @@ -1308,11 +1291,6 @@ public: class DeleteOpIsIdBased : public Base { public: void run() { - // Replication is not supported by mobile SE. - if (mongo::storageGlobalParams.engine == "mobile") { - return; - } - // These inserts don't write oplog entries. insert(BSON("_id" << 0 << "a" << 10)); insert(BSON("_id" << 1 << "a" << 11)); diff --git a/src/mongo/embedded/index_builds_coordinator_embedded.h b/src/mongo/embedded/index_builds_coordinator_embedded.h index 813458354e6..9587fc06ab2 100644 --- a/src/mongo/embedded/index_builds_coordinator_embedded.h +++ b/src/mongo/embedded/index_builds_coordinator_embedded.h @@ -37,7 +37,7 @@ class OperationContext; class ServiceContext; /** - * This implementation of the IndexBuildsCoordinator is for embedded (mobile) server nodes. Nothing + * This implementation of the IndexBuildsCoordinator is for embedded server nodes. Nothing * is run asynchronously and no network calls are made. Index builds are run without awaiting cross * replica set communications. * |