diff options
author | Henrik Edin <henrik.edin@mongodb.com> | 2019-03-26 10:27:48 -0400 |
---|---|---|
committer | Henrik Edin <henrik.edin@mongodb.com> | 2019-04-10 13:19:54 -0400 |
commit | 3873c080388d5148afde90d7e02b2c4f5a297867 (patch) | |
tree | bb414dce2e816abba56045a8f9d60987fdb01a40 | |
parent | 73536c31314daef6c68217aed5f8d6ddd432d15b (diff) | |
download | mongo-3873c080388d5148afde90d7e02b2c4f5a297867.tar.gz |
SERVER-32709 Improve performance with the mobile storage engine.
SQLite is configured with synchronous=NORMAL for all sessions.
Reduced temporary memory allocations when constructing SQL statements.
Implemented waitUntilDurable when this storage engine is used in mongod only.
(cherry picked from commit 2ee7dcb2bcb4372a49b8584c43bb65caea6212c8)
18 files changed, 345 insertions, 203 deletions
diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 38b352d3948..1dd989d8014 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -11854,7 +11854,7 @@ buildvariants: test_flags: >- --storageEngine=mobile --excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact - --excludeWithAnyTags=SERVER-32709,SERVER-32869 + --excludeWithAnyTags=SERVER-32869,SERVER-40561 compile_flags: >- -j$(grep -c ^processor /proc/cpuinfo) --mobile-se=on @@ -11932,7 +11932,7 @@ buildvariants: test_flags: >- --storageEngine=mobile --excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact - --excludeWithAnyTags=SERVER-32709,SERVER-32869 + --excludeWithAnyTags=SERVER-32869,SERVER-40561 compile_flags: >- -j$(grep -c ^processor /proc/cpuinfo) --mobile-se=on @@ -12008,7 +12008,7 @@ buildvariants: test_flags: >- --storageEngine=mobile --excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact - --excludeWithAnyTags=SERVER-32709,SERVER-32869 + --excludeWithAnyTags=SERVER-32869,SERVER-40561 compile_env: DEVELOPER_DIR=/Applications/Xcode8.3.app compile_flags: >- -j$(sysctl -n hw.logicalcpu) diff --git a/jstests/concurrency/fsm_workloads/reindex.js b/jstests/concurrency/fsm_workloads/reindex.js index 7ca348c005b..0ec81ef1ff3 100644 --- a/jstests/concurrency/fsm_workloads/reindex.js +++ b/jstests/concurrency/fsm_workloads/reindex.js @@ -6,7 +6,7 @@ * Bulk inserts 1000 documents and builds indexes. Then alternates between reindexing and querying * against the collection. Operates on a separate collection for each thread. * - * @tags: [SERVER-32709] + * @tags: [SERVER-40561] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/reindex_background.js b/jstests/concurrency/fsm_workloads/reindex_background.js index 1e18e60922c..d3958fc8de1 100644 --- a/jstests/concurrency/fsm_workloads/reindex_background.js +++ b/jstests/concurrency/fsm_workloads/reindex_background.js @@ -8,7 +8,7 @@ * that because indexes are initially built in the background, reindexing is also done in the * background. * - * @tags: [SERVER-32709] + * @tags: [SERVER-40561] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js index 5b47bc43de2..0c6758c0262 100644 --- a/jstests/noPassthrough/indexbg1.js +++ b/jstests/noPassthrough/indexbg1.js @@ -1,5 +1,5 @@ // Test background index creation -// @tags: [SERVER-32709] +// @tags: [SERVER-40561] (function() { "use strict"; diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js index 3e570c9dd94..70dddac5e55 100644 --- a/jstests/noPassthrough/indexbg2.js +++ b/jstests/noPassthrough/indexbg2.js @@ -1,5 +1,5 @@ // Test background index creation w/ constraints -// @tags: [SERVER-32709, requires_document_locking] +// @tags: [SERVER-40561, requires_document_locking] (function() { "use strict"; diff --git a/jstests/noPassthroughWithMongod/geo_polygon.js b/jstests/noPassthroughWithMongod/geo_polygon.js index 4ce1f8d2ad7..7c23442d4de 100644 --- a/jstests/noPassthroughWithMongod/geo_polygon.js +++ b/jstests/noPassthroughWithMongod/geo_polygon.js @@ -1,5 +1,5 @@ // -// @tags: [SERVER-32709] +// @tags: [SERVER-40561] // t = db.geo_polygon4; diff --git a/src/mongo/db/storage/mobile/mobile_global_options.h b/src/mongo/db/storage/mobile/mobile_global_options.h index 16277c6fcc1..44a495f7d70 100644 --- a/src/mongo/db/storage/mobile/mobile_global_options.h +++ b/src/mongo/db/storage/mobile/mobile_global_options.h @@ -44,7 +44,7 @@ public: // This is used by the Mobile SE and allows users to set the value // passed to SQLite's PRAGMA synchronous command - std::int32_t mobileDurabilityLevel; + std::uint32_t mobileDurabilityLevel; }; extern MobileGlobalOptions mobileGlobalOptions; diff --git a/src/mongo/db/storage/mobile/mobile_index.cpp b/src/mongo/db/storage/mobile/mobile_index.cpp index 4860e524d24..cdfef05abd5 100644 --- a/src/mongo/db/storage/mobile/mobile_index.cpp +++ b/src/mongo/db/storage/mobile/mobile_index.cpp @@ -105,8 +105,8 @@ Status MobileIndex::doInsert(OperationContext* opCtx, session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); } - std::string insertQuery = "INSERT INTO \"" + _ident + "\" (key, value) VALUES (?, ?);"; - SqliteStatement insertStmt(*session, insertQuery); + SqliteStatement insertStmt( + *session, "INSERT INTO \"", _ident, "\" (key, value) VALUES (?, ?);"); insertStmt.bindBlob(0, key.getBuffer(), key.getSize()); insertStmt.bindBlob(1, value.getBuffer(), value.getSize()); @@ -126,7 +126,7 @@ Status MobileIndex::doInsert(OperationContext* opCtx, return Status::OK(); } } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); return Status::OK(); } @@ -144,13 +144,8 @@ void MobileIndex::unindex(OperationContext* opCtx, void MobileIndex::_doDelete(OperationContext* opCtx, const KeyString& key, KeyString* value) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - str::stream deleteQuery; - deleteQuery << "DELETE FROM \"" << _ident << "\" WHERE key = ?"; - if (value) { - deleteQuery << " AND value = ?"; - } - deleteQuery << ";"; - SqliteStatement deleteStmt(*session, deleteQuery); + SqliteStatement deleteStmt( + *session, "DELETE FROM \"", _ident, "\" WHERE key = ?", value ? " AND value = ?" : "", ";"); deleteStmt.bindBlob(0, key.getBuffer(), key.getSize()); if (value) { @@ -166,7 +161,7 @@ void MobileIndex::fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const { if (fullResults) { - doValidate(opCtx, fullResults); + embedded::doValidate(opCtx, fullResults); if (!fullResults->valid) { return; } @@ -188,10 +183,11 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const { // Sum the number of bytes in each column. // SQLite aggregate functions return null if the column is empty or has only nulls, so return 0 // bytes if there is no data in the column. - str::stream sizeQuery; - sizeQuery << "SELECT IFNULL(SUM(LENGTH(key)), 0) + " - << "IFNULL(SUM(LENGTH(value)), 0) FROM \"" << _ident + "\";"; - SqliteStatement sizeStmt(*session, sizeQuery); + SqliteStatement sizeStmt(*session, + "SELECT IFNULL(SUM(LENGTH(key)), 0) + ", + "IFNULL(SUM(LENGTH(value)), 0) FROM \"", + _ident, + "\";"); sizeStmt.step(SQLITE_ROW); @@ -201,8 +197,8 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const { long long MobileIndex::numEntries(OperationContext* opCtx) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string countQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";"; - SqliteStatement countStmt(*session, countQuery); + + SqliteStatement countStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); countStmt.step(SQLITE_ROW); long long numRecs = countStmt.getColInt(0); @@ -211,14 +207,14 @@ long long MobileIndex::numEntries(OperationContext* opCtx) const { bool MobileIndex::isEmpty(OperationContext* opCtx) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string emptyCheckQuery = "SELECT * FROM \"" + _ident + "\" LIMIT 1;"; - SqliteStatement emptyCheckStmt(*session, emptyCheckQuery); + + SqliteStatement emptyCheckStmt(*session, "SELECT * FROM \"", _ident, "\" LIMIT 1;"); int status = emptyCheckStmt.step(); if (status == SQLITE_DONE) { return true; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); return false; } @@ -229,9 +225,9 @@ Status MobileIndex::initAsEmpty(OperationContext* opCtx) { Status MobileIndex::create(OperationContext* opCtx, const std::string& ident) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - std::string createTableQuery = - "CREATE TABLE \"" + ident + "\"(key BLOB PRIMARY KEY, value BLOB);"; - SqliteStatement createTableStmt(*session, createTableQuery.c_str()); + + SqliteStatement createTableStmt( + *session, "CREATE TABLE \"", ident, "\"(key BLOB PRIMARY KEY, value BLOB);"); createTableStmt.step(SQLITE_DONE); return Status::OK(); @@ -250,8 +246,7 @@ Status MobileIndex::dupKeyCheck(OperationContext* opCtx, bool MobileIndex::_isDup(OperationContext* opCtx, const BSONObj& key, RecordId recId) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string dupCheckQuery = "SELECT value FROM \"" + _ident + "\" WHERE key = ?;"; - SqliteStatement dupCheckStmt(*session, dupCheckQuery); + SqliteStatement dupCheckStmt(*session, "SELECT value FROM \"", _ident, "\" WHERE key = ?;"); KeyString keyStr(_keyStringVersion, key, _ordering); @@ -273,7 +268,7 @@ bool MobileIndex::_isDup(OperationContext* opCtx, const BSONObj& key, RecordId r return false; } } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); return isEntryFound; } @@ -401,11 +396,15 @@ public: _savedTypeBits(index.getKeyStringVersion()), _startPosition(index.getKeyStringVersion()) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - str::stream cursorQuery; - cursorQuery << "SELECT key, value FROM \"" << _index.getIdent() << "\" WHERE key "; - cursorQuery << (_isForward ? ">=" : "<=") << " ? ORDER BY key "; - cursorQuery << (_isForward ? "ASC" : "DESC") << ";"; - _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery); + + _stmt = stdx::make_unique<SqliteStatement>(*session, + "SELECT key, value FROM \"", + _index.getIdent(), + "\" WHERE key ", + (_isForward ? ">=" : "<="), + " ? ORDER BY key ", + (_isForward ? "ASC" : "DESC"), + ";"); } virtual ~CursorBase() {} @@ -515,7 +514,7 @@ protected: _isEOF = true; return; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); _isEOF = false; diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp index ef9637ee21f..d68e48d81c3 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp @@ -56,126 +56,80 @@ namespace mongo { class MobileSession; class SqliteStatement; -MobileKVEngine::MobileKVEngine(const std::string& path, std::int32_t durabilityLevel) { +MobileKVEngine::MobileKVEngine(const std::string& path, std::uint32_t durabilityLevel) { _initDBPath(path); // Initialize the database to be in WAL mode. sqlite3* initSession; int status = sqlite3_open(_path.c_str(), &initSession); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); // Guarantees that sqlite3_close() will be called when the function returns. ON_BLOCK_EXIT([&initSession] { sqlite3_close(initSession); }); - // Ensure SQLite is operating in the WAL mode. + embedded::configureSession(initSession); + + // Check and ensure that WAL mode is working as expected + // This is not something that we want to be configurable { sqlite3_stmt* stmt; - status = sqlite3_prepare_v2(initSession, "PRAGMA journal_mode=WAL;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + status = sqlite3_prepare_v2(initSession, "PRAGMA journal_mode;", -1, &stmt, NULL); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current mode in SQLite, ensure it is "wal" mode. const void* colText = sqlite3_column_text(stmt, 0); const char* mode = reinterpret_cast<const char*>(colText); fassert(37001, !strcmp(mode, "wal")); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database opened in WAL mode"; } - // Let's set and enforce the synchronous mode - // We allow the user to specify a value between 0 and 3 - // using the mobileDurabilityLevel option + // Check and ensure that synchronous mode is working as expected { - std::string synchronousPragma = - "PRAGMA synchronous = " + std::to_string(durabilityLevel) + ";"; - - char* errMsg = NULL; - status = sqlite3_exec(initSession, synchronousPragma.c_str(), NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - sqlite3_stmt* stmt; status = sqlite3_prepare_v2(initSession, "PRAGMA synchronous;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current "synchronous" setting - int sync_val = sqlite3_column_int(stmt, 0); + std::uint32_t sync_val = sqlite3_column_int(stmt, 0); fassert(50869, sync_val == durabilityLevel); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database has synchronous " - << "mode set to " << durabilityLevel; + << "set to: " << durabilityLevel; } - // Set and ensure SQLite is operating with F_FULLFSYNC on darwin kernels + // Check and ensure that we were able to set the F_FULLFSYNC fcntl on darwin kernels + // This prevents data corruption as fsync doesn't work as expected + // This is not something that we want to be configurable { - char* errMsg = NULL; - status = sqlite3_exec(initSession, "PRAGMA fullfsync = 1;", NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - // When the error message is not NULL, it is allocated through sqlite3_malloc and must be - // freed before exiting the method. If the error message is NULL, sqlite3_free is a no-op. - sqlite3_free(errMsg); - - // Confirm that the setting holds sqlite3_stmt* stmt; status = sqlite3_prepare_v2(initSession, "PRAGMA fullfsync;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current fullsync setting, ensure it is enabled. int fullfsync_val = sqlite3_column_int(stmt, 0); fassert(50868, fullfsync_val == 1); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database is set to fsync " << "with F_FULLFSYNC if the platform supports it (currently" << " only darwin kernels). Value: " << fullfsync_val; } - // Let's set additional non-critical settings that should improve performance and resource usage - { - // Allow for periodic calls to purge deleted records and prune db size on disk - // Still requires manual vacuum calls using `PRAGMA incremental_vacuum(N);` - char* errMsg = NULL; - status = - sqlite3_exec(initSession, "PRAGMA auto_vacuum = incremental;", NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - - // Set the cache size to 10MiB - // value is passed as "-<#ofKiB>" - errMsg = NULL; - status = sqlite3_exec(initSession, "PRAGMA cache_size = -10240;", NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - - // Enable memory mapping for the data and set the size at 50MiB - errMsg = NULL; - status = sqlite3_exec(initSession, "PRAGMA mmap_size = 52428800;", NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - - // Cap the WAL size at 5MiB so that it's regularly pruned - errMsg = NULL; - status = - sqlite3_exec(initSession, "PRAGMA journal_size_limit = 5242880;", NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Completed all SQLite database configuration"; - } - _sessionPool.reset(new MobileSessionPool(_path)); } @@ -267,7 +221,7 @@ Status MobileKVEngine::dropIdent(OperationContext* opCtx, StringData ident) { std::string dropQuery = "DROP TABLE IF EXISTS \"" + ident + "\";"; try { - SqliteStatement::execQuery(session, dropQuery.c_str()); + SqliteStatement::execQuery(session, dropQuery); } catch (const WriteConflictException&) { // It is possible that this drop fails because of transaction running in parallel. // We pretend that it succeeded, queue it for now and keep retrying later. @@ -287,8 +241,7 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); // Get key-value column names. - std::string colNameQuery = "PRAGMA table_info(\"" + ident + "\")"; - SqliteStatement colNameStmt(*session, colNameQuery); + SqliteStatement colNameStmt(*session, "PRAGMA table_info(\"", ident, "\")"); colNameStmt.step(SQLITE_ROW); std::string keyColName(static_cast<const char*>(colNameStmt.getColText(1))); @@ -297,10 +250,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) colNameStmt.step(SQLITE_DONE); // Get total data size of key-value columns. - str::stream dataSizeQuery; - dataSizeQuery << "SELECT IFNULL(SUM(LENGTH(" << keyColName << ")), 0) + " - << "IFNULL(SUM(LENGTH(" << valueColName << ")), 0) FROM \"" << ident + "\";"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + SqliteStatement dataSizeStmt(*session, + "SELECT IFNULL(SUM(LENGTH(", + keyColName, + ")), 0) + ", + "IFNULL(SUM(LENGTH(", + valueColName, + ")), 0) FROM \"", + ident, + "\";"); dataSizeStmt.step(SQLITE_ROW); return dataSizeStmt.getColInt(0); @@ -309,15 +267,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string findTableQuery = "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;"; - SqliteStatement findTableStmt(*session, findTableQuery); + SqliteStatement findTableStmt(*session, + "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;"); findTableStmt.bindText(0, ident.rawData(), ident.size()); int status = findTableStmt.step(); if (status == SQLITE_DONE) { return false; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); return true; } @@ -325,15 +283,15 @@ bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const { std::vector<std::string> MobileKVEngine::getAllIdents(OperationContext* opCtx) const { std::vector<std::string> idents; MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string getTablesQuery = "SELECT name FROM sqlite_master WHERE type='table';"; - SqliteStatement getTablesStmt(*session, getTablesQuery); + + SqliteStatement getTablesStmt(*session, "SELECT name FROM sqlite_master WHERE type='table';"); int status; while ((status = getTablesStmt.step()) == SQLITE_ROW) { std::string tableName(reinterpret_cast<const char*>(getTablesStmt.getColText(0))); idents.push_back(tableName); } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); return idents; } diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h index 17b34d8c308..d8f4d2b3013 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.h +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h @@ -44,7 +44,7 @@ class JournalListener; class MobileKVEngine : public KVEngine { public: - MobileKVEngine(const std::string& path, std::int32_t durabilityLevel); + MobileKVEngine(const std::string& path, std::uint32_t durabilityLevel); RecoveryUnit* newRecoveryUnit() override; diff --git a/src/mongo/db/storage/mobile/mobile_record_store.cpp b/src/mongo/db/storage/mobile/mobile_record_store.cpp index b5ce45ac107..fa20782ef45 100644 --- a/src/mongo/db/storage/mobile/mobile_record_store.cpp +++ b/src/mongo/db/storage/mobile/mobile_record_store.cpp @@ -63,13 +63,17 @@ public: bool forward) : _opCtx(opCtx), _forward(forward) { - str::stream cursorQuery; - cursorQuery << "SELECT rec_id, data from \"" << ident << "\" " - << "WHERE rec_id " << (forward ? '>' : '<') << " ? " - << "ORDER BY rec_id " << (forward ? "ASC" : "DESC") << ';'; - MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx); - _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery); + _stmt = stdx::make_unique<SqliteStatement>(*session, + "SELECT rec_id, data from \"", + ident, + "\" ", + "WHERE rec_id ", + (forward ? ">" : "<"), + " ? ", + "ORDER BY rec_id ", + (forward ? "ASC" : "DESC"), + ";"); _startIdNum = (forward ? RecordId::min().repr() : RecordId::max().repr()); _savedId = RecordId(_startIdNum); @@ -91,7 +95,7 @@ public: } // Checks no error was thrown and that step retrieved a row. - checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next"); + embedded::checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next"); long long recId = _stmt->getColInt(0); const void* data = _stmt->getColBlob(1); @@ -192,8 +196,7 @@ MobileRecordStore::MobileRecordStore(OperationContext* opCtx, // Determines the nextId to be used for a new record. MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string maxRecIdQuery = "SELECT IFNULL(MAX(rec_id), 0) FROM \"" + _ident + "\";"; - SqliteStatement maxRecIdStmt(*session, maxRecIdQuery); + SqliteStatement maxRecIdStmt(*session, "SELECT IFNULL(MAX(rec_id), 0) FROM \"", _ident, "\";"); maxRecIdStmt.step(SQLITE_ROW); @@ -215,8 +218,8 @@ void MobileRecordStore::_initDataSizeIfNeeded_inlock(OperationContext* opCtx) co } MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string dataSizeQuery = "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"" + _ident + "\";"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"", _ident, "\";"); dataSizeStmt.step(SQLITE_ROW); int64_t dataSize = dataSizeStmt.getColInt(0); @@ -237,8 +240,7 @@ void MobileRecordStore::_initNumRecsIfNeeded_inlock(OperationContext* opCtx) con } MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string numRecordsQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";"; - SqliteStatement numRecordsStmt(*session, numRecordsQuery); + SqliteStatement numRecordsStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); numRecordsStmt.step(SQLITE_ROW); @@ -265,8 +267,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, const RecordId& recId, RecordData* rd) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string sqlQuery = "SELECT data FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement stmt(*session, sqlQuery); + SqliteStatement stmt(*session, "SELECT data FROM \"", _ident, "\" WHERE rec_id = ?;"); stmt.bindInt(0, recId.repr()); @@ -274,7 +275,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, if (status == SQLITE_DONE) { return false; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); const void* recData = stmt.getColBlob(0); int nBytes = stmt.getColBytes(0); @@ -284,9 +285,9 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& recId) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - std::string dataSizeQuery = - "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); dataSizeStmt.bindInt(0, recId.repr()); dataSizeStmt.step(SQLITE_ROW); @@ -294,8 +295,7 @@ void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& re _changeNumRecs(opCtx, -1); _changeDataSize(opCtx, -dataSizeBefore); - std::string deleteQuery = "DELETE FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement deleteStmt(*session, deleteQuery); + SqliteStatement deleteStmt(*session, "DELETE FROM \"", _ident, "\" WHERE rec_id = ?;"); deleteStmt.bindInt(0, recId.repr()); deleteStmt.step(SQLITE_DONE); } @@ -308,9 +308,8 @@ StatusWith<RecordId> MobileRecordStore::insertRecord( _changeNumRecs(opCtx, 1); _changeDataSize(opCtx, len); - std::string insertQuery = - "INSERT OR REPLACE INTO \"" + _ident + "\"(rec_id, data) VALUES(?, ?);"; - SqliteStatement insertStmt(*session, insertQuery); + SqliteStatement insertStmt( + *session, "INSERT OR REPLACE INTO \"", _ident, "\"(rec_id, data) VALUES(?, ?);"); RecordId recId = _nextId(); insertStmt.bindInt(0, recId.repr()); insertStmt.bindBlob(1, data, len); @@ -350,9 +349,9 @@ Status MobileRecordStore::updateRecord(OperationContext* opCtx, bool enforceQuota, UpdateNotifier* notifier) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - std::string dataSizeQuery = - "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); dataSizeStmt.bindInt(0, recId.repr()); dataSizeStmt.step(SQLITE_ROW); @@ -363,8 +362,7 @@ Status MobileRecordStore::updateRecord(OperationContext* opCtx, fassert(37054, notifier->recordStoreGoingToUpdateInPlace(opCtx, recId)); } - std::string updateQuery = "UPDATE \"" + _ident + "\" SET data = ? " + "WHERE rec_id = ?;"; - SqliteStatement updateStmt(*session, updateQuery); + SqliteStatement updateStmt(*session, "UPDATE \"", _ident, "\" SET data = ? WHERE rec_id = ?;"); updateStmt.bindBlob(0, data, len); updateStmt.bindInt(1, recId.repr()); updateStmt.step(SQLITE_DONE); @@ -403,8 +401,7 @@ Status MobileRecordStore::truncate(OperationContext* opCtx) { int64_t dataSizeBefore = dataSize(opCtx); _changeDataSize(opCtx, -dataSizeBefore); - std::string deleteForTruncateQuery = "DELETE FROM \"" + _ident + "\";"; - SqliteStatement::execQuery(session, deleteForTruncateQuery); + SqliteStatement::execQuery(session, "DELETE FROM \"", _ident, "\";"); return Status::OK(); } @@ -419,7 +416,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, ValidateResults* results, BSONObjBuilder* output) { if (level == kValidateFull) { - doValidate(opCtx, results); + embedded::doValidate(opCtx, results); } if (!results->valid) { @@ -429,8 +426,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); try { - std::string selectQuery = "SELECT rec_id, data FROM \"" + _ident + "\";"; - SqliteStatement selectStmt(*session, selectQuery); + SqliteStatement selectStmt(*session, "SELECT rec_id, data FROM \"", _ident, "\";"); int interruptInterval = 4096; long long actualNumRecs = 0; @@ -459,7 +455,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, if (!status.isOK() || validatedSize != static_cast<size_t>(dataSize)) { if (results->valid) { std::string errMsg = "detected one or more invalid documents"; - validateLogAndAppendError(results, errMsg); + embedded::validateLogAndAppendError(results, errMsg); } ++numInvalidRecs; @@ -470,7 +466,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, if (status == SQLITE_CORRUPT) { uasserted(ErrorCodes::UnknownError, sqlite3_errstr(status)); } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); // Verify that _numRecs and _dataSize are accurate. int64_t cachedNumRecs = numRecords(opCtx); @@ -479,7 +475,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, errMsg << "cached number of records does not match actual number of records - "; errMsg << "cached number of records = " << cachedNumRecs << "; "; errMsg << "actual number of records = " << actualNumRecs; - validateLogAndAppendError(results, errMsg); + embedded::validateLogAndAppendError(results, errMsg); } int64_t cachedDataSize = dataSize(opCtx); if (_resetDataSizeIfNeeded(opCtx, actualDataSize)) { @@ -487,7 +483,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, errMsg << "cached data size does not match actual data size - "; errMsg << "cached data size = " << cachedDataSize << "; "; errMsg << "actual data size = " << actualDataSize; - validateLogAndAppendError(results, errMsg); + embedded::validateLogAndAppendError(results, errMsg); } if (level == kValidateFull) { @@ -497,7 +493,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx, } catch (const DBException& e) { std::string errMsg = "record store is corrupt, could not read documents - " + e.toString(); - validateLogAndAppendError(results, errMsg); + embedded::validateLogAndAppendError(results, errMsg); } return Status::OK(); @@ -609,9 +605,10 @@ boost::optional<RecordId> MobileRecordStore::oplogStartHack( */ void MobileRecordStore::create(OperationContext* opCtx, const std::string& ident) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - std::string sqlQuery = - "CREATE TABLE IF NOT EXISTS \"" + ident + "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));"; - SqliteStatement::execQuery(session, sqlQuery); + SqliteStatement::execQuery(session, + "CREATE TABLE IF NOT EXISTS \"", + ident, + "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));"); } } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp index cc169d956d8..c4c9347bd9e 100644 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp +++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp @@ -34,8 +34,10 @@ #include <string> +#include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/operation_context.h" +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/mobile/mobile_recovery_unit.h" #include "mongo/db/storage/mobile/mobile_sqlite_statement.h" #include "mongo/db/storage/mobile/mobile_util.h" @@ -127,6 +129,41 @@ void MobileRecoveryUnit::abortUnitOfWork() { _abort(); } +bool MobileRecoveryUnit::waitUntilDurable() { + // This is going to be slow as we're taking a global X lock and doing a full checkpoint. This + // should not be needed to do on Android or iOS if we are on WAL and synchronous=NORMAL which + // are our default settings. The system will make sure any non-flushed writes will not be lost + // before going down but our powercycle test bench require it. Therefore make sure embedded does + // not call this (by disabling writeConcern j:true) but allow it when this is used inside + // mongod. + if (mobileGlobalOptions.mobileDurabilityLevel < 2) { + OperationContext* opCtx = Client::getCurrent()->getOperationContext(); + _ensureSession(opCtx); + RECOVERY_UNIT_TRACE() << "waitUntilDurable called, attempting to perform a checkpoint"; + int framesInWAL = 0; + int checkpointedFrames = 0; + int ret; + { + Lock::GlobalLock lk(opCtx, MODE_X); + // Use FULL mode to guarantee durability + ret = sqlite3_wal_checkpoint_v2(_session.get()->getSession(), + NULL, + SQLITE_CHECKPOINT_FULL, + &framesInWAL, + &checkpointedFrames); + } + embedded::checkStatus(ret, SQLITE_OK, "sqlite3_wal_checkpoint_v2"); + fassert(51164, + framesInWAL != -1 && checkpointedFrames != -1 && framesInWAL == checkpointedFrames); + RECOVERY_UNIT_TRACE() << "Checkpointed " << checkpointedFrames << " of the " << framesInWAL + << " total frames in the WAL"; + } else { + RECOVERY_UNIT_TRACE() << "No checkpoint attempted -- in full synchronous mode"; + } + + return true; +} + void MobileRecoveryUnit::abandonSnapshot() { invariant(!_inUnitOfWork); if (_active) { @@ -223,4 +260,4 @@ void MobileRecoveryUnit::_txnClose(bool commit) { void MobileRecoveryUnit::enqueueFailedDrop(std::string& dropQuery) { _sessionPool->failedDropsQueue.enqueueOp(dropQuery); } -} +} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.h b/src/mongo/db/storage/mobile/mobile_recovery_unit.h index 000f01a971c..e65c3681b82 100644 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.h +++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.h @@ -55,10 +55,7 @@ public: void beginUnitOfWork(OperationContext* opCtx) override; void commitUnitOfWork() override; void abortUnitOfWork() override; - - bool waitUntilDurable() override { - return true; - } + bool waitUntilDurable() override; void abandonSnapshot() override; diff --git a/src/mongo/db/storage/mobile/mobile_session_pool.cpp b/src/mongo/db/storage/mobile/mobile_session_pool.cpp index f09faf9e1ae..a289c4ebc52 100644 --- a/src/mongo/db/storage/mobile/mobile_session_pool.cpp +++ b/src/mongo/db/storage/mobile/mobile_session_pool.cpp @@ -120,7 +120,8 @@ std::unique_ptr<MobileSession> MobileSessionPool::getSession(OperationContext* o if (_curPoolSize < _maxPoolSize) { sqlite3* session; int status = sqlite3_open(_path.c_str(), &session); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::configureSession(session); _curPoolSize++; return stdx::make_unique<MobileSession>(session, this); } @@ -167,7 +168,7 @@ void MobileSessionPool::shutDown() { sqlite3* session; int status = sqlite3_open(_path.c_str(), &session); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); std::unique_ptr<MobileSession> mobSession = stdx::make_unique<MobileSession>(session, this); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Executing queued drops at shutdown"; failedDropsQueue.execAndDequeueAllOps(mobSession.get()); diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp index 0b26d25e1e7..ec07b5695a9 100644 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp +++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp @@ -43,24 +43,20 @@ #include "mongo/util/scopeguard.h" #define SQLITE_STMT_TRACE() LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite Stmt ID:" << _id << " " +#define SQLITE_STMT_TRACE_ENABLED() \ + (::mongo::logger::globalLogDomain()->shouldLog( \ + MongoLogDefaultComponent_component, \ + ::mongo::LogstreamBuilder::severityCast(MOBILE_TRACE_LEVEL))) namespace mongo { AtomicInt64 SqliteStatement::_nextID(0); -SqliteStatement::SqliteStatement(const MobileSession& session, const std::string& sqlQuery) { - // Increment the global instance count and assign this instance an id. - _id = _nextID.addAndFetch(1); - _sqlQuery = sqlQuery; - - prepare(session); -} - void SqliteStatement::finalize() { if (!_stmt) { return; } - SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery.data(); int status = sqlite3_finalize(_stmt); fassert(37053, status == _exceptionStatus); @@ -68,16 +64,16 @@ void SqliteStatement::finalize() { } void SqliteStatement::prepare(const MobileSession& session) { - SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery.data(); - int status = sqlite3_prepare_v2( - session.getSession(), _sqlQuery.c_str(), _sqlQuery.length() + 1, &_stmt, NULL); + int status = + sqlite3_prepare_v2(session.getSession(), _sqlQuery.data(), _sqlQuery.size(), &_stmt, NULL); if (status == SQLITE_BUSY) { SQLITE_STMT_TRACE() << "Throwing writeConflictException, " - << "SQLITE_BUSY while preparing: " << _sqlQuery; + << "SQLITE_BUSY while preparing: " << _sqlQuery.data(); throw WriteConflictException(); } else if (status != SQLITE_OK) { - SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery.data(); std::string errMsg = "sqlite3_prepare_v2 failed: "; errMsg += sqlite3_errstr(status); uasserted(ErrorCodes::UnknownError, errMsg); @@ -86,29 +82,35 @@ void SqliteStatement::prepare(const MobileSession& session) { SqliteStatement::~SqliteStatement() { finalize(); + + static_assert( + sizeof(SqliteStatement) == + sizeof(std::aligned_storage_t<sizeof(SqliteStatement), alignof(SqliteStatement)>), + "expected size to be exactly its aligned storage size to not waste memory, " + "adjust kMaxFixedSize to make this true"); } void SqliteStatement::bindInt(int paramIndex, int64_t intValue) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_int64(_stmt, paramIndex + 1, intValue); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::bindBlob(int paramIndex, const void* data, int len) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_blob(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::bindText(int paramIndex, const char* data, int len) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_text(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::clearBindings() { int status = sqlite3_clear_bindings(_stmt); - checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings"); } int SqliteStatement::step(int desiredStatus) { @@ -117,12 +119,15 @@ int SqliteStatement::step(int desiredStatus) { // A non-negative desiredStatus indicates that checkStatus should assert that the returned // status is equivalent to the desired status. if (desiredStatus >= 0) { - checkStatus(status, desiredStatus, "sqlite3_step"); + embedded::checkStatus(status, desiredStatus, "sqlite3_step"); } - char* full_stmt = sqlite3_expanded_sql(_stmt); - SQLITE_STMT_TRACE() << sqliteStatusToStr(status) << " - on stepping: " << full_stmt; - sqlite3_free(full_stmt); + if (SQLITE_STMT_TRACE_ENABLED()) { + char* full_stmt = sqlite3_expanded_sql(_stmt); + SQLITE_STMT_TRACE() << embedded::sqliteStatusToStr(status) + << " - on stepping: " << full_stmt; + sqlite3_free(full_stmt); + } return status; } @@ -143,11 +148,11 @@ const void* SqliteStatement::getColText(int colIndex) { return sqlite3_column_text(_stmt, colIndex); } -void SqliteStatement::execQuery(MobileSession* session, const std::string& query) { +void SqliteStatement::_execQuery(sqlite3* session, const char* query) { LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite sqlite3_exec: " << query; char* errMsg = NULL; - int status = sqlite3_exec(session->getSession(), query.c_str(), NULL, NULL, &errMsg); + int status = sqlite3_exec(session, query, NULL, NULL, &errMsg); if (status == SQLITE_BUSY || status == SQLITE_LOCKED) { LOG(MOBILE_TRACE_LEVEL) << "MobileSE: " << (status == SQLITE_BUSY ? "Busy" : "Locked") @@ -156,7 +161,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query } // The only return value from sqlite3_exec in a success case is SQLITE_OK. - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); // When the error message is not NULL, it is allocated through sqlite3_malloc and must be freed // before exiting the method. If the error message is NULL, sqlite3_free is a no-op. @@ -165,7 +170,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query void SqliteStatement::reset() { int status = sqlite3_reset(_stmt); - checkStatus(status, SQLITE_OK, "sqlite3_reset"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_reset"); } } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h index 719bc49a895..e24f62800dd 100644 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h +++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h @@ -36,6 +36,8 @@ #include "mongo/db/storage/mobile/mobile_session.h" #include "mongo/platform/atomic_word.h" +#include <boost/container/small_vector.hpp> + namespace mongo { /** @@ -47,7 +49,8 @@ public: /** * Creates and prepares a SQLite statement. */ - SqliteStatement(const MobileSession& session, const std::string& sqlQuery); + template <class... Args> + SqliteStatement(const MobileSession& session, Args&&... args); /** * Finalizes the prepared statement. @@ -116,7 +119,29 @@ public: * None of the rows retrieved, if any, are saved before the query is finalized. Thus, this * method should not be used for read operations. */ - static void execQuery(MobileSession* session, const std::string& query); + template <class... Args> + static void execQuery(sqlite3* session, const char* first, Args&&... args) { + // If we just have a single char*, we're good to go, no need to build a new string. + constexpr std::size_t num = sizeof...(args); + if (!num) { + _execQuery(session, first); + } else { + _execQueryBuilder(session, first, std::forward<Args>(args)...); + } + } + template <class... Args> + static void execQuery(sqlite3* session, const std::string& first, Args&&... args) { + execQuery(session, first.c_str(), std::forward<Args>(args)...); + } + template <class... Args> + static void execQuery(sqlite3* session, StringData first, Args&&... args) { + // StringData may not be null terminated, so build a new string + _execQueryBuilder(session, first, std::forward<Args>(args)...); + } + template <class... Args> + static void execQuery(MobileSession* session, Args&&... args) { + execQuery(session->getSession(), std::forward<Args>(args)...); + } /** * Finalizes a prepared statement. @@ -131,13 +156,100 @@ public: uint64_t _id; private: + static void _execQuery(sqlite3* session, const char* query); + template <class... Args> + static void _execQueryBuilder(sqlite3* session, Args&&... args); + static AtomicInt64 _nextID; sqlite3_stmt* _stmt; - std::string _sqlQuery; // If the most recent call to sqlite3_step on this statement returned an error, the error is // returned again when the statement is finalized. This is used to verify that the last error // code returned matches the finalize error code, if there is any. int _exceptionStatus = SQLITE_OK; + + // Static memory that fits short SQL statements to avoid a temporary memory allocation + static constexpr size_t kMaxFixedSize = 96; + using SqlQuery_t = boost::container::small_vector<char, kMaxFixedSize>; + SqlQuery_t _sqlQuery; + + template <std::size_t N> + static void queryAppend(SqlQuery_t& dest, char const (&str)[N], std::true_type) { + dest.insert(dest.end(), str, str + N - 1); + } + static void queryAppend(SqlQuery_t& dest, StringData sd, std::false_type) { + dest.insert(dest.end(), sd.begin(), sd.end()); + } + + template <class Arg> + static void queryAppendAll(SqlQuery_t& dest, Arg&& arg) { + queryAppend(dest, + std::forward<Arg>(arg), + std::is_array<typename std::remove_reference<Arg>::type>()); + } + + template <class Arg, class... Args> + static void queryAppendAll(SqlQuery_t& dest, Arg&& first, Args&&... args) { + queryAppend(dest, + std::forward<Arg>(first), + std::is_array<typename std::remove_reference<Arg>::type>()); + queryAppendAll(dest, std::forward<Args>(args)...); + } }; + +namespace detail { +// Most of the strings we build statements with are static strings so we can calculate their length +// during compile time. +// Arrays decay to pointer in overload resolution, force that to not happen by providing true_type +// as second argument if array +template <std::size_t N> +constexpr std::size_t stringLength(char const (&)[N], std::true_type) { + // Omit the null terminator, will added back when we call reserve later + return N - 1; +} +inline std::size_t stringLength(StringData sd, std::false_type) { + return sd.size(); +} + +template <class Arg> +std::size_t totalStringLength(Arg&& arg) { + return stringLength(std::forward<Arg>(arg), + std::is_array<typename std::remove_reference<Arg>::type>()); +} + +template <class Arg, class... Args> +std::size_t totalStringLength(Arg&& first, Args&&... args) { + return stringLength(std::forward<Arg>(first), + std::is_array<typename std::remove_reference<Arg>::type>()) + + totalStringLength(std::forward<Args>(args)...); +} + +} // namespace detail + +template <class... Args> +SqliteStatement::SqliteStatement(const MobileSession& session, Args&&... args) { + // Increment the global instance count and assign this instance an id. + _id = _nextID.addAndFetch(1); + + // Reserve the size we need once to avoid any additional allocations + _sqlQuery.reserve(detail::totalStringLength(std::forward<Args>(args)...) + 1); + + // Copy all substrings into buffer for SQL statement + queryAppendAll(_sqlQuery, std::forward<Args>(args)...); + _sqlQuery.push_back('\0'); + + prepare(session); +} + +template <class... Args> +void SqliteStatement::_execQueryBuilder(sqlite3* session, Args&&... args) { + SqlQuery_t sqlQuery; + + sqlQuery.reserve(detail::totalStringLength(std::forward<Args>(args)...) + 1); + + queryAppendAll(sqlQuery, std::forward<Args>(args)...); + sqlQuery.push_back('\0'); + _execQuery(session, sqlQuery.data()); +} + } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.cpp b/src/mongo/db/storage/mobile/mobile_util.cpp index be8a88b8e6c..aac9b8a39bf 100644 --- a/src/mongo/db/storage/mobile/mobile_util.cpp +++ b/src/mongo/db/storage/mobile/mobile_util.cpp @@ -35,11 +35,13 @@ #include <sqlite3.h> +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/mobile/mobile_recovery_unit.h" #include "mongo/db/storage/mobile/mobile_sqlite_statement.h" #include "mongo/db/storage/mobile/mobile_util.h" namespace mongo { +namespace embedded { using std::string; @@ -152,9 +154,8 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM void doValidate(OperationContext* opCtx, ValidateResults* results) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string validateQuery = "PRAGMA integrity_check;"; try { - SqliteStatement validateStmt(*session, validateQuery); + SqliteStatement validateStmt(*session, "PRAGMA integrity_check;"); int status; // By default, the integrity check returns the first 100 errors found. @@ -182,4 +183,31 @@ void doValidate(OperationContext* opCtx, ValidateResults* results) { } } +void configureSession(sqlite3* session) { + auto executePragma = [session](auto pragma, auto value) { + SqliteStatement::execQuery(session, "PRAGMA ", pragma, " = ", value, ";"); + LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE session configuration: " << pragma << " = " << value; + }; + // We don't manually use VACUUM so set incremental mode to reclaim space + executePragma("auto_vacuum"_sd, "incremental"_sd); + + // Set SQLite in Write-Ahead Logging mode. https://sqlite.org/wal.html + executePragma("journal_mode"_sd, "WAL"_sd); + + // synchronous = NORMAL(1) is recommended with WAL, but we allow it to be overriden + executePragma("synchronous"_sd, std::to_string(mobileGlobalOptions.mobileDurabilityLevel)); + + // Set full fsync on OSX (only supported there) to ensure durability + executePragma("fullfsync"_sd, "1"_sd); + + // We just use SQLite as key-value store, so disable foreign keys + executePragma("foreign_keys"_sd, "0"_sd); + + // Set some additional internal sizes for this session + executePragma("cache_size"_sd, "-10240"_sd); // 10KB (passed as negative means KB) + executePragma("mmap_size"_sd, "52428800"_sd); // 50MB + executePragma("journal_size_limit"_sd, "5242880"_sd); // 5MB +} + +} // namespace embedded } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.h b/src/mongo/db/storage/mobile/mobile_util.h index 3c7b18dde7d..9e9c429ef50 100644 --- a/src/mongo/db/storage/mobile/mobile_util.h +++ b/src/mongo/db/storage/mobile/mobile_util.h @@ -39,6 +39,7 @@ #define MOBILE_TRACE_LEVEL MOBILE_LOG_LEVEL_HIGH namespace mongo { +namespace embedded { /** * Converts SQLite return codes to MongoDB statuses. @@ -65,4 +66,11 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM */ void doValidate(OperationContext* opCtx, ValidateResults* results); +/** + * Sets the SQLite Pragmas that we want (https://www.sqlite.org/pragma.html) + * These should generally improve behavior, performance, and resource usage + */ +void configureSession(sqlite3* session); + +} // namespace embedded } // namespace mongo |