diff options
author | Henrik Edin <henrik.edin@mongodb.com> | 2019-03-26 10:27:48 -0400 |
---|---|---|
committer | Henrik Edin <henrik.edin@mongodb.com> | 2019-04-05 10:16:25 -0400 |
commit | 2ee7dcb2bcb4372a49b8584c43bb65caea6212c8 (patch) | |
tree | ad51e1b759f865f0706e14c208c044e2dfa653e7 | |
parent | 7a6a21915dcde1f232e7bbde5aa3738be69befd4 (diff) | |
download | mongo-2ee7dcb2bcb4372a49b8584c43bb65caea6212c8.tar.gz |
SERVER-32709 Improve performance with the mobile storage engine.
SQLite is configured with synchronous=NORMAL for all sessions.
Reduced temporary memory allocations when constructing SQL statements.
Implemented waitUntilDurable when this storage engine is used in mongod only.
-rw-r--r-- | jstests/concurrency/fsm_workloads/reindex.js | 2 | ||||
-rw-r--r-- | jstests/concurrency/fsm_workloads/reindex_background.js | 2 | ||||
-rw-r--r-- | jstests/noPassthrough/indexbg1.js | 1 | ||||
-rw-r--r-- | jstests/noPassthrough/indexbg2.js | 2 | ||||
-rw-r--r-- | jstests/noPassthroughWithMongod/geo_polygon.js | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_index.cpp | 62 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_kv_engine.cpp | 82 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_record_store.cpp | 71 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_recovery_unit.cpp | 39 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_recovery_unit.h | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_session_pool.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp | 57 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_sqlite_statement.h | 99 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_util.cpp | 33 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_util.h | 8 |
15 files changed, 309 insertions, 163 deletions
diff --git a/jstests/concurrency/fsm_workloads/reindex.js b/jstests/concurrency/fsm_workloads/reindex.js index ff9d0604ed5..8ca7f8223ae 100644 --- a/jstests/concurrency/fsm_workloads/reindex.js +++ b/jstests/concurrency/fsm_workloads/reindex.js @@ -5,8 +5,6 @@ * * Bulk inserts 1000 documents and builds indexes. Then alternates between reindexing and querying * against the collection. Operates on a separate collection for each thread. - * - * @tags: [SERVER-32709] */ var $config = (function() { diff --git a/jstests/concurrency/fsm_workloads/reindex_background.js b/jstests/concurrency/fsm_workloads/reindex_background.js index 79aad331087..9e781f4a4ef 100644 --- a/jstests/concurrency/fsm_workloads/reindex_background.js +++ b/jstests/concurrency/fsm_workloads/reindex_background.js @@ -8,7 +8,7 @@ * that because indexes are initially built in the background, reindexing is also done in the * background. * - * @tags: [SERVER-32709, creates_background_indexes] + * @tags: [creates_background_indexes] */ load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js index d0713f9c303..2248d86392a 100644 --- a/jstests/noPassthrough/indexbg1.js +++ b/jstests/noPassthrough/indexbg1.js @@ -1,5 +1,4 @@ // Test background index creation -// @tags: [SERVER-32709] (function() { "use strict"; diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js index 913fac1d1a2..086ca385437 100644 --- a/jstests/noPassthrough/indexbg2.js +++ b/jstests/noPassthrough/indexbg2.js @@ -1,5 +1,5 @@ // Test background index creation w/ constraints -// @tags: [SERVER-32709, requires_document_locking] +// @tags: [requires_document_locking] (function() { "use strict"; diff --git a/jstests/noPassthroughWithMongod/geo_polygon.js b/jstests/noPassthroughWithMongod/geo_polygon.js index 4ce1f8d2ad7..d2b271c32d7 100644 --- a/jstests/noPassthroughWithMongod/geo_polygon.js +++ b/jstests/noPassthroughWithMongod/geo_polygon.js @@ -1,7 +1,3 @@ -// -// @tags: [SERVER-32709] -// - t = db.geo_polygon4; t.drop(); diff --git a/src/mongo/db/storage/mobile/mobile_index.cpp b/src/mongo/db/storage/mobile/mobile_index.cpp index f95440dfa85..a1c54273ce0 100644 --- a/src/mongo/db/storage/mobile/mobile_index.cpp +++ b/src/mongo/db/storage/mobile/mobile_index.cpp @@ -102,8 +102,8 @@ StatusWith<SpecialFormatInserted> MobileIndex::doInsert(OperationContext* opCtx, session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); } - std::string insertQuery = "INSERT INTO \"" + _ident + "\" (key, value) VALUES (?, ?);"; - SqliteStatement insertStmt(*session, insertQuery); + SqliteStatement insertStmt( + *session, "INSERT INTO \"", _ident, "\" (key, value) VALUES (?, ?);"); insertStmt.bindBlob(0, key.getBuffer(), key.getSize()); insertStmt.bindBlob(1, value.getBuffer(), value.getSize()); @@ -124,7 +124,7 @@ StatusWith<SpecialFormatInserted> MobileIndex::doInsert(OperationContext* opCtx, SpecialFormatInserted::NoSpecialFormatInserted); } } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); if (key.getTypeBits().isLongEncoding()) return StatusWith<SpecialFormatInserted>(SpecialFormatInserted::LongTypeBitsInserted); @@ -145,13 +145,8 @@ void MobileIndex::unindex(OperationContext* opCtx, void MobileIndex::_doDelete(OperationContext* opCtx, const KeyString& key, KeyString* value) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - str::stream deleteQuery; - deleteQuery << "DELETE FROM \"" << _ident << "\" WHERE key = ?"; - if (value) { - deleteQuery << " AND value = ?"; - } - deleteQuery << ";"; - SqliteStatement deleteStmt(*session, deleteQuery); + SqliteStatement deleteStmt( + *session, "DELETE FROM \"", _ident, "\" WHERE key = ?", value ? " AND value = ?" : "", ";"); deleteStmt.bindBlob(0, key.getBuffer(), key.getSize()); if (value) { @@ -167,7 +162,7 @@ void MobileIndex::fullValidate(OperationContext* opCtx, long long* numKeysOut, ValidateResults* fullResults) const { if (fullResults) { - doValidate(opCtx, fullResults); + embedded::doValidate(opCtx, fullResults); if (!fullResults->valid) { return; } @@ -189,10 +184,11 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const { // Sum the number of bytes in each column. // SQLite aggregate functions return null if the column is empty or has only nulls, so return 0 // bytes if there is no data in the column. - str::stream sizeQuery; - sizeQuery << "SELECT IFNULL(SUM(LENGTH(key)), 0) + " - << "IFNULL(SUM(LENGTH(value)), 0) FROM \"" << _ident + "\";"; - SqliteStatement sizeStmt(*session, sizeQuery); + SqliteStatement sizeStmt(*session, + "SELECT IFNULL(SUM(LENGTH(key)), 0) + ", + "IFNULL(SUM(LENGTH(value)), 0) FROM \"", + _ident, + "\";"); sizeStmt.step(SQLITE_ROW); @@ -202,8 +198,8 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const { long long MobileIndex::numEntries(OperationContext* opCtx) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string countQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";"; - SqliteStatement countStmt(*session, countQuery); + + SqliteStatement countStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); countStmt.step(SQLITE_ROW); long long numRecs = countStmt.getColInt(0); @@ -212,14 +208,14 @@ long long MobileIndex::numEntries(OperationContext* opCtx) const { bool MobileIndex::isEmpty(OperationContext* opCtx) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string emptyCheckQuery = "SELECT * FROM \"" + _ident + "\" LIMIT 1;"; - SqliteStatement emptyCheckStmt(*session, emptyCheckQuery); + + SqliteStatement emptyCheckStmt(*session, "SELECT * FROM \"", _ident, "\" LIMIT 1;"); int status = emptyCheckStmt.step(); if (status == SQLITE_DONE) { return true; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); return false; } @@ -230,9 +226,9 @@ Status MobileIndex::initAsEmpty(OperationContext* opCtx) { Status MobileIndex::create(OperationContext* opCtx, const std::string& ident) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - std::string createTableQuery = - "CREATE TABLE \"" + ident + "\"(key BLOB PRIMARY KEY, value BLOB);"; - SqliteStatement createTableStmt(*session, createTableQuery.c_str()); + + SqliteStatement createTableStmt( + *session, "CREATE TABLE \"", ident, "\"(key BLOB PRIMARY KEY, value BLOB);"); createTableStmt.step(SQLITE_DONE); return Status::OK(); @@ -249,8 +245,8 @@ Status MobileIndex::dupKeyCheck(OperationContext* opCtx, const BSONObj& key) { bool MobileIndex::_isDup(OperationContext* opCtx, const BSONObj& key) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string dupCheckQuery = "SELECT COUNT(*) FROM \"" + _ident + "\" WHERE key = ?;"; - SqliteStatement dupCheckStmt(*session, dupCheckQuery); + + SqliteStatement dupCheckStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\" WHERE key = ?;"); KeyString keyStr(_keyStringVersion, key, _ordering); dupCheckStmt.bindBlob(0, keyStr.getBuffer(), keyStr.getSize()); @@ -389,11 +385,15 @@ public: _savedTypeBits(index.getKeyStringVersion()), _startPosition(index.getKeyStringVersion()) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - str::stream cursorQuery; - cursorQuery << "SELECT key, value FROM \"" << _index.getIdent() << "\" WHERE key "; - cursorQuery << (_isForward ? ">=" : "<=") << " ? ORDER BY key "; - cursorQuery << (_isForward ? "ASC" : "DESC") << ";"; - _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery); + + _stmt = stdx::make_unique<SqliteStatement>(*session, + "SELECT key, value FROM \"", + _index.getIdent(), + "\" WHERE key ", + (_isForward ? ">=" : "<="), + " ? ORDER BY key ", + (_isForward ? "ASC" : "DESC"), + ";"); } virtual ~CursorBase() {} @@ -503,7 +503,7 @@ protected: _isEOF = true; return; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); _isEOF = false; diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp index 5621e10c03f..14842514d28 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp @@ -65,90 +65,68 @@ MobileKVEngine::MobileKVEngine(const std::string& path, // Initialize the database to be in WAL mode. sqlite3* initSession; int status = sqlite3_open(_path.c_str(), &initSession); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); // Guarantees that sqlite3_close() will be called when the function returns. ON_BLOCK_EXIT([&initSession] { sqlite3_close(initSession); }); - // Set all of our SQLite pragmas (https://www.sqlite.org/pragma.html) - // These should generally improve behavior, performance, and resource usage - { - for (auto it : - {std::string("journal_mode = WAL"), - "synchronous = " + std::to_string(durabilityLevel), - std::string("fullfsync = 1"), - // Allow for periodic calls to purge deleted records and prune db size on disk - // Still requires manual vacuum calls using `PRAGMA incremental_vacuum(N);` - std::string("auto_vacuum = incremental"), - "cache_size = -" + std::to_string(cacheSizeKB), - "mmap_size = " + std::to_string(mmapSizeKB * 1024), - "journal_size_limit = " + std::to_string(journalSizeLimitKB * 1024)}) { - std::string execPragma = "PRAGMA " + it + ";"; - char* errMsg = NULL; - std::int32_t status = - sqlite3_exec(initSession, execPragma.c_str(), NULL, NULL, &errMsg); - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); - sqlite3_free(errMsg); - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE configuration: " << execPragma; - } - - LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Completed all SQLite database configuration"; - } + embedded::configureSession(initSession); - // Check and enforce WAL mode + // Check and ensure that WAL mode is working as expected // This is not something that we want to be configurable { sqlite3_stmt* stmt; status = sqlite3_prepare_v2(initSession, "PRAGMA journal_mode;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current mode in SQLite, ensure it is "wal" mode. const void* colText = sqlite3_column_text(stmt, 0); const char* mode = reinterpret_cast<const char*>(colText); fassert(37001, !strcmp(mode, "wal")); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database opened in WAL mode"; } - // Check and enforce the synchronous mode + // Check and ensure that synchronous mode is working as expected { sqlite3_stmt* stmt; status = sqlite3_prepare_v2(initSession, "PRAGMA synchronous;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current "synchronous" setting std::uint32_t sync_val = sqlite3_column_int(stmt, 0); fassert(50869, sync_val == durabilityLevel); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database has synchronous " << "set to: " << durabilityLevel; } - // Check and enforce that we are using the F_FULLFSYNC fcntl on darwin kernels + // Check and ensure that we were able to set the F_FULLFSYNC fcntl on darwin kernels // This prevents data corruption as fsync doesn't work as expected + // This is not something that we want to be configurable { sqlite3_stmt* stmt; status = sqlite3_prepare_v2(initSession, "PRAGMA fullfsync;", -1, &stmt, NULL); - checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); // Pragma returns current fullsync setting, ensure it is enabled. int fullfsync_val = sqlite3_column_int(stmt, 0); fassert(50868, fullfsync_val == 1); status = sqlite3_finalize(stmt); - checkStatus(status, SQLITE_OK, "sqlite3_finalize"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database is set to fsync " << "with F_FULLFSYNC if the platform supports it (currently" @@ -254,7 +232,7 @@ Status MobileKVEngine::dropIdent(OperationContext* opCtx, StringData ident) { std::string dropQuery = "DROP TABLE IF EXISTS \"" + ident + "\";"; try { - SqliteStatement::execQuery(session, dropQuery.c_str()); + SqliteStatement::execQuery(session, dropQuery); } catch (const WriteConflictException&) { // It is possible that this drop fails because of transaction running in parallel. // We pretend that it succeeded, queue it for now and keep retrying later. @@ -274,8 +252,7 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); // Get key-value column names. - std::string colNameQuery = "PRAGMA table_info(\"" + ident + "\")"; - SqliteStatement colNameStmt(*session, colNameQuery); + SqliteStatement colNameStmt(*session, "PRAGMA table_info(\"", ident, "\")"); colNameStmt.step(SQLITE_ROW); std::string keyColName(static_cast<const char*>(colNameStmt.getColText(1))); @@ -284,10 +261,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) colNameStmt.step(SQLITE_DONE); // Get total data size of key-value columns. - str::stream dataSizeQuery; - dataSizeQuery << "SELECT IFNULL(SUM(LENGTH(" << keyColName << ")), 0) + " - << "IFNULL(SUM(LENGTH(" << valueColName << ")), 0) FROM \"" << ident + "\";"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + SqliteStatement dataSizeStmt(*session, + "SELECT IFNULL(SUM(LENGTH(", + keyColName, + ")), 0) + ", + "IFNULL(SUM(LENGTH(", + valueColName, + ")), 0) FROM \"", + ident, + "\";"); dataSizeStmt.step(SQLITE_ROW); return dataSizeStmt.getColInt(0); @@ -296,15 +278,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident) bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string findTableQuery = "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;"; - SqliteStatement findTableStmt(*session, findTableQuery); + SqliteStatement findTableStmt(*session, + "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;"); findTableStmt.bindText(0, ident.rawData(), ident.size()); int status = findTableStmt.step(); if (status == SQLITE_DONE) { return false; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); return true; } @@ -312,15 +294,15 @@ bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const { std::vector<std::string> MobileKVEngine::getAllIdents(OperationContext* opCtx) const { std::vector<std::string> idents; MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string getTablesQuery = "SELECT name FROM sqlite_master WHERE type='table';"; - SqliteStatement getTablesStmt(*session, getTablesQuery); + + SqliteStatement getTablesStmt(*session, "SELECT name FROM sqlite_master WHERE type='table';"); int status; while ((status = getTablesStmt.step()) == SQLITE_ROW) { std::string tableName(reinterpret_cast<const char*>(getTablesStmt.getColText(0))); idents.push_back(tableName); } - checkStatus(status, SQLITE_DONE, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step"); return idents; } diff --git a/src/mongo/db/storage/mobile/mobile_record_store.cpp b/src/mongo/db/storage/mobile/mobile_record_store.cpp index d83ba4f5c89..c40c915a90a 100644 --- a/src/mongo/db/storage/mobile/mobile_record_store.cpp +++ b/src/mongo/db/storage/mobile/mobile_record_store.cpp @@ -62,13 +62,17 @@ public: bool forward) : _opCtx(opCtx), _forward(forward) { - str::stream cursorQuery; - cursorQuery << "SELECT rec_id, data from \"" << ident << "\" " - << "WHERE rec_id " << (forward ? '>' : '<') << " ? " - << "ORDER BY rec_id " << (forward ? "ASC" : "DESC") << ';'; - MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx); - _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery); + _stmt = stdx::make_unique<SqliteStatement>(*session, + "SELECT rec_id, data from \"", + ident, + "\" ", + "WHERE rec_id ", + (forward ? ">" : "<"), + " ? ", + "ORDER BY rec_id ", + (forward ? "ASC" : "DESC"), + ";"); _startIdNum = (forward ? RecordId::min().repr() : RecordId::max().repr()); _savedId = RecordId(_startIdNum); @@ -90,7 +94,7 @@ public: } // Checks no error was thrown and that step retrieved a row. - checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next"); + embedded::checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next"); long long recId = _stmt->getColInt(0); const void* data = _stmt->getColBlob(1); @@ -196,8 +200,7 @@ MobileRecordStore::MobileRecordStore(OperationContext* opCtx, // Determines the nextId to be used for a new record. MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string maxRecIdQuery = "SELECT IFNULL(MAX(rec_id), 0) FROM \"" + _ident + "\";"; - SqliteStatement maxRecIdStmt(*session, maxRecIdQuery); + SqliteStatement maxRecIdStmt(*session, "SELECT IFNULL(MAX(rec_id), 0) FROM \"", _ident, "\";"); maxRecIdStmt.step(SQLITE_ROW); @@ -219,8 +222,8 @@ void MobileRecordStore::_initDataSizeIfNeeded_inlock(OperationContext* opCtx) co } MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string dataSizeQuery = "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"" + _ident + "\";"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"", _ident, "\";"); dataSizeStmt.step(SQLITE_ROW); int64_t dataSize = dataSizeStmt.getColInt(0); @@ -241,8 +244,7 @@ void MobileRecordStore::_initNumRecsIfNeeded_inlock(OperationContext* opCtx) con } MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string numRecordsQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";"; - SqliteStatement numRecordsStmt(*session, numRecordsQuery); + SqliteStatement numRecordsStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";"); numRecordsStmt.step(SQLITE_ROW); @@ -262,8 +264,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, const RecordId& recId, RecordData* rd) const { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string sqlQuery = "SELECT data FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement stmt(*session, sqlQuery); + SqliteStatement stmt(*session, "SELECT data FROM \"", _ident, "\" WHERE rec_id = ?;"); stmt.bindInt(0, recId.repr()); @@ -271,7 +272,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, if (status == SQLITE_DONE) { return false; } - checkStatus(status, SQLITE_ROW, "sqlite3_step"); + embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step"); const void* recData = stmt.getColBlob(0); int nBytes = stmt.getColBytes(0); @@ -281,9 +282,9 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx, void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& recId) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - std::string dataSizeQuery = - "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); dataSizeStmt.bindInt(0, recId.repr()); dataSizeStmt.step(SQLITE_ROW); @@ -291,8 +292,7 @@ void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& re _changeNumRecs(opCtx, -1); _changeDataSize(opCtx, -dataSizeBefore); - std::string deleteQuery = "DELETE FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement deleteStmt(*session, deleteQuery); + SqliteStatement deleteStmt(*session, "DELETE FROM \"", _ident, "\" WHERE rec_id = ?;"); deleteStmt.bindInt(0, recId.repr()); deleteStmt.step(SQLITE_DONE); } @@ -303,6 +303,9 @@ Status MobileRecordStore::insertRecords(OperationContext* opCtx, // Inserts record into SQLite table (or replaces if duplicate record id). MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); + SqliteStatement insertStmt( + *session, "INSERT OR REPLACE INTO \"", _ident, "\"(rec_id, data) VALUES(?, ?);"); + for (auto& record : *inOutRecords) { const auto data = record.data.data(); const auto len = record.data.size(); @@ -310,15 +313,13 @@ Status MobileRecordStore::insertRecords(OperationContext* opCtx, _changeNumRecs(opCtx, 1); _changeDataSize(opCtx, len); - std::string insertQuery = - "INSERT OR REPLACE INTO \"" + _ident + "\"(rec_id, data) VALUES(?, ?);"; - SqliteStatement insertStmt(*session, insertQuery); RecordId recId = _nextId(); insertStmt.bindInt(0, recId.repr()); insertStmt.bindBlob(1, data, len); insertStmt.step(SQLITE_DONE); record.id = recId; + insertStmt.reset(); } return Status::OK(); @@ -353,17 +354,17 @@ Status MobileRecordStore::updateRecord(OperationContext* opCtx, const char* data, int len) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false); - std::string dataSizeQuery = - "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;"; - SqliteStatement dataSizeStmt(*session, dataSizeQuery); + + SqliteStatement dataSizeStmt( + *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;"); dataSizeStmt.bindInt(0, recId.repr()); dataSizeStmt.step(SQLITE_ROW); int64_t dataSizeBefore = dataSizeStmt.getColInt(0); _changeDataSize(opCtx, -dataSizeBefore + len); - std::string updateQuery = "UPDATE \"" + _ident + "\" SET data = ? " + "WHERE rec_id = ?;"; - SqliteStatement updateStmt(*session, updateQuery); + SqliteStatement updateStmt( + *session, "UPDATE \"", _ident, "\" SET data = ? ", "WHERE rec_id = ?;"); updateStmt.bindBlob(0, data, len); updateStmt.bindInt(1, recId.repr()); updateStmt.step(SQLITE_DONE); @@ -402,8 +403,7 @@ Status MobileRecordStore::truncate(OperationContext* opCtx) { int64_t dataSizeBefore = dataSize(opCtx); _changeDataSize(opCtx, -dataSizeBefore); - std::string deleteForTruncateQuery = "DELETE FROM \"" + _ident + "\";"; - SqliteStatement::execQuery(session, deleteForTruncateQuery); + SqliteStatement::execQuery(session, "DELETE FROM \"", _ident, "\";"); return Status::OK(); } @@ -417,7 +417,7 @@ void MobileRecordStore::validate(OperationContext* opCtx, ValidateResults* results, BSONObjBuilder* output) { if (level == kValidateFull) { - doValidate(opCtx, results); + embedded::doValidate(opCtx, results); } } @@ -527,9 +527,10 @@ boost::optional<RecordId> MobileRecordStore::oplogStartHack( */ void MobileRecordStore::create(OperationContext* opCtx, const std::string& ident) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx); - std::string sqlQuery = - "CREATE TABLE IF NOT EXISTS \"" + ident + "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));"; - SqliteStatement::execQuery(session, sqlQuery); + SqliteStatement::execQuery(session, + "CREATE TABLE IF NOT EXISTS \"", + ident, + "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));"); } void MobileRecordStore::updateStatsAfterRepair(OperationContext* opCtx, diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp index b3a9c37f843..f3f04b5972c 100644 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp +++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp @@ -33,8 +33,10 @@ #include <string> +#include "mongo/db/concurrency/d_concurrency.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/operation_context.h" +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/mobile/mobile_recovery_unit.h" #include "mongo/db/storage/mobile/mobile_sqlite_statement.h" #include "mongo/db/storage/mobile/mobile_util.h" @@ -126,6 +128,41 @@ void MobileRecoveryUnit::abortUnitOfWork() { _abort(); } +bool MobileRecoveryUnit::waitUntilDurable() { + // This is going to be slow as we're taking a global X lock and doing a full checkpoint. This + // should not be needed to do on Android or iOS if we are on WAL and synchronous=NORMAL which + // are our default settings. The system will make sure any non-flushed writes will not be lost + // before going down but our powercycle test bench require it. Therefore make sure embedded does + // not call this (by disabling writeConcern j:true) but allow it when this is used inside + // mongod. + if (mobileGlobalOptions.mobileDurabilityLevel < 2) { + OperationContext* opCtx = Client::getCurrent()->getOperationContext(); + _ensureSession(opCtx); + RECOVERY_UNIT_TRACE() << "waitUntilDurable called, attempting to perform a checkpoint"; + int framesInWAL = 0; + int checkpointedFrames = 0; + int ret; + { + Lock::GlobalLock lk(opCtx, MODE_X); + // Use FULL mode to guarantee durability + ret = sqlite3_wal_checkpoint_v2(_session.get()->getSession(), + NULL, + SQLITE_CHECKPOINT_FULL, + &framesInWAL, + &checkpointedFrames); + } + embedded::checkStatus(ret, SQLITE_OK, "sqlite3_wal_checkpoint_v2"); + fassert(51160, + framesInWAL != -1 && checkpointedFrames != -1 && framesInWAL == checkpointedFrames); + RECOVERY_UNIT_TRACE() << "Checkpointed " << checkpointedFrames << " of the " << framesInWAL + << " total frames in the WAL"; + } else { + RECOVERY_UNIT_TRACE() << "No checkpoint attempted -- in full synchronous mode"; + } + + return true; +} + void MobileRecoveryUnit::abandonSnapshot() { invariant(!_inUnitOfWork); if (_active) { @@ -222,4 +259,4 @@ void MobileRecoveryUnit::_txnClose(bool commit) { void MobileRecoveryUnit::enqueueFailedDrop(std::string& dropQuery) { _sessionPool->failedDropsQueue.enqueueOp(dropQuery); } -} +} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.h b/src/mongo/db/storage/mobile/mobile_recovery_unit.h index 864ced082fb..6cce4542c89 100644 --- a/src/mongo/db/storage/mobile/mobile_recovery_unit.h +++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.h @@ -54,10 +54,7 @@ public: void beginUnitOfWork(OperationContext* opCtx) override; void commitUnitOfWork() override; void abortUnitOfWork() override; - - bool waitUntilDurable() override { - return true; - } + bool waitUntilDurable() override; void abandonSnapshot() override; diff --git a/src/mongo/db/storage/mobile/mobile_session_pool.cpp b/src/mongo/db/storage/mobile/mobile_session_pool.cpp index bfea09fb01b..38f7b1b0575 100644 --- a/src/mongo/db/storage/mobile/mobile_session_pool.cpp +++ b/src/mongo/db/storage/mobile/mobile_session_pool.cpp @@ -119,7 +119,8 @@ std::unique_ptr<MobileSession> MobileSessionPool::getSession(OperationContext* o if (_curPoolSize < _maxPoolSize) { sqlite3* session; int status = sqlite3_open(_path.c_str(), &session); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::configureSession(session); _curPoolSize++; return stdx::make_unique<MobileSession>(session, this); } @@ -166,7 +167,7 @@ void MobileSessionPool::shutDown() { sqlite3* session; int status = sqlite3_open(_path.c_str(), &session); - checkStatus(status, SQLITE_OK, "sqlite3_open"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_open"); std::unique_ptr<MobileSession> mobSession = stdx::make_unique<MobileSession>(session, this); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Executing queued drops at shutdown"; failedDropsQueue.execAndDequeueAllOps(mobSession.get()); diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp index d96150e6427..cebd769dc2f 100644 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp +++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp @@ -42,24 +42,20 @@ #include "mongo/util/scopeguard.h" #define SQLITE_STMT_TRACE() LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite Stmt ID:" << _id << " " +#define SQLITE_STMT_TRACE_ENABLED() \ + (::mongo::logger::globalLogDomain()->shouldLog( \ + MongoLogDefaultComponent_component, \ + ::mongo::LogstreamBuilder::severityCast(MOBILE_TRACE_LEVEL))) namespace mongo { AtomicWord<long long> SqliteStatement::_nextID(0); -SqliteStatement::SqliteStatement(const MobileSession& session, const std::string& sqlQuery) { - // Increment the global instance count and assign this instance an id. - _id = _nextID.addAndFetch(1); - _sqlQuery = sqlQuery; - - prepare(session); -} - void SqliteStatement::finalize() { if (!_stmt) { return; } - SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery.data(); int status = sqlite3_finalize(_stmt); fassert(37053, status == _exceptionStatus); @@ -67,16 +63,16 @@ void SqliteStatement::finalize() { } void SqliteStatement::prepare(const MobileSession& session) { - SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery.data(); - int status = sqlite3_prepare_v2( - session.getSession(), _sqlQuery.c_str(), _sqlQuery.length() + 1, &_stmt, NULL); + int status = + sqlite3_prepare_v2(session.getSession(), _sqlQuery.data(), _sqlQuery.size(), &_stmt, NULL); if (status == SQLITE_BUSY) { SQLITE_STMT_TRACE() << "Throwing writeConflictException, " - << "SQLITE_BUSY while preparing: " << _sqlQuery; + << "SQLITE_BUSY while preparing: " << _sqlQuery.data(); throw WriteConflictException(); } else if (status != SQLITE_OK) { - SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery; + SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery.data(); std::string errMsg = "sqlite3_prepare_v2 failed: "; errMsg += sqlite3_errstr(status); uasserted(ErrorCodes::UnknownError, errMsg); @@ -85,29 +81,35 @@ void SqliteStatement::prepare(const MobileSession& session) { SqliteStatement::~SqliteStatement() { finalize(); + + static_assert( + sizeof(SqliteStatement) == + sizeof(std::aligned_storage_t<sizeof(SqliteStatement), alignof(SqliteStatement)>), + "expected size to be exactly its aligned storage size to not waste memory, " + "adjust kMaxFixedSize to make this true"); } void SqliteStatement::bindInt(int paramIndex, int64_t intValue) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_int64(_stmt, paramIndex + 1, intValue); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::bindBlob(int paramIndex, const void* data, int len) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_blob(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::bindText(int paramIndex, const char* data, int len) { // SQLite bind methods begin paramater indexes at 1 rather than 0. int status = sqlite3_bind_text(_stmt, paramIndex + 1, data, len, SQLITE_STATIC); - checkStatus(status, SQLITE_OK, "sqlite3_bind"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind"); } void SqliteStatement::clearBindings() { int status = sqlite3_clear_bindings(_stmt); - checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings"); } int SqliteStatement::step(int desiredStatus) { @@ -116,12 +118,15 @@ int SqliteStatement::step(int desiredStatus) { // A non-negative desiredStatus indicates that checkStatus should assert that the returned // status is equivalent to the desired status. if (desiredStatus >= 0) { - checkStatus(status, desiredStatus, "sqlite3_step"); + embedded::checkStatus(status, desiredStatus, "sqlite3_step"); } - char* full_stmt = sqlite3_expanded_sql(_stmt); - SQLITE_STMT_TRACE() << sqliteStatusToStr(status) << " - on stepping: " << full_stmt; - sqlite3_free(full_stmt); + if (SQLITE_STMT_TRACE_ENABLED()) { + char* full_stmt = sqlite3_expanded_sql(_stmt); + SQLITE_STMT_TRACE() << embedded::sqliteStatusToStr(status) + << " - on stepping: " << full_stmt; + sqlite3_free(full_stmt); + } return status; } @@ -142,11 +147,11 @@ const void* SqliteStatement::getColText(int colIndex) { return sqlite3_column_text(_stmt, colIndex); } -void SqliteStatement::execQuery(MobileSession* session, const std::string& query) { +void SqliteStatement::_execQuery(sqlite3* session, const char* query) { LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite sqlite3_exec: " << query; char* errMsg = NULL; - int status = sqlite3_exec(session->getSession(), query.c_str(), NULL, NULL, &errMsg); + int status = sqlite3_exec(session, query, NULL, NULL, &errMsg); if (status == SQLITE_BUSY || status == SQLITE_LOCKED) { LOG(MOBILE_TRACE_LEVEL) << "MobileSE: " << (status == SQLITE_BUSY ? "Busy" : "Locked") @@ -155,7 +160,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query } // The only return value from sqlite3_exec in a success case is SQLITE_OK. - checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); // When the error message is not NULL, it is allocated through sqlite3_malloc and must be freed // before exiting the method. If the error message is NULL, sqlite3_free is a no-op. @@ -164,7 +169,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query void SqliteStatement::reset() { int status = sqlite3_reset(_stmt); - checkStatus(status, SQLITE_OK, "sqlite3_reset"); + embedded::checkStatus(status, SQLITE_OK, "sqlite3_reset"); } } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h index 6a527d947f2..51a6c8a61e2 100644 --- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h +++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h @@ -35,6 +35,8 @@ #include "mongo/db/storage/mobile/mobile_session.h" #include "mongo/platform/atomic_word.h" +#include <boost/container/small_vector.hpp> + namespace mongo { /** @@ -46,7 +48,8 @@ public: /** * Creates and prepares a SQLite statement. */ - SqliteStatement(const MobileSession& session, const std::string& sqlQuery); + template <class... Args> + SqliteStatement(const MobileSession& session, Args&&... args); /** * Finalizes the prepared statement. @@ -115,7 +118,29 @@ public: * None of the rows retrieved, if any, are saved before the query is finalized. Thus, this * method should not be used for read operations. */ - static void execQuery(MobileSession* session, const std::string& query); + template <class... Args> + static void execQuery(sqlite3* session, const char* first, Args&&... args) { + // If we just have a single char*, we're good to go, no need to build a new string. + constexpr std::size_t num = sizeof...(args); + if (!num) { + _execQuery(session, first); + } else { + _execQueryBuilder(session, first, std::forward<Args>(args)...); + } + } + template <class... Args> + static void execQuery(sqlite3* session, const std::string& first, Args&&... args) { + execQuery(session, first.c_str(), std::forward<Args>(args)...); + } + template <class... Args> + static void execQuery(sqlite3* session, StringData first, Args&&... args) { + // StringData may not be null terminated, so build a new string + _execQueryBuilder(session, first, std::forward<Args>(args)...); + } + template <class... Args> + static void execQuery(MobileSession* session, Args&&... args) { + execQuery(session->getSession(), std::forward<Args>(args)...); + } /** * Finalizes a prepared statement. @@ -130,13 +155,81 @@ public: uint64_t _id; private: + static void _execQuery(sqlite3* session, const char* query); + template <class... Args> + static void _execQueryBuilder(sqlite3* session, Args&&... args); + static AtomicWord<long long> _nextID; sqlite3_stmt* _stmt; - std::string _sqlQuery; // If the most recent call to sqlite3_step on this statement returned an error, the error is // returned again when the statement is finalized. This is used to verify that the last error // code returned matches the finalize error code, if there is any. int _exceptionStatus = SQLITE_OK; + + // Static memory that fits short SQL statements to avoid a temporary memory allocation + static constexpr size_t kMaxFixedSize = 96; + using SqlQuery_t = boost::container::small_vector<char, kMaxFixedSize>; + SqlQuery_t _sqlQuery; + + template <std::size_t N> + static void queryAppend(SqlQuery_t& dest, char const (&str)[N], std::true_type) { + dest.insert(dest.end(), str, str + N - 1); + } + static void queryAppend(SqlQuery_t& dest, StringData sd, std::false_type) { + dest.insert(dest.end(), sd.begin(), sd.end()); + } }; + +namespace detail { +// Most of the strings we build statements with are static strings so we can calculate their length +// during compile time. +// Arrays decay to pointer in overload resolution, force that to not happen by providing true_type +// as second argument if array +template <std::size_t N> +constexpr std::size_t stringLength(char const (&)[N], std::true_type) { + // Omit the null terminator, will added back when we call reserve later + return N - 1; +} +inline std::size_t stringLength(StringData sd, std::false_type) { + return sd.size(); +} + +} // namespace detail + +template <class... Args> +SqliteStatement::SqliteStatement(const MobileSession& session, Args&&... args) { + // Increment the global instance count and assign this instance an id. + _id = _nextID.addAndFetch(1); + + // Reserve the size we need once to avoid any additional allocations + _sqlQuery.reserve((detail::stringLength(std::forward<Args>(args), + std::is_array<std::remove_reference_t<Args>>()) + + ...) + + 1); + + // Copy all substrings into buffer for SQL statement + (queryAppend( + _sqlQuery, std::forward<Args>(args), std::is_array<std::remove_reference_t<Args>>()), + ...); + _sqlQuery.push_back('\0'); + + prepare(session); +} + +template <class... Args> +void SqliteStatement::_execQueryBuilder(sqlite3* session, Args&&... args) { + SqlQuery_t sqlQuery; + + sqlQuery.reserve((detail::stringLength(std::forward<Args>(args), + std::is_array<std::remove_reference_t<Args>>()) + + ...) + + 1); + (queryAppend( + sqlQuery, std::forward<Args>(args), std::is_array<std::remove_reference_t<Args>>()), + ...); + sqlQuery.push_back('\0'); + _execQuery(session, sqlQuery.data()); +} + } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.cpp b/src/mongo/db/storage/mobile/mobile_util.cpp index 24d903baae7..79111401972 100644 --- a/src/mongo/db/storage/mobile/mobile_util.cpp +++ b/src/mongo/db/storage/mobile/mobile_util.cpp @@ -34,11 +34,13 @@ #include <sqlite3.h> +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/mobile/mobile_recovery_unit.h" #include "mongo/db/storage/mobile/mobile_sqlite_statement.h" #include "mongo/db/storage/mobile/mobile_util.h" namespace mongo { +namespace embedded { using std::string; @@ -151,9 +153,8 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM void doValidate(OperationContext* opCtx, ValidateResults* results) { MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx); - std::string validateQuery = "PRAGMA integrity_check;"; try { - SqliteStatement validateStmt(*session, validateQuery); + SqliteStatement validateStmt(*session, "PRAGMA integrity_check;"); int status; // By default, the integrity check returns the first 100 errors found. @@ -181,4 +182,32 @@ void doValidate(OperationContext* opCtx, ValidateResults* results) { } } +void configureSession(sqlite3* session) { + auto executePragma = [session](auto pragma, auto value) { + SqliteStatement::execQuery(session, "PRAGMA ", pragma, " = ", value, ";"); + LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE session configuration: " << pragma << " = " << value; + }; + // Set SQLite in Write-Ahead Logging mode. https://sqlite.org/wal.html + executePragma("journal_mode"_sd, "WAL"_sd); + + // synchronous = NORMAL(1) is recommended with WAL, but we allow it to be overriden + executePragma("synchronous"_sd, std::to_string(mobileGlobalOptions.mobileDurabilityLevel)); + + // Set full fsync on OSX (only supported there) to ensure durability + executePragma("fullfsync"_sd, "1"_sd); + + // We don't manually use VACUUM so set incremental mode to reclaim space + executePragma("auto_vacuum"_sd, "incremental"_sd); + + // We just use SQLite as key-value store, so disable foreign keys + executePragma("foreign_keys"_sd, "0"_sd); + + // Set some additional internal sizes for this session + executePragma("cache_size"_sd, std::to_string(mobileGlobalOptions.mobileCacheSizeKB)); + executePragma("mmap_size"_sd, std::to_string(mobileGlobalOptions.mobileMmapSizeKB * 1024)); + executePragma("journal_size_limit"_sd, + std::to_string(mobileGlobalOptions.mobileJournalSizeLimitKB * 1024)); +} + +} // namespace embedded } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_util.h b/src/mongo/db/storage/mobile/mobile_util.h index 1a10ba666d1..d96e32a076c 100644 --- a/src/mongo/db/storage/mobile/mobile_util.h +++ b/src/mongo/db/storage/mobile/mobile_util.h @@ -38,6 +38,7 @@ #define MOBILE_TRACE_LEVEL MOBILE_LOG_LEVEL_HIGH namespace mongo { +namespace embedded { /** * Converts SQLite return codes to MongoDB statuses. @@ -64,4 +65,11 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM */ void doValidate(OperationContext* opCtx, ValidateResults* results); +/** + * Sets the SQLite Pragmas that we want (https://www.sqlite.org/pragma.html) + * These should generally improve behavior, performance, and resource usage + */ +void configureSession(sqlite3* session); + +} // namespace embedded } // namespace mongo |