summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHenrik Edin <henrik.edin@mongodb.com>2019-03-26 10:27:48 -0400
committerHenrik Edin <henrik.edin@mongodb.com>2019-04-10 13:19:54 -0400
commit3873c080388d5148afde90d7e02b2c4f5a297867 (patch)
treebb414dce2e816abba56045a8f9d60987fdb01a40
parent73536c31314daef6c68217aed5f8d6ddd432d15b (diff)
downloadmongo-3873c080388d5148afde90d7e02b2c4f5a297867.tar.gz
SERVER-32709 Improve performance with the mobile storage engine.
SQLite is configured with synchronous=NORMAL for all sessions. Reduced temporary memory allocations when constructing SQL statements. Implemented waitUntilDurable when this storage engine is used in mongod only. (cherry picked from commit 2ee7dcb2bcb4372a49b8584c43bb65caea6212c8)
-rw-r--r--etc/evergreen.yml6
-rw-r--r--jstests/concurrency/fsm_workloads/reindex.js2
-rw-r--r--jstests/concurrency/fsm_workloads/reindex_background.js2
-rw-r--r--jstests/noPassthrough/indexbg1.js2
-rw-r--r--jstests/noPassthrough/indexbg2.js2
-rw-r--r--jstests/noPassthroughWithMongod/geo_polygon.js2
-rw-r--r--src/mongo/db/storage/mobile/mobile_global_options.h2
-rw-r--r--src/mongo/db/storage/mobile/mobile_index.cpp63
-rw-r--r--src/mongo/db/storage/mobile/mobile_kv_engine.cpp120
-rw-r--r--src/mongo/db/storage/mobile/mobile_kv_engine.h2
-rw-r--r--src/mongo/db/storage/mobile/mobile_record_store.cpp81
-rw-r--r--src/mongo/db/storage/mobile/mobile_recovery_unit.cpp39
-rw-r--r--src/mongo/db/storage/mobile/mobile_recovery_unit.h5
-rw-r--r--src/mongo/db/storage/mobile/mobile_session_pool.cpp5
-rw-r--r--src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp57
-rw-r--r--src/mongo/db/storage/mobile/mobile_sqlite_statement.h118
-rw-r--r--src/mongo/db/storage/mobile/mobile_util.cpp32
-rw-r--r--src/mongo/db/storage/mobile/mobile_util.h8
18 files changed, 345 insertions, 203 deletions
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 38b352d3948..1dd989d8014 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -11854,7 +11854,7 @@ buildvariants:
test_flags: >-
--storageEngine=mobile
--excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact
- --excludeWithAnyTags=SERVER-32709,SERVER-32869
+ --excludeWithAnyTags=SERVER-32869,SERVER-40561
compile_flags: >-
-j$(grep -c ^processor /proc/cpuinfo)
--mobile-se=on
@@ -11932,7 +11932,7 @@ buildvariants:
test_flags: >-
--storageEngine=mobile
--excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact
- --excludeWithAnyTags=SERVER-32709,SERVER-32869
+ --excludeWithAnyTags=SERVER-32869,SERVER-40561
compile_flags: >-
-j$(grep -c ^processor /proc/cpuinfo)
--mobile-se=on
@@ -12008,7 +12008,7 @@ buildvariants:
test_flags: >-
--storageEngine=mobile
--excludeWithAnyTags=requires_mmapv1,requires_wiredtiger,requires_replication,requires_sharding,uses_transactions,requires_capped,requires_profiling,requires_compact
- --excludeWithAnyTags=SERVER-32709,SERVER-32869
+ --excludeWithAnyTags=SERVER-32869,SERVER-40561
compile_env: DEVELOPER_DIR=/Applications/Xcode8.3.app
compile_flags: >-
-j$(sysctl -n hw.logicalcpu)
diff --git a/jstests/concurrency/fsm_workloads/reindex.js b/jstests/concurrency/fsm_workloads/reindex.js
index 7ca348c005b..0ec81ef1ff3 100644
--- a/jstests/concurrency/fsm_workloads/reindex.js
+++ b/jstests/concurrency/fsm_workloads/reindex.js
@@ -6,7 +6,7 @@
* Bulk inserts 1000 documents and builds indexes. Then alternates between reindexing and querying
* against the collection. Operates on a separate collection for each thread.
*
- * @tags: [SERVER-32709]
+ * @tags: [SERVER-40561]
*/
var $config = (function() {
diff --git a/jstests/concurrency/fsm_workloads/reindex_background.js b/jstests/concurrency/fsm_workloads/reindex_background.js
index 1e18e60922c..d3958fc8de1 100644
--- a/jstests/concurrency/fsm_workloads/reindex_background.js
+++ b/jstests/concurrency/fsm_workloads/reindex_background.js
@@ -8,7 +8,7 @@
* that because indexes are initially built in the background, reindexing is also done in the
* background.
*
- * @tags: [SERVER-32709]
+ * @tags: [SERVER-40561]
*/
load('jstests/concurrency/fsm_libs/extend_workload.js'); // for extendWorkload
diff --git a/jstests/noPassthrough/indexbg1.js b/jstests/noPassthrough/indexbg1.js
index 5b47bc43de2..0c6758c0262 100644
--- a/jstests/noPassthrough/indexbg1.js
+++ b/jstests/noPassthrough/indexbg1.js
@@ -1,5 +1,5 @@
// Test background index creation
-// @tags: [SERVER-32709]
+// @tags: [SERVER-40561]
(function() {
"use strict";
diff --git a/jstests/noPassthrough/indexbg2.js b/jstests/noPassthrough/indexbg2.js
index 3e570c9dd94..70dddac5e55 100644
--- a/jstests/noPassthrough/indexbg2.js
+++ b/jstests/noPassthrough/indexbg2.js
@@ -1,5 +1,5 @@
// Test background index creation w/ constraints
-// @tags: [SERVER-32709, requires_document_locking]
+// @tags: [SERVER-40561, requires_document_locking]
(function() {
"use strict";
diff --git a/jstests/noPassthroughWithMongod/geo_polygon.js b/jstests/noPassthroughWithMongod/geo_polygon.js
index 4ce1f8d2ad7..7c23442d4de 100644
--- a/jstests/noPassthroughWithMongod/geo_polygon.js
+++ b/jstests/noPassthroughWithMongod/geo_polygon.js
@@ -1,5 +1,5 @@
//
-// @tags: [SERVER-32709]
+// @tags: [SERVER-40561]
//
t = db.geo_polygon4;
diff --git a/src/mongo/db/storage/mobile/mobile_global_options.h b/src/mongo/db/storage/mobile/mobile_global_options.h
index 16277c6fcc1..44a495f7d70 100644
--- a/src/mongo/db/storage/mobile/mobile_global_options.h
+++ b/src/mongo/db/storage/mobile/mobile_global_options.h
@@ -44,7 +44,7 @@ public:
// This is used by the Mobile SE and allows users to set the value
// passed to SQLite's PRAGMA synchronous command
- std::int32_t mobileDurabilityLevel;
+ std::uint32_t mobileDurabilityLevel;
};
extern MobileGlobalOptions mobileGlobalOptions;
diff --git a/src/mongo/db/storage/mobile/mobile_index.cpp b/src/mongo/db/storage/mobile/mobile_index.cpp
index 4860e524d24..cdfef05abd5 100644
--- a/src/mongo/db/storage/mobile/mobile_index.cpp
+++ b/src/mongo/db/storage/mobile/mobile_index.cpp
@@ -105,8 +105,8 @@ Status MobileIndex::doInsert(OperationContext* opCtx,
session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx);
}
- std::string insertQuery = "INSERT INTO \"" + _ident + "\" (key, value) VALUES (?, ?);";
- SqliteStatement insertStmt(*session, insertQuery);
+ SqliteStatement insertStmt(
+ *session, "INSERT INTO \"", _ident, "\" (key, value) VALUES (?, ?);");
insertStmt.bindBlob(0, key.getBuffer(), key.getSize());
insertStmt.bindBlob(1, value.getBuffer(), value.getSize());
@@ -126,7 +126,7 @@ Status MobileIndex::doInsert(OperationContext* opCtx,
return Status::OK();
}
}
- checkStatus(status, SQLITE_DONE, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step");
return Status::OK();
}
@@ -144,13 +144,8 @@ void MobileIndex::unindex(OperationContext* opCtx,
void MobileIndex::_doDelete(OperationContext* opCtx, const KeyString& key, KeyString* value) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false);
- str::stream deleteQuery;
- deleteQuery << "DELETE FROM \"" << _ident << "\" WHERE key = ?";
- if (value) {
- deleteQuery << " AND value = ?";
- }
- deleteQuery << ";";
- SqliteStatement deleteStmt(*session, deleteQuery);
+ SqliteStatement deleteStmt(
+ *session, "DELETE FROM \"", _ident, "\" WHERE key = ?", value ? " AND value = ?" : "", ";");
deleteStmt.bindBlob(0, key.getBuffer(), key.getSize());
if (value) {
@@ -166,7 +161,7 @@ void MobileIndex::fullValidate(OperationContext* opCtx,
long long* numKeysOut,
ValidateResults* fullResults) const {
if (fullResults) {
- doValidate(opCtx, fullResults);
+ embedded::doValidate(opCtx, fullResults);
if (!fullResults->valid) {
return;
}
@@ -188,10 +183,11 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const {
// Sum the number of bytes in each column.
// SQLite aggregate functions return null if the column is empty or has only nulls, so return 0
// bytes if there is no data in the column.
- str::stream sizeQuery;
- sizeQuery << "SELECT IFNULL(SUM(LENGTH(key)), 0) + "
- << "IFNULL(SUM(LENGTH(value)), 0) FROM \"" << _ident + "\";";
- SqliteStatement sizeStmt(*session, sizeQuery);
+ SqliteStatement sizeStmt(*session,
+ "SELECT IFNULL(SUM(LENGTH(key)), 0) + ",
+ "IFNULL(SUM(LENGTH(value)), 0) FROM \"",
+ _ident,
+ "\";");
sizeStmt.step(SQLITE_ROW);
@@ -201,8 +197,8 @@ long long MobileIndex::getSpaceUsedBytes(OperationContext* opCtx) const {
long long MobileIndex::numEntries(OperationContext* opCtx) const {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string countQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";";
- SqliteStatement countStmt(*session, countQuery);
+
+ SqliteStatement countStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";");
countStmt.step(SQLITE_ROW);
long long numRecs = countStmt.getColInt(0);
@@ -211,14 +207,14 @@ long long MobileIndex::numEntries(OperationContext* opCtx) const {
bool MobileIndex::isEmpty(OperationContext* opCtx) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string emptyCheckQuery = "SELECT * FROM \"" + _ident + "\" LIMIT 1;";
- SqliteStatement emptyCheckStmt(*session, emptyCheckQuery);
+
+ SqliteStatement emptyCheckStmt(*session, "SELECT * FROM \"", _ident, "\" LIMIT 1;");
int status = emptyCheckStmt.step();
if (status == SQLITE_DONE) {
return true;
}
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
return false;
}
@@ -229,9 +225,9 @@ Status MobileIndex::initAsEmpty(OperationContext* opCtx) {
Status MobileIndex::create(OperationContext* opCtx, const std::string& ident) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx);
- std::string createTableQuery =
- "CREATE TABLE \"" + ident + "\"(key BLOB PRIMARY KEY, value BLOB);";
- SqliteStatement createTableStmt(*session, createTableQuery.c_str());
+
+ SqliteStatement createTableStmt(
+ *session, "CREATE TABLE \"", ident, "\"(key BLOB PRIMARY KEY, value BLOB);");
createTableStmt.step(SQLITE_DONE);
return Status::OK();
@@ -250,8 +246,7 @@ Status MobileIndex::dupKeyCheck(OperationContext* opCtx,
bool MobileIndex::_isDup(OperationContext* opCtx, const BSONObj& key, RecordId recId) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string dupCheckQuery = "SELECT value FROM \"" + _ident + "\" WHERE key = ?;";
- SqliteStatement dupCheckStmt(*session, dupCheckQuery);
+ SqliteStatement dupCheckStmt(*session, "SELECT value FROM \"", _ident, "\" WHERE key = ?;");
KeyString keyStr(_keyStringVersion, key, _ordering);
@@ -273,7 +268,7 @@ bool MobileIndex::_isDup(OperationContext* opCtx, const BSONObj& key, RecordId r
return false;
}
}
- checkStatus(status, SQLITE_DONE, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step");
return isEntryFound;
}
@@ -401,11 +396,15 @@ public:
_savedTypeBits(index.getKeyStringVersion()),
_startPosition(index.getKeyStringVersion()) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- str::stream cursorQuery;
- cursorQuery << "SELECT key, value FROM \"" << _index.getIdent() << "\" WHERE key ";
- cursorQuery << (_isForward ? ">=" : "<=") << " ? ORDER BY key ";
- cursorQuery << (_isForward ? "ASC" : "DESC") << ";";
- _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery);
+
+ _stmt = stdx::make_unique<SqliteStatement>(*session,
+ "SELECT key, value FROM \"",
+ _index.getIdent(),
+ "\" WHERE key ",
+ (_isForward ? ">=" : "<="),
+ " ? ORDER BY key ",
+ (_isForward ? "ASC" : "DESC"),
+ ";");
}
virtual ~CursorBase() {}
@@ -515,7 +514,7 @@ protected:
_isEOF = true;
return;
}
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
_isEOF = false;
diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
index ef9637ee21f..d68e48d81c3 100644
--- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
+++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp
@@ -56,126 +56,80 @@ namespace mongo {
class MobileSession;
class SqliteStatement;
-MobileKVEngine::MobileKVEngine(const std::string& path, std::int32_t durabilityLevel) {
+MobileKVEngine::MobileKVEngine(const std::string& path, std::uint32_t durabilityLevel) {
_initDBPath(path);
// Initialize the database to be in WAL mode.
sqlite3* initSession;
int status = sqlite3_open(_path.c_str(), &initSession);
- checkStatus(status, SQLITE_OK, "sqlite3_open");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_open");
// Guarantees that sqlite3_close() will be called when the function returns.
ON_BLOCK_EXIT([&initSession] { sqlite3_close(initSession); });
- // Ensure SQLite is operating in the WAL mode.
+ embedded::configureSession(initSession);
+
+ // Check and ensure that WAL mode is working as expected
+ // This is not something that we want to be configurable
{
sqlite3_stmt* stmt;
- status = sqlite3_prepare_v2(initSession, "PRAGMA journal_mode=WAL;", -1, &stmt, NULL);
- checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
+ status = sqlite3_prepare_v2(initSession, "PRAGMA journal_mode;", -1, &stmt, NULL);
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
status = sqlite3_step(stmt);
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
// Pragma returns current mode in SQLite, ensure it is "wal" mode.
const void* colText = sqlite3_column_text(stmt, 0);
const char* mode = reinterpret_cast<const char*>(colText);
fassert(37001, !strcmp(mode, "wal"));
status = sqlite3_finalize(stmt);
- checkStatus(status, SQLITE_OK, "sqlite3_finalize");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize");
LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database opened in WAL mode";
}
- // Let's set and enforce the synchronous mode
- // We allow the user to specify a value between 0 and 3
- // using the mobileDurabilityLevel option
+ // Check and ensure that synchronous mode is working as expected
{
- std::string synchronousPragma =
- "PRAGMA synchronous = " + std::to_string(durabilityLevel) + ";";
-
- char* errMsg = NULL;
- status = sqlite3_exec(initSession, synchronousPragma.c_str(), NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- sqlite3_free(errMsg);
-
sqlite3_stmt* stmt;
status = sqlite3_prepare_v2(initSession, "PRAGMA synchronous;", -1, &stmt, NULL);
- checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
status = sqlite3_step(stmt);
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
// Pragma returns current "synchronous" setting
- int sync_val = sqlite3_column_int(stmt, 0);
+ std::uint32_t sync_val = sqlite3_column_int(stmt, 0);
fassert(50869, sync_val == durabilityLevel);
status = sqlite3_finalize(stmt);
- checkStatus(status, SQLITE_OK, "sqlite3_finalize");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize");
LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database has synchronous "
- << "mode set to " << durabilityLevel;
+ << "set to: " << durabilityLevel;
}
- // Set and ensure SQLite is operating with F_FULLFSYNC on darwin kernels
+ // Check and ensure that we were able to set the F_FULLFSYNC fcntl on darwin kernels
+ // This prevents data corruption as fsync doesn't work as expected
+ // This is not something that we want to be configurable
{
- char* errMsg = NULL;
- status = sqlite3_exec(initSession, "PRAGMA fullfsync = 1;", NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- // When the error message is not NULL, it is allocated through sqlite3_malloc and must be
- // freed before exiting the method. If the error message is NULL, sqlite3_free is a no-op.
- sqlite3_free(errMsg);
-
- // Confirm that the setting holds
sqlite3_stmt* stmt;
status = sqlite3_prepare_v2(initSession, "PRAGMA fullfsync;", -1, &stmt, NULL);
- checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2");
status = sqlite3_step(stmt);
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
// Pragma returns current fullsync setting, ensure it is enabled.
int fullfsync_val = sqlite3_column_int(stmt, 0);
fassert(50868, fullfsync_val == 1);
status = sqlite3_finalize(stmt);
- checkStatus(status, SQLITE_OK, "sqlite3_finalize");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_finalize");
LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database is set to fsync "
<< "with F_FULLFSYNC if the platform supports it (currently"
<< " only darwin kernels). Value: " << fullfsync_val;
}
- // Let's set additional non-critical settings that should improve performance and resource usage
- {
- // Allow for periodic calls to purge deleted records and prune db size on disk
- // Still requires manual vacuum calls using `PRAGMA incremental_vacuum(N);`
- char* errMsg = NULL;
- status =
- sqlite3_exec(initSession, "PRAGMA auto_vacuum = incremental;", NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- sqlite3_free(errMsg);
-
- // Set the cache size to 10MiB
- // value is passed as "-<#ofKiB>"
- errMsg = NULL;
- status = sqlite3_exec(initSession, "PRAGMA cache_size = -10240;", NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- sqlite3_free(errMsg);
-
- // Enable memory mapping for the data and set the size at 50MiB
- errMsg = NULL;
- status = sqlite3_exec(initSession, "PRAGMA mmap_size = 52428800;", NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- sqlite3_free(errMsg);
-
- // Cap the WAL size at 5MiB so that it's regularly pruned
- errMsg = NULL;
- status =
- sqlite3_exec(initSession, "PRAGMA journal_size_limit = 5242880;", NULL, NULL, &errMsg);
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
- sqlite3_free(errMsg);
-
- LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Completed all SQLite database configuration";
- }
-
_sessionPool.reset(new MobileSessionPool(_path));
}
@@ -267,7 +221,7 @@ Status MobileKVEngine::dropIdent(OperationContext* opCtx, StringData ident) {
std::string dropQuery = "DROP TABLE IF EXISTS \"" + ident + "\";";
try {
- SqliteStatement::execQuery(session, dropQuery.c_str());
+ SqliteStatement::execQuery(session, dropQuery);
} catch (const WriteConflictException&) {
// It is possible that this drop fails because of transaction running in parallel.
// We pretend that it succeeded, queue it for now and keep retrying later.
@@ -287,8 +241,7 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident)
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
// Get key-value column names.
- std::string colNameQuery = "PRAGMA table_info(\"" + ident + "\")";
- SqliteStatement colNameStmt(*session, colNameQuery);
+ SqliteStatement colNameStmt(*session, "PRAGMA table_info(\"", ident, "\")");
colNameStmt.step(SQLITE_ROW);
std::string keyColName(static_cast<const char*>(colNameStmt.getColText(1)));
@@ -297,10 +250,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident)
colNameStmt.step(SQLITE_DONE);
// Get total data size of key-value columns.
- str::stream dataSizeQuery;
- dataSizeQuery << "SELECT IFNULL(SUM(LENGTH(" << keyColName << ")), 0) + "
- << "IFNULL(SUM(LENGTH(" << valueColName << ")), 0) FROM \"" << ident + "\";";
- SqliteStatement dataSizeStmt(*session, dataSizeQuery);
+ SqliteStatement dataSizeStmt(*session,
+ "SELECT IFNULL(SUM(LENGTH(",
+ keyColName,
+ ")), 0) + ",
+ "IFNULL(SUM(LENGTH(",
+ valueColName,
+ ")), 0) FROM \"",
+ ident,
+ "\";");
dataSizeStmt.step(SQLITE_ROW);
return dataSizeStmt.getColInt(0);
@@ -309,15 +267,15 @@ int64_t MobileKVEngine::getIdentSize(OperationContext* opCtx, StringData ident)
bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string findTableQuery = "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;";
- SqliteStatement findTableStmt(*session, findTableQuery);
+ SqliteStatement findTableStmt(*session,
+ "SELECT * FROM sqlite_master WHERE type='table' AND name = ?;");
findTableStmt.bindText(0, ident.rawData(), ident.size());
int status = findTableStmt.step();
if (status == SQLITE_DONE) {
return false;
}
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
return true;
}
@@ -325,15 +283,15 @@ bool MobileKVEngine::hasIdent(OperationContext* opCtx, StringData ident) const {
std::vector<std::string> MobileKVEngine::getAllIdents(OperationContext* opCtx) const {
std::vector<std::string> idents;
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string getTablesQuery = "SELECT name FROM sqlite_master WHERE type='table';";
- SqliteStatement getTablesStmt(*session, getTablesQuery);
+
+ SqliteStatement getTablesStmt(*session, "SELECT name FROM sqlite_master WHERE type='table';");
int status;
while ((status = getTablesStmt.step()) == SQLITE_ROW) {
std::string tableName(reinterpret_cast<const char*>(getTablesStmt.getColText(0)));
idents.push_back(tableName);
}
- checkStatus(status, SQLITE_DONE, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step");
return idents;
}
diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h
index 17b34d8c308..d8f4d2b3013 100644
--- a/src/mongo/db/storage/mobile/mobile_kv_engine.h
+++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h
@@ -44,7 +44,7 @@ class JournalListener;
class MobileKVEngine : public KVEngine {
public:
- MobileKVEngine(const std::string& path, std::int32_t durabilityLevel);
+ MobileKVEngine(const std::string& path, std::uint32_t durabilityLevel);
RecoveryUnit* newRecoveryUnit() override;
diff --git a/src/mongo/db/storage/mobile/mobile_record_store.cpp b/src/mongo/db/storage/mobile/mobile_record_store.cpp
index b5ce45ac107..fa20782ef45 100644
--- a/src/mongo/db/storage/mobile/mobile_record_store.cpp
+++ b/src/mongo/db/storage/mobile/mobile_record_store.cpp
@@ -63,13 +63,17 @@ public:
bool forward)
: _opCtx(opCtx), _forward(forward) {
- str::stream cursorQuery;
- cursorQuery << "SELECT rec_id, data from \"" << ident << "\" "
- << "WHERE rec_id " << (forward ? '>' : '<') << " ? "
- << "ORDER BY rec_id " << (forward ? "ASC" : "DESC") << ';';
-
MobileSession* session = MobileRecoveryUnit::get(_opCtx)->getSession(_opCtx);
- _stmt = stdx::make_unique<SqliteStatement>(*session, cursorQuery);
+ _stmt = stdx::make_unique<SqliteStatement>(*session,
+ "SELECT rec_id, data from \"",
+ ident,
+ "\" ",
+ "WHERE rec_id ",
+ (forward ? ">" : "<"),
+ " ? ",
+ "ORDER BY rec_id ",
+ (forward ? "ASC" : "DESC"),
+ ";");
_startIdNum = (forward ? RecordId::min().repr() : RecordId::max().repr());
_savedId = RecordId(_startIdNum);
@@ -91,7 +95,7 @@ public:
}
// Checks no error was thrown and that step retrieved a row.
- checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next");
+ embedded::checkStatus(status, SQLITE_ROW, "_stmt->step() in MobileCursor's next");
long long recId = _stmt->getColInt(0);
const void* data = _stmt->getColBlob(1);
@@ -192,8 +196,7 @@ MobileRecordStore::MobileRecordStore(OperationContext* opCtx,
// Determines the nextId to be used for a new record.
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string maxRecIdQuery = "SELECT IFNULL(MAX(rec_id), 0) FROM \"" + _ident + "\";";
- SqliteStatement maxRecIdStmt(*session, maxRecIdQuery);
+ SqliteStatement maxRecIdStmt(*session, "SELECT IFNULL(MAX(rec_id), 0) FROM \"", _ident, "\";");
maxRecIdStmt.step(SQLITE_ROW);
@@ -215,8 +218,8 @@ void MobileRecordStore::_initDataSizeIfNeeded_inlock(OperationContext* opCtx) co
}
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string dataSizeQuery = "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"" + _ident + "\";";
- SqliteStatement dataSizeStmt(*session, dataSizeQuery);
+ SqliteStatement dataSizeStmt(
+ *session, "SELECT IFNULL(SUM(LENGTH(data)), 0) FROM \"", _ident, "\";");
dataSizeStmt.step(SQLITE_ROW);
int64_t dataSize = dataSizeStmt.getColInt(0);
@@ -237,8 +240,7 @@ void MobileRecordStore::_initNumRecsIfNeeded_inlock(OperationContext* opCtx) con
}
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string numRecordsQuery = "SELECT COUNT(*) FROM \"" + _ident + "\";";
- SqliteStatement numRecordsStmt(*session, numRecordsQuery);
+ SqliteStatement numRecordsStmt(*session, "SELECT COUNT(*) FROM \"", _ident, "\";");
numRecordsStmt.step(SQLITE_ROW);
@@ -265,8 +267,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx,
const RecordId& recId,
RecordData* rd) const {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string sqlQuery = "SELECT data FROM \"" + _ident + "\" WHERE rec_id = ?;";
- SqliteStatement stmt(*session, sqlQuery);
+ SqliteStatement stmt(*session, "SELECT data FROM \"", _ident, "\" WHERE rec_id = ?;");
stmt.bindInt(0, recId.repr());
@@ -274,7 +275,7 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx,
if (status == SQLITE_DONE) {
return false;
}
- checkStatus(status, SQLITE_ROW, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_ROW, "sqlite3_step");
const void* recData = stmt.getColBlob(0);
int nBytes = stmt.getColBytes(0);
@@ -284,9 +285,9 @@ bool MobileRecordStore::findRecord(OperationContext* opCtx,
void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& recId) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false);
- std::string dataSizeQuery =
- "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;";
- SqliteStatement dataSizeStmt(*session, dataSizeQuery);
+
+ SqliteStatement dataSizeStmt(
+ *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;");
dataSizeStmt.bindInt(0, recId.repr());
dataSizeStmt.step(SQLITE_ROW);
@@ -294,8 +295,7 @@ void MobileRecordStore::deleteRecord(OperationContext* opCtx, const RecordId& re
_changeNumRecs(opCtx, -1);
_changeDataSize(opCtx, -dataSizeBefore);
- std::string deleteQuery = "DELETE FROM \"" + _ident + "\" WHERE rec_id = ?;";
- SqliteStatement deleteStmt(*session, deleteQuery);
+ SqliteStatement deleteStmt(*session, "DELETE FROM \"", _ident, "\" WHERE rec_id = ?;");
deleteStmt.bindInt(0, recId.repr());
deleteStmt.step(SQLITE_DONE);
}
@@ -308,9 +308,8 @@ StatusWith<RecordId> MobileRecordStore::insertRecord(
_changeNumRecs(opCtx, 1);
_changeDataSize(opCtx, len);
- std::string insertQuery =
- "INSERT OR REPLACE INTO \"" + _ident + "\"(rec_id, data) VALUES(?, ?);";
- SqliteStatement insertStmt(*session, insertQuery);
+ SqliteStatement insertStmt(
+ *session, "INSERT OR REPLACE INTO \"", _ident, "\"(rec_id, data) VALUES(?, ?);");
RecordId recId = _nextId();
insertStmt.bindInt(0, recId.repr());
insertStmt.bindBlob(1, data, len);
@@ -350,9 +349,9 @@ Status MobileRecordStore::updateRecord(OperationContext* opCtx,
bool enforceQuota,
UpdateNotifier* notifier) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx, false);
- std::string dataSizeQuery =
- "SELECT IFNULL(LENGTH(data), 0) FROM \"" + _ident + "\" WHERE rec_id = ?;";
- SqliteStatement dataSizeStmt(*session, dataSizeQuery);
+
+ SqliteStatement dataSizeStmt(
+ *session, "SELECT IFNULL(LENGTH(data), 0) FROM \"", _ident, "\" WHERE rec_id = ?;");
dataSizeStmt.bindInt(0, recId.repr());
dataSizeStmt.step(SQLITE_ROW);
@@ -363,8 +362,7 @@ Status MobileRecordStore::updateRecord(OperationContext* opCtx,
fassert(37054, notifier->recordStoreGoingToUpdateInPlace(opCtx, recId));
}
- std::string updateQuery = "UPDATE \"" + _ident + "\" SET data = ? " + "WHERE rec_id = ?;";
- SqliteStatement updateStmt(*session, updateQuery);
+ SqliteStatement updateStmt(*session, "UPDATE \"", _ident, "\" SET data = ? WHERE rec_id = ?;");
updateStmt.bindBlob(0, data, len);
updateStmt.bindInt(1, recId.repr());
updateStmt.step(SQLITE_DONE);
@@ -403,8 +401,7 @@ Status MobileRecordStore::truncate(OperationContext* opCtx) {
int64_t dataSizeBefore = dataSize(opCtx);
_changeDataSize(opCtx, -dataSizeBefore);
- std::string deleteForTruncateQuery = "DELETE FROM \"" + _ident + "\";";
- SqliteStatement::execQuery(session, deleteForTruncateQuery);
+ SqliteStatement::execQuery(session, "DELETE FROM \"", _ident, "\";");
return Status::OK();
}
@@ -419,7 +416,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
ValidateResults* results,
BSONObjBuilder* output) {
if (level == kValidateFull) {
- doValidate(opCtx, results);
+ embedded::doValidate(opCtx, results);
}
if (!results->valid) {
@@ -429,8 +426,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
try {
- std::string selectQuery = "SELECT rec_id, data FROM \"" + _ident + "\";";
- SqliteStatement selectStmt(*session, selectQuery);
+ SqliteStatement selectStmt(*session, "SELECT rec_id, data FROM \"", _ident, "\";");
int interruptInterval = 4096;
long long actualNumRecs = 0;
@@ -459,7 +455,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
if (!status.isOK() || validatedSize != static_cast<size_t>(dataSize)) {
if (results->valid) {
std::string errMsg = "detected one or more invalid documents";
- validateLogAndAppendError(results, errMsg);
+ embedded::validateLogAndAppendError(results, errMsg);
}
++numInvalidRecs;
@@ -470,7 +466,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
if (status == SQLITE_CORRUPT) {
uasserted(ErrorCodes::UnknownError, sqlite3_errstr(status));
}
- checkStatus(status, SQLITE_DONE, "sqlite3_step");
+ embedded::checkStatus(status, SQLITE_DONE, "sqlite3_step");
// Verify that _numRecs and _dataSize are accurate.
int64_t cachedNumRecs = numRecords(opCtx);
@@ -479,7 +475,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
errMsg << "cached number of records does not match actual number of records - ";
errMsg << "cached number of records = " << cachedNumRecs << "; ";
errMsg << "actual number of records = " << actualNumRecs;
- validateLogAndAppendError(results, errMsg);
+ embedded::validateLogAndAppendError(results, errMsg);
}
int64_t cachedDataSize = dataSize(opCtx);
if (_resetDataSizeIfNeeded(opCtx, actualDataSize)) {
@@ -487,7 +483,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
errMsg << "cached data size does not match actual data size - ";
errMsg << "cached data size = " << cachedDataSize << "; ";
errMsg << "actual data size = " << actualDataSize;
- validateLogAndAppendError(results, errMsg);
+ embedded::validateLogAndAppendError(results, errMsg);
}
if (level == kValidateFull) {
@@ -497,7 +493,7 @@ Status MobileRecordStore::validate(OperationContext* opCtx,
} catch (const DBException& e) {
std::string errMsg = "record store is corrupt, could not read documents - " + e.toString();
- validateLogAndAppendError(results, errMsg);
+ embedded::validateLogAndAppendError(results, errMsg);
}
return Status::OK();
@@ -609,9 +605,10 @@ boost::optional<RecordId> MobileRecordStore::oplogStartHack(
*/
void MobileRecordStore::create(OperationContext* opCtx, const std::string& ident) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSessionNoTxn(opCtx);
- std::string sqlQuery =
- "CREATE TABLE IF NOT EXISTS \"" + ident + "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));";
- SqliteStatement::execQuery(session, sqlQuery);
+ SqliteStatement::execQuery(session,
+ "CREATE TABLE IF NOT EXISTS \"",
+ ident,
+ "\"(rec_id INT, data BLOB, PRIMARY KEY(rec_id));");
}
} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp
index cc169d956d8..c4c9347bd9e 100644
--- a/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp
+++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.cpp
@@ -34,8 +34,10 @@
#include <string>
+#include "mongo/db/concurrency/d_concurrency.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/operation_context.h"
+#include "mongo/db/storage/mobile/mobile_global_options.h"
#include "mongo/db/storage/mobile/mobile_recovery_unit.h"
#include "mongo/db/storage/mobile/mobile_sqlite_statement.h"
#include "mongo/db/storage/mobile/mobile_util.h"
@@ -127,6 +129,41 @@ void MobileRecoveryUnit::abortUnitOfWork() {
_abort();
}
+bool MobileRecoveryUnit::waitUntilDurable() {
+ // This is going to be slow as we're taking a global X lock and doing a full checkpoint. This
+ // should not be needed to do on Android or iOS if we are on WAL and synchronous=NORMAL which
+ // are our default settings. The system will make sure any non-flushed writes will not be lost
+ // before going down but our powercycle test bench require it. Therefore make sure embedded does
+ // not call this (by disabling writeConcern j:true) but allow it when this is used inside
+ // mongod.
+ if (mobileGlobalOptions.mobileDurabilityLevel < 2) {
+ OperationContext* opCtx = Client::getCurrent()->getOperationContext();
+ _ensureSession(opCtx);
+ RECOVERY_UNIT_TRACE() << "waitUntilDurable called, attempting to perform a checkpoint";
+ int framesInWAL = 0;
+ int checkpointedFrames = 0;
+ int ret;
+ {
+ Lock::GlobalLock lk(opCtx, MODE_X);
+ // Use FULL mode to guarantee durability
+ ret = sqlite3_wal_checkpoint_v2(_session.get()->getSession(),
+ NULL,
+ SQLITE_CHECKPOINT_FULL,
+ &framesInWAL,
+ &checkpointedFrames);
+ }
+ embedded::checkStatus(ret, SQLITE_OK, "sqlite3_wal_checkpoint_v2");
+ fassert(51164,
+ framesInWAL != -1 && checkpointedFrames != -1 && framesInWAL == checkpointedFrames);
+ RECOVERY_UNIT_TRACE() << "Checkpointed " << checkpointedFrames << " of the " << framesInWAL
+ << " total frames in the WAL";
+ } else {
+ RECOVERY_UNIT_TRACE() << "No checkpoint attempted -- in full synchronous mode";
+ }
+
+ return true;
+}
+
void MobileRecoveryUnit::abandonSnapshot() {
invariant(!_inUnitOfWork);
if (_active) {
@@ -223,4 +260,4 @@ void MobileRecoveryUnit::_txnClose(bool commit) {
void MobileRecoveryUnit::enqueueFailedDrop(std::string& dropQuery) {
_sessionPool->failedDropsQueue.enqueueOp(dropQuery);
}
-}
+} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_recovery_unit.h b/src/mongo/db/storage/mobile/mobile_recovery_unit.h
index 000f01a971c..e65c3681b82 100644
--- a/src/mongo/db/storage/mobile/mobile_recovery_unit.h
+++ b/src/mongo/db/storage/mobile/mobile_recovery_unit.h
@@ -55,10 +55,7 @@ public:
void beginUnitOfWork(OperationContext* opCtx) override;
void commitUnitOfWork() override;
void abortUnitOfWork() override;
-
- bool waitUntilDurable() override {
- return true;
- }
+ bool waitUntilDurable() override;
void abandonSnapshot() override;
diff --git a/src/mongo/db/storage/mobile/mobile_session_pool.cpp b/src/mongo/db/storage/mobile/mobile_session_pool.cpp
index f09faf9e1ae..a289c4ebc52 100644
--- a/src/mongo/db/storage/mobile/mobile_session_pool.cpp
+++ b/src/mongo/db/storage/mobile/mobile_session_pool.cpp
@@ -120,7 +120,8 @@ std::unique_ptr<MobileSession> MobileSessionPool::getSession(OperationContext* o
if (_curPoolSize < _maxPoolSize) {
sqlite3* session;
int status = sqlite3_open(_path.c_str(), &session);
- checkStatus(status, SQLITE_OK, "sqlite3_open");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_open");
+ embedded::configureSession(session);
_curPoolSize++;
return stdx::make_unique<MobileSession>(session, this);
}
@@ -167,7 +168,7 @@ void MobileSessionPool::shutDown() {
sqlite3* session;
int status = sqlite3_open(_path.c_str(), &session);
- checkStatus(status, SQLITE_OK, "sqlite3_open");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_open");
std::unique_ptr<MobileSession> mobSession = stdx::make_unique<MobileSession>(session, this);
LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Executing queued drops at shutdown";
failedDropsQueue.execAndDequeueAllOps(mobSession.get());
diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp
index 0b26d25e1e7..ec07b5695a9 100644
--- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp
+++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.cpp
@@ -43,24 +43,20 @@
#include "mongo/util/scopeguard.h"
#define SQLITE_STMT_TRACE() LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite Stmt ID:" << _id << " "
+#define SQLITE_STMT_TRACE_ENABLED() \
+ (::mongo::logger::globalLogDomain()->shouldLog( \
+ MongoLogDefaultComponent_component, \
+ ::mongo::LogstreamBuilder::severityCast(MOBILE_TRACE_LEVEL)))
namespace mongo {
AtomicInt64 SqliteStatement::_nextID(0);
-SqliteStatement::SqliteStatement(const MobileSession& session, const std::string& sqlQuery) {
- // Increment the global instance count and assign this instance an id.
- _id = _nextID.addAndFetch(1);
- _sqlQuery = sqlQuery;
-
- prepare(session);
-}
-
void SqliteStatement::finalize() {
if (!_stmt) {
return;
}
- SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery;
+ SQLITE_STMT_TRACE() << "Finalize: " << _sqlQuery.data();
int status = sqlite3_finalize(_stmt);
fassert(37053, status == _exceptionStatus);
@@ -68,16 +64,16 @@ void SqliteStatement::finalize() {
}
void SqliteStatement::prepare(const MobileSession& session) {
- SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery;
+ SQLITE_STMT_TRACE() << "Preparing: " << _sqlQuery.data();
- int status = sqlite3_prepare_v2(
- session.getSession(), _sqlQuery.c_str(), _sqlQuery.length() + 1, &_stmt, NULL);
+ int status =
+ sqlite3_prepare_v2(session.getSession(), _sqlQuery.data(), _sqlQuery.size(), &_stmt, NULL);
if (status == SQLITE_BUSY) {
SQLITE_STMT_TRACE() << "Throwing writeConflictException, "
- << "SQLITE_BUSY while preparing: " << _sqlQuery;
+ << "SQLITE_BUSY while preparing: " << _sqlQuery.data();
throw WriteConflictException();
} else if (status != SQLITE_OK) {
- SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery;
+ SQLITE_STMT_TRACE() << "Error while preparing: " << _sqlQuery.data();
std::string errMsg = "sqlite3_prepare_v2 failed: ";
errMsg += sqlite3_errstr(status);
uasserted(ErrorCodes::UnknownError, errMsg);
@@ -86,29 +82,35 @@ void SqliteStatement::prepare(const MobileSession& session) {
SqliteStatement::~SqliteStatement() {
finalize();
+
+ static_assert(
+ sizeof(SqliteStatement) ==
+ sizeof(std::aligned_storage_t<sizeof(SqliteStatement), alignof(SqliteStatement)>),
+ "expected size to be exactly its aligned storage size to not waste memory, "
+ "adjust kMaxFixedSize to make this true");
}
void SqliteStatement::bindInt(int paramIndex, int64_t intValue) {
// SQLite bind methods begin paramater indexes at 1 rather than 0.
int status = sqlite3_bind_int64(_stmt, paramIndex + 1, intValue);
- checkStatus(status, SQLITE_OK, "sqlite3_bind");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind");
}
void SqliteStatement::bindBlob(int paramIndex, const void* data, int len) {
// SQLite bind methods begin paramater indexes at 1 rather than 0.
int status = sqlite3_bind_blob(_stmt, paramIndex + 1, data, len, SQLITE_STATIC);
- checkStatus(status, SQLITE_OK, "sqlite3_bind");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind");
}
void SqliteStatement::bindText(int paramIndex, const char* data, int len) {
// SQLite bind methods begin paramater indexes at 1 rather than 0.
int status = sqlite3_bind_text(_stmt, paramIndex + 1, data, len, SQLITE_STATIC);
- checkStatus(status, SQLITE_OK, "sqlite3_bind");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_bind");
}
void SqliteStatement::clearBindings() {
int status = sqlite3_clear_bindings(_stmt);
- checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_clear_bindings");
}
int SqliteStatement::step(int desiredStatus) {
@@ -117,12 +119,15 @@ int SqliteStatement::step(int desiredStatus) {
// A non-negative desiredStatus indicates that checkStatus should assert that the returned
// status is equivalent to the desired status.
if (desiredStatus >= 0) {
- checkStatus(status, desiredStatus, "sqlite3_step");
+ embedded::checkStatus(status, desiredStatus, "sqlite3_step");
}
- char* full_stmt = sqlite3_expanded_sql(_stmt);
- SQLITE_STMT_TRACE() << sqliteStatusToStr(status) << " - on stepping: " << full_stmt;
- sqlite3_free(full_stmt);
+ if (SQLITE_STMT_TRACE_ENABLED()) {
+ char* full_stmt = sqlite3_expanded_sql(_stmt);
+ SQLITE_STMT_TRACE() << embedded::sqliteStatusToStr(status)
+ << " - on stepping: " << full_stmt;
+ sqlite3_free(full_stmt);
+ }
return status;
}
@@ -143,11 +148,11 @@ const void* SqliteStatement::getColText(int colIndex) {
return sqlite3_column_text(_stmt, colIndex);
}
-void SqliteStatement::execQuery(MobileSession* session, const std::string& query) {
+void SqliteStatement::_execQuery(sqlite3* session, const char* query) {
LOG(MOBILE_TRACE_LEVEL) << "MobileSE: SQLite sqlite3_exec: " << query;
char* errMsg = NULL;
- int status = sqlite3_exec(session->getSession(), query.c_str(), NULL, NULL, &errMsg);
+ int status = sqlite3_exec(session, query, NULL, NULL, &errMsg);
if (status == SQLITE_BUSY || status == SQLITE_LOCKED) {
LOG(MOBILE_TRACE_LEVEL) << "MobileSE: " << (status == SQLITE_BUSY ? "Busy" : "Locked")
@@ -156,7 +161,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query
}
// The only return value from sqlite3_exec in a success case is SQLITE_OK.
- checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg);
// When the error message is not NULL, it is allocated through sqlite3_malloc and must be freed
// before exiting the method. If the error message is NULL, sqlite3_free is a no-op.
@@ -165,7 +170,7 @@ void SqliteStatement::execQuery(MobileSession* session, const std::string& query
void SqliteStatement::reset() {
int status = sqlite3_reset(_stmt);
- checkStatus(status, SQLITE_OK, "sqlite3_reset");
+ embedded::checkStatus(status, SQLITE_OK, "sqlite3_reset");
}
} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h
index 719bc49a895..e24f62800dd 100644
--- a/src/mongo/db/storage/mobile/mobile_sqlite_statement.h
+++ b/src/mongo/db/storage/mobile/mobile_sqlite_statement.h
@@ -36,6 +36,8 @@
#include "mongo/db/storage/mobile/mobile_session.h"
#include "mongo/platform/atomic_word.h"
+#include <boost/container/small_vector.hpp>
+
namespace mongo {
/**
@@ -47,7 +49,8 @@ public:
/**
* Creates and prepares a SQLite statement.
*/
- SqliteStatement(const MobileSession& session, const std::string& sqlQuery);
+ template <class... Args>
+ SqliteStatement(const MobileSession& session, Args&&... args);
/**
* Finalizes the prepared statement.
@@ -116,7 +119,29 @@ public:
* None of the rows retrieved, if any, are saved before the query is finalized. Thus, this
* method should not be used for read operations.
*/
- static void execQuery(MobileSession* session, const std::string& query);
+ template <class... Args>
+ static void execQuery(sqlite3* session, const char* first, Args&&... args) {
+ // If we just have a single char*, we're good to go, no need to build a new string.
+ constexpr std::size_t num = sizeof...(args);
+ if (!num) {
+ _execQuery(session, first);
+ } else {
+ _execQueryBuilder(session, first, std::forward<Args>(args)...);
+ }
+ }
+ template <class... Args>
+ static void execQuery(sqlite3* session, const std::string& first, Args&&... args) {
+ execQuery(session, first.c_str(), std::forward<Args>(args)...);
+ }
+ template <class... Args>
+ static void execQuery(sqlite3* session, StringData first, Args&&... args) {
+ // StringData may not be null terminated, so build a new string
+ _execQueryBuilder(session, first, std::forward<Args>(args)...);
+ }
+ template <class... Args>
+ static void execQuery(MobileSession* session, Args&&... args) {
+ execQuery(session->getSession(), std::forward<Args>(args)...);
+ }
/**
* Finalizes a prepared statement.
@@ -131,13 +156,100 @@ public:
uint64_t _id;
private:
+ static void _execQuery(sqlite3* session, const char* query);
+ template <class... Args>
+ static void _execQueryBuilder(sqlite3* session, Args&&... args);
+
static AtomicInt64 _nextID;
sqlite3_stmt* _stmt;
- std::string _sqlQuery;
// If the most recent call to sqlite3_step on this statement returned an error, the error is
// returned again when the statement is finalized. This is used to verify that the last error
// code returned matches the finalize error code, if there is any.
int _exceptionStatus = SQLITE_OK;
+
+ // Static memory that fits short SQL statements to avoid a temporary memory allocation
+ static constexpr size_t kMaxFixedSize = 96;
+ using SqlQuery_t = boost::container::small_vector<char, kMaxFixedSize>;
+ SqlQuery_t _sqlQuery;
+
+ template <std::size_t N>
+ static void queryAppend(SqlQuery_t& dest, char const (&str)[N], std::true_type) {
+ dest.insert(dest.end(), str, str + N - 1);
+ }
+ static void queryAppend(SqlQuery_t& dest, StringData sd, std::false_type) {
+ dest.insert(dest.end(), sd.begin(), sd.end());
+ }
+
+ template <class Arg>
+ static void queryAppendAll(SqlQuery_t& dest, Arg&& arg) {
+ queryAppend(dest,
+ std::forward<Arg>(arg),
+ std::is_array<typename std::remove_reference<Arg>::type>());
+ }
+
+ template <class Arg, class... Args>
+ static void queryAppendAll(SqlQuery_t& dest, Arg&& first, Args&&... args) {
+ queryAppend(dest,
+ std::forward<Arg>(first),
+ std::is_array<typename std::remove_reference<Arg>::type>());
+ queryAppendAll(dest, std::forward<Args>(args)...);
+ }
};
+
+namespace detail {
+// Most of the strings we build statements with are static strings so we can calculate their length
+// during compile time.
+// Arrays decay to pointer in overload resolution, force that to not happen by providing true_type
+// as second argument if array
+template <std::size_t N>
+constexpr std::size_t stringLength(char const (&)[N], std::true_type) {
+ // Omit the null terminator, will added back when we call reserve later
+ return N - 1;
+}
+inline std::size_t stringLength(StringData sd, std::false_type) {
+ return sd.size();
+}
+
+template <class Arg>
+std::size_t totalStringLength(Arg&& arg) {
+ return stringLength(std::forward<Arg>(arg),
+ std::is_array<typename std::remove_reference<Arg>::type>());
+}
+
+template <class Arg, class... Args>
+std::size_t totalStringLength(Arg&& first, Args&&... args) {
+ return stringLength(std::forward<Arg>(first),
+ std::is_array<typename std::remove_reference<Arg>::type>()) +
+ totalStringLength(std::forward<Args>(args)...);
+}
+
+} // namespace detail
+
+template <class... Args>
+SqliteStatement::SqliteStatement(const MobileSession& session, Args&&... args) {
+ // Increment the global instance count and assign this instance an id.
+ _id = _nextID.addAndFetch(1);
+
+ // Reserve the size we need once to avoid any additional allocations
+ _sqlQuery.reserve(detail::totalStringLength(std::forward<Args>(args)...) + 1);
+
+ // Copy all substrings into buffer for SQL statement
+ queryAppendAll(_sqlQuery, std::forward<Args>(args)...);
+ _sqlQuery.push_back('\0');
+
+ prepare(session);
+}
+
+template <class... Args>
+void SqliteStatement::_execQueryBuilder(sqlite3* session, Args&&... args) {
+ SqlQuery_t sqlQuery;
+
+ sqlQuery.reserve(detail::totalStringLength(std::forward<Args>(args)...) + 1);
+
+ queryAppendAll(sqlQuery, std::forward<Args>(args)...);
+ sqlQuery.push_back('\0');
+ _execQuery(session, sqlQuery.data());
+}
+
} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_util.cpp b/src/mongo/db/storage/mobile/mobile_util.cpp
index be8a88b8e6c..aac9b8a39bf 100644
--- a/src/mongo/db/storage/mobile/mobile_util.cpp
+++ b/src/mongo/db/storage/mobile/mobile_util.cpp
@@ -35,11 +35,13 @@
#include <sqlite3.h>
+#include "mongo/db/storage/mobile/mobile_global_options.h"
#include "mongo/db/storage/mobile/mobile_recovery_unit.h"
#include "mongo/db/storage/mobile/mobile_sqlite_statement.h"
#include "mongo/db/storage/mobile/mobile_util.h"
namespace mongo {
+namespace embedded {
using std::string;
@@ -152,9 +154,8 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM
void doValidate(OperationContext* opCtx, ValidateResults* results) {
MobileSession* session = MobileRecoveryUnit::get(opCtx)->getSession(opCtx);
- std::string validateQuery = "PRAGMA integrity_check;";
try {
- SqliteStatement validateStmt(*session, validateQuery);
+ SqliteStatement validateStmt(*session, "PRAGMA integrity_check;");
int status;
// By default, the integrity check returns the first 100 errors found.
@@ -182,4 +183,31 @@ void doValidate(OperationContext* opCtx, ValidateResults* results) {
}
}
+void configureSession(sqlite3* session) {
+ auto executePragma = [session](auto pragma, auto value) {
+ SqliteStatement::execQuery(session, "PRAGMA ", pragma, " = ", value, ";");
+ LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE session configuration: " << pragma << " = " << value;
+ };
+ // We don't manually use VACUUM so set incremental mode to reclaim space
+ executePragma("auto_vacuum"_sd, "incremental"_sd);
+
+ // Set SQLite in Write-Ahead Logging mode. https://sqlite.org/wal.html
+ executePragma("journal_mode"_sd, "WAL"_sd);
+
+ // synchronous = NORMAL(1) is recommended with WAL, but we allow it to be overriden
+ executePragma("synchronous"_sd, std::to_string(mobileGlobalOptions.mobileDurabilityLevel));
+
+ // Set full fsync on OSX (only supported there) to ensure durability
+ executePragma("fullfsync"_sd, "1"_sd);
+
+ // We just use SQLite as key-value store, so disable foreign keys
+ executePragma("foreign_keys"_sd, "0"_sd);
+
+ // Set some additional internal sizes for this session
+ executePragma("cache_size"_sd, "-10240"_sd); // 10KB (passed as negative means KB)
+ executePragma("mmap_size"_sd, "52428800"_sd); // 50MB
+ executePragma("journal_size_limit"_sd, "5242880"_sd); // 5MB
+}
+
+} // namespace embedded
} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_util.h b/src/mongo/db/storage/mobile/mobile_util.h
index 3c7b18dde7d..9e9c429ef50 100644
--- a/src/mongo/db/storage/mobile/mobile_util.h
+++ b/src/mongo/db/storage/mobile/mobile_util.h
@@ -39,6 +39,7 @@
#define MOBILE_TRACE_LEVEL MOBILE_LOG_LEVEL_HIGH
namespace mongo {
+namespace embedded {
/**
* Converts SQLite return codes to MongoDB statuses.
@@ -65,4 +66,11 @@ void validateLogAndAppendError(ValidateResults* results, const std::string& errM
*/
void doValidate(OperationContext* opCtx, ValidateResults* results);
+/**
+ * Sets the SQLite Pragmas that we want (https://www.sqlite.org/pragma.html)
+ * These should generally improve behavior, performance, and resource usage
+ */
+void configureSession(sqlite3* session);
+
+} // namespace embedded
} // namespace mongo