diff options
author | Matt Lord <mattalord@gmail.com> | 2019-03-11 02:29:39 -0400 |
---|---|---|
committer | Andrew Morrow <acm@mongodb.com> | 2019-03-16 13:10:01 -0400 |
commit | 21d8ae0a1666446984886293b65069784eb117bd (patch) | |
tree | f8ed153662e8f9fc528f8c0e94fc46371ce228ad | |
parent | fb4667590b65c717789d81c088a08f54cbf10a6e (diff) | |
download | mongo-21d8ae0a1666446984886293b65069784eb117bd.tar.gz |
SERVER-39259 Mobile SE: Adjust SQLite Pragmas for Improved Performance
(cherry picked from commit 0cfc32589d5f6436e83d6d0d290849cf3798cbae)
-rw-r--r-- | src/mongo/db/storage/mobile/SConscript | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_global_options.cpp | 54 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_global_options.h | 52 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_global_options.idl | 52 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_init.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_kv_engine.cpp | 62 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_kv_engine.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_options_init.cpp | 53 | ||||
-rw-r--r-- | src/mongo/embedded/embedded_options.cpp | 7 |
10 files changed, 284 insertions, 17 deletions
diff --git a/src/mongo/db/storage/mobile/SConscript b/src/mongo/db/storage/mobile/SConscript index 5d627509ffd..b2d2b34e886 100644 --- a/src/mongo/db/storage/mobile/SConscript +++ b/src/mongo/db/storage/mobile/SConscript @@ -16,6 +16,8 @@ env.Library( 'mobile_session_pool.cpp', 'mobile_sqlite_statement.cpp', 'mobile_util.cpp', + 'mobile_global_options.cpp', + 'mobile_options_init.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/base', @@ -31,7 +33,10 @@ env.Library( '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', '$BUILD_DIR/mongo/db/storage/oplog_hack', '$BUILD_DIR/third_party/shim_sqlite', - ] + ], + LIBDEPS_PRIVATE= [ + '$BUILD_DIR/mongo/util/options_parser/options_parser', + ], ) serveronlyLibDepsDependents = [] diff --git a/src/mongo/db/storage/mobile/mobile_global_options.cpp b/src/mongo/db/storage/mobile/mobile_global_options.cpp new file mode 100644 index 00000000000..ff3a7830340 --- /dev/null +++ b/src/mongo/db/storage/mobile/mobile_global_options.cpp @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage + +#include "mongo/platform/basic.h" + +#include "mongo/db/storage/mobile/mobile_global_options.h" + +#include "mongo/util/log.h" + +namespace moe = mongo::optionenvironment; + +namespace mongo { + +MobileGlobalOptions mobileGlobalOptions; + +Status MobileGlobalOptions::store(const moe::Environment& params) { + // Mobile storage engine options + if (params.count("storage.mobile.durabilityLevel")) { + mobileGlobalOptions.mobileDurabilityLevel = + params["storage.mobile.durabilityLevel"].as<int>(); + } + + return Status::OK(); +} + +} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_global_options.h b/src/mongo/db/storage/mobile/mobile_global_options.h new file mode 100644 index 00000000000..16277c6fcc1 --- /dev/null +++ b/src/mongo/db/storage/mobile/mobile_global_options.h @@ -0,0 +1,52 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <string> + +#include "mongo/base/status.h" +#include "mongo/util/options_parser/environment.h" + +namespace mongo { + +class MobileGlobalOptions { +public: + MobileGlobalOptions() : mobileDurabilityLevel(1){}; + + Status store(const optionenvironment::Environment& params); + + // This is used by the Mobile SE and allows users to set the value + // passed to SQLite's PRAGMA synchronous command + std::int32_t mobileDurabilityLevel; +}; + +extern MobileGlobalOptions mobileGlobalOptions; + +} // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_global_options.idl b/src/mongo/db/storage/mobile/mobile_global_options.idl new file mode 100644 index 00000000000..a8ad241e76e --- /dev/null +++ b/src/mongo/db/storage/mobile/mobile_global_options.idl @@ -0,0 +1,52 @@ +# Copyright (C) 2019-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + cpp_includes: + - "mongo/db/storage/mobile/mobile_global_options.h" + configs: + section: 'Mobile options' + source: [ cli, ini, yaml ] + +configs: + # Mobile storage engine options + "storage.mobile.durabilityLevel": + description: >- + How often you want the Mobile SE to fsync data to disk. This is a trade-off between + durability guarantees and performance. This value maps directly to SQLite's synchronous + PRAGMA. The default value is 1 or NORMAL durability, where it's possible to have a + write rolled back if the device crashes or loses power (you are always safe from + application crashes). If write guarantees are more important than performance, set + this to 2 or 3. Conversely, if you are fine with somewhat ephemeral local data then + you can set this to 0. + short_name: mobileDurabilityLevel + cpp_varname: 'mobileDurabilityLevel' + arg_vartype: Int + default: 1 + validator: {gte: 0, lte: 3} diff --git a/src/mongo/db/storage/mobile/mobile_init.cpp b/src/mongo/db/storage/mobile/mobile_init.cpp index 809286a55b0..702ccdf2b99 100644 --- a/src/mongo/db/storage/mobile/mobile_init.cpp +++ b/src/mongo/db/storage/mobile/mobile_init.cpp @@ -35,6 +35,7 @@ #include "mongo/base/init.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/kv/kv_storage_engine.h" +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/mobile/mobile_kv_engine.h" #include "mongo/db/storage/storage_engine_init.h" #include "mongo/db/storage/storage_options.h" @@ -54,7 +55,9 @@ public: options.directoryPerDB = params.directoryperdb; options.forRepair = params.repair; - MobileKVEngine* kvEngine = new MobileKVEngine(params.dbpath); + MobileKVEngine* kvEngine = + new MobileKVEngine(params.dbpath, mobileGlobalOptions.mobileDurabilityLevel); + return new KVStorageEngine(kvEngine, options); } diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp index 66d16c70d71..ef9637ee21f 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.cpp +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.cpp @@ -56,7 +56,7 @@ namespace mongo { class MobileSession; class SqliteStatement; -MobileKVEngine::MobileKVEngine(const std::string& path) { +MobileKVEngine::MobileKVEngine(const std::string& path, std::int32_t durabilityLevel) { _initDBPath(path); // Initialize the database to be in WAL mode. @@ -86,30 +86,37 @@ MobileKVEngine::MobileKVEngine(const std::string& path) { LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database opened in WAL mode"; } - // Ensure SQLite is operating with "synchronous" flag set to FULL. + // Let's set and enforce the synchronous mode + // We allow the user to specify a value between 0 and 3 + // using the mobileDurabilityLevel option { -#define PRAGMA_SYNC_FULL 2 + std::string synchronousPragma = + "PRAGMA synchronous = " + std::to_string(durabilityLevel) + ";"; + + char* errMsg = NULL; + status = sqlite3_exec(initSession, synchronousPragma.c_str(), NULL, NULL, &errMsg); + checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + sqlite3_free(errMsg); sqlite3_stmt* stmt; - status = sqlite3_prepare_v2(initSession, "PRAGMA main.synchronous;", -1, &stmt, NULL); + status = sqlite3_prepare_v2(initSession, "PRAGMA synchronous;", -1, &stmt, NULL); checkStatus(status, SQLITE_OK, "sqlite3_prepare_v2"); status = sqlite3_step(stmt); checkStatus(status, SQLITE_ROW, "sqlite3_step"); - // Pragma returns current "synchronous" setting, ensure it is FULL. + // Pragma returns current "synchronous" setting int sync_val = sqlite3_column_int(stmt, 0); - fassert(50869, sync_val == PRAGMA_SYNC_FULL); + fassert(50869, sync_val == durabilityLevel); status = sqlite3_finalize(stmt); checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database has synchronous " - << "set to FULL"; + << "mode set to " << durabilityLevel; } - // Set and ensure SQLite is operating with F_FULLFSYNC if the platform permits. + // Set and ensure SQLite is operating with F_FULLFSYNC on darwin kernels { - // Set to use F_FULLFSYNC char* errMsg = NULL; status = sqlite3_exec(initSession, "PRAGMA fullfsync = 1;", NULL, NULL, &errMsg); checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); @@ -132,8 +139,41 @@ MobileKVEngine::MobileKVEngine(const std::string& path) { checkStatus(status, SQLITE_OK, "sqlite3_finalize"); LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Confirmed SQLite database is set to fsync " - << "with F_FULLFSYNC if the platform supports" - << ". Val: " << fullfsync_val; + << "with F_FULLFSYNC if the platform supports it (currently" + << " only darwin kernels). Value: " << fullfsync_val; + } + + // Let's set additional non-critical settings that should improve performance and resource usage + { + // Allow for periodic calls to purge deleted records and prune db size on disk + // Still requires manual vacuum calls using `PRAGMA incremental_vacuum(N);` + char* errMsg = NULL; + status = + sqlite3_exec(initSession, "PRAGMA auto_vacuum = incremental;", NULL, NULL, &errMsg); + checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + sqlite3_free(errMsg); + + // Set the cache size to 10MiB + // value is passed as "-<#ofKiB>" + errMsg = NULL; + status = sqlite3_exec(initSession, "PRAGMA cache_size = -10240;", NULL, NULL, &errMsg); + checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + sqlite3_free(errMsg); + + // Enable memory mapping for the data and set the size at 50MiB + errMsg = NULL; + status = sqlite3_exec(initSession, "PRAGMA mmap_size = 52428800;", NULL, NULL, &errMsg); + checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + sqlite3_free(errMsg); + + // Cap the WAL size at 5MiB so that it's regularly pruned + errMsg = NULL; + status = + sqlite3_exec(initSession, "PRAGMA journal_size_limit = 5242880;", NULL, NULL, &errMsg); + checkStatus(status, SQLITE_OK, "sqlite3_exec", errMsg); + sqlite3_free(errMsg); + + LOG(MOBILE_LOG_LEVEL_LOW) << "MobileSE: Completed all SQLite database configuration"; } _sessionPool.reset(new MobileSessionPool(_path)); diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h index a2b9aef230d..17b34d8c308 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.h +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h @@ -44,7 +44,7 @@ class JournalListener; class MobileKVEngine : public KVEngine { public: - MobileKVEngine(const std::string& path); + MobileKVEngine(const std::string& path, std::int32_t durabilityLevel); RecoveryUnit* newRecoveryUnit() override; diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp b/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp index 12a995fa988..ccbabff8403 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp +++ b/src/mongo/db/storage/mobile/mobile_kv_engine_test.cpp @@ -40,12 +40,12 @@ namespace { class MobileKVHarnessHelper : public KVHarnessHelper { public: - MobileKVHarnessHelper() : _dbPath("mobile_kv_engine_harness") { - _engine = stdx::make_unique<MobileKVEngine>(_dbPath.path()); + MobileKVHarnessHelper() : _dbPath("mobile_kv_engine_harness"), _mobileDurabilityLevel(1) { + _engine = stdx::make_unique<MobileKVEngine>(_dbPath.path(), _mobileDurabilityLevel); } virtual KVEngine* restartEngine() { - _engine.reset(new MobileKVEngine(_dbPath.path())); + _engine.reset(new MobileKVEngine(_dbPath.path(), _mobileDurabilityLevel)); return _engine.get(); } @@ -56,6 +56,7 @@ public: private: std::unique_ptr<MobileKVEngine> _engine; unittest::TempDir _dbPath; + std::int32_t _mobileDurabilityLevel; }; std::unique_ptr<KVHarnessHelper> makeHelper() { diff --git a/src/mongo/db/storage/mobile/mobile_options_init.cpp b/src/mongo/db/storage/mobile/mobile_options_init.cpp new file mode 100644 index 00000000000..e004088f12d --- /dev/null +++ b/src/mongo/db/storage/mobile/mobile_options_init.cpp @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/util/options_parser/startup_option_init.h" + +#include <iostream> + +#include "mongo/db/storage/mobile/mobile_global_options.h" +#include "mongo/util/exit_code.h" +#include "mongo/util/options_parser/startup_options.h" + +namespace moe = mongo::optionenvironment; + +namespace mongo { + +MONGO_STARTUP_OPTIONS_STORE(MobileOptions)(InitializerContext* context) { + Status ret = mobileGlobalOptions.store(moe::startupOptionsParsed); + if (!ret.isOK()) { + std::cerr << ret.toString() << std::endl; + std::cerr << "try '" << context->args()[0] << " --help' for more information" << std::endl; + ::_exit(EXIT_BADOPTIONS); + } + return Status::OK(); +} +} // namespace mongo diff --git a/src/mongo/embedded/embedded_options.cpp b/src/mongo/embedded/embedded_options.cpp index e89f5b7d267..8cc29ac799e 100644 --- a/src/mongo/embedded/embedded_options.cpp +++ b/src/mongo/embedded/embedded_options.cpp @@ -34,6 +34,7 @@ #include "mongo/db/server_options.h" #include "mongo/db/server_options_helpers.h" +#include "mongo/db/storage/mobile/mobile_global_options.h" #include "mongo/db/storage/storage_options.h" #include <boost/filesystem.hpp> @@ -114,6 +115,12 @@ Status storeOptions(const moe::Environment& params) { if (params.count("storage.dbPath")) { storageGlobalParams.dbpath = params["storage.dbPath"].as<string>(); } + + if (params.count("storage.mobile.durabilityLevel")) { + mobileGlobalOptions.mobileDurabilityLevel = + params["storage.mobile.durabilityLevel"].as<int>(); + } + #ifdef _WIN32 if (storageGlobalParams.dbpath.size() > 1 && storageGlobalParams.dbpath[storageGlobalParams.dbpath.size() - 1] == '/') { |