diff options
author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2017-04-21 06:51:55 +1000 |
---|---|---|
committer | David Hows <howsdav@gmail.com> | 2017-08-10 12:12:27 +1000 |
commit | fad92bef4b8207c5ce392c4402080c0460ccd5ba (patch) | |
tree | 1926041e0dd08e46a3f389b80453e1fd88ccf7ed | |
parent | 9ea7ad5950415500142a41eb07cf8cb131ed52de (diff) | |
download | mongo-fad92bef4b8207c5ce392c4402080c0460ccd5ba.tar.gz |
SERVER-26239 Improve handling of WT_CACHE_FULL for inMemory storage engine
(cherry picked from commit 8b437e7a762e3ef99848659dc0d68df1e2add0a4)
-rw-r--r-- | src/mongo/db/db.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/dbhelpers.cpp | 22 | ||||
-rw-r--r-- | src/mongo/db/pipeline/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/sorter/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/sorter/sorter.cpp | 33 | ||||
-rw-r--r-- | src/mongo/db/storage/SConscript | 15 | ||||
-rw-r--r-- | src/mongo/db/storage/encryption_hooks.cpp | 96 | ||||
-rw-r--r-- | src/mongo/db/storage/encryption_hooks.h | 99 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp | 34 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h | 75 |
10 files changed, 248 insertions, 134 deletions
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2dbb2b1065e..a1257ec0a65 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -103,10 +103,10 @@ #include "mongo/db/startup_warnings_mongod.h" #include "mongo/db/stats/counters.h" #include "mongo/db/stats/snapshots.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/mmap_v1/mmap_v1_options.h" #include "mongo/db/storage/storage_engine.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/ttl.h" #include "mongo/db/wire_version.h" #include "mongo/executor/network_interface_factory.h" @@ -596,7 +596,7 @@ ExitCode _initAndListen(int listenPort) { getGlobalServiceContext()->initializeGlobalStorageEngine(); #ifdef MONGO_CONFIG_WIREDTIGER_ENABLED - if (WiredTigerCustomizationHooks::get(getGlobalServiceContext())->restartRequired()) { + if (EncryptionHooks::get(getGlobalServiceContext())->restartRequired()) { exitCleanly(EXIT_CLEAN); } #endif diff --git a/src/mongo/db/dbhelpers.cpp b/src/mongo/db/dbhelpers.cpp index fbf979277e2..68e86ea9bb5 100644 --- a/src/mongo/db/dbhelpers.cpp +++ b/src/mongo/db/dbhelpers.cpp @@ -62,8 +62,8 @@ #include "mongo/db/s/sharding_state.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/data_protector.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/write_concern.h" #include "mongo/db/write_concern_options.h" #include "mongo/s/shard_key_pattern.h" @@ -487,19 +487,19 @@ Helpers::RemoveSaver::RemoveSaver(const string& a, const string& b, const string ss << why << "." << terseCurrentTime(false) << "." << NUM++ << ".bson"; _file /= ss.str(); - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { - _protector = hooks->getDataProtector(); - _file += hooks->getProtectedPathSuffix(); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { + _protector = encryptionHooks->getDataProtector(); + _file += encryptionHooks->getProtectedPathSuffix(); } } Helpers::RemoveSaver::~RemoveSaver() { if (_protector && _out) { - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - invariant(hooks->enabled()); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + invariant(encryptionHooks->enabled()); - size_t protectedSizeMax = hooks->additionalBytesForProtectedBuffer(); + size_t protectedSizeMax = encryptionHooks->additionalBytesForProtectedBuffer(); std::unique_ptr<uint8_t[]> protectedBuffer(new uint8_t[protectedSizeMax]); size_t resultLen; @@ -562,10 +562,10 @@ Status Helpers::RemoveSaver::goingToDelete(const BSONObj& o) { std::unique_ptr<uint8_t[]> protectedBuffer; if (_protector) { - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - invariant(hooks->enabled()); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + invariant(encryptionHooks->enabled()); - size_t protectedSizeMax = dataSize + hooks->additionalBytesForProtectedBuffer(); + size_t protectedSizeMax = dataSize + encryptionHooks->additionalBytesForProtectedBuffer(); protectedBuffer.reset(new uint8_t[protectedSizeMax]); size_t resultLen; diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index 98a66a02f48..ca8617f3d3d 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -252,8 +252,8 @@ docSourceEnv.Library( '$BUILD_DIR/mongo/db/matcher/expression_algo', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/stats/top', + '$BUILD_DIR/mongo/db/storage/encryption_hooks', '$BUILD_DIR/mongo/db/storage/storage_options', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger_customization_hooks', '$BUILD_DIR/third_party/shim_snappy', ], LIBDEPS_TAGS=[ diff --git a/src/mongo/db/sorter/SConscript b/src/mongo/db/sorter/SConscript index 959f6cdb919..1ceb05088af 100644 --- a/src/mongo/db/sorter/SConscript +++ b/src/mongo/db/sorter/SConscript @@ -5,6 +5,6 @@ sorterEnv.InjectThirdPartyIncludePaths(libraries=['snappy']) sorterEnv.CppUnitTest('sorter_test', 'sorter_test.cpp', LIBDEPS=['$BUILD_DIR/mongo/db/service_context', - '$BUILD_DIR/mongo/db/storage/wiredtiger/storage_wiredtiger_customization_hooks', + '$BUILD_DIR/mongo/db/storage/encryption_hooks', '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/third_party/shim_snappy']) diff --git a/src/mongo/db/sorter/sorter.cpp b/src/mongo/db/sorter/sorter.cpp index 7f6e8866562..0165e0adccc 100644 --- a/src/mongo/db/sorter/sorter.cpp +++ b/src/mongo/db/sorter/sorter.cpp @@ -55,8 +55,8 @@ #include "mongo/config.h" #include "mongo/db/jsobj.h" #include "mongo/db/service_context.h" +#include "mongo/db/storage/encryption_hooks.h" #include "mongo/db/storage/storage_options.h" -#include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/platform/atomic_word.h" #include "mongo/s/mongos_options.h" #include "mongo/util/assert_util.h" @@ -226,15 +226,16 @@ private: read(_buffer.get(), blockSize); massert(16816, "file too short?", !_done); - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { std::unique_ptr<char[]> out(new char[blockSize]); size_t outLen; - Status status = hooks->unprotectTmpData(reinterpret_cast<uint8_t*>(_buffer.get()), - blockSize, - reinterpret_cast<uint8_t*>(out.get()), - blockSize, - &outLen); + Status status = + encryptionHooks->unprotectTmpData(reinterpret_cast<uint8_t*>(_buffer.get()), + blockSize, + reinterpret_cast<uint8_t*>(out.get()), + blockSize, + &outLen); massert(28841, str::stream() << "Failed to unprotect data: " << status.toString(), status.isOK()); @@ -884,16 +885,16 @@ void SortedFileWriter<Key, Value>::spill() { } std::unique_ptr<char[]> out; - auto hooks = WiredTigerCustomizationHooks::get(getGlobalServiceContext()); - if (hooks->enabled()) { - size_t protectedSizeMax = size + hooks->additionalBytesForProtectedBuffer(); + auto encryptionHooks = EncryptionHooks::get(getGlobalServiceContext()); + if (encryptionHooks->enabled()) { + size_t protectedSizeMax = size + encryptionHooks->additionalBytesForProtectedBuffer(); out.reset(new char[protectedSizeMax]); size_t resultLen; - Status status = hooks->protectTmpData(reinterpret_cast<const uint8_t*>(outBuffer), - size, - reinterpret_cast<uint8_t*>(out.get()), - protectedSizeMax, - &resultLen); + Status status = encryptionHooks->protectTmpData(reinterpret_cast<const uint8_t*>(outBuffer), + size, + reinterpret_cast<uint8_t*>(out.get()), + protectedSizeMax, + &resultLen); massert(28842, str::stream() << "Failed to compress data: " << status.toString(), status.isOK()); diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index d65046ed6fa..e887882df87 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -11,7 +11,6 @@ env.SConscript( ], ) - env.Library( target='journal_listener', source=[ @@ -41,7 +40,6 @@ env.Library( ], ) - env.Library( target='bson_collection_catalog_entry', source=[ @@ -75,6 +73,19 @@ env.Library( ) env.Library( + target='encryption_hooks', + source= [ + 'encryption_hooks.cpp', + ], + LIBDEPS= ['$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/service_context'], + PROGDEPS_DEPENDENTS=[ + '$BUILD_DIR/mongo/mongod', + '$BUILD_DIR/mongo/mongos', + ], + ) + +env.Library( target='storage_options', source=[ 'storage_options.cpp', diff --git a/src/mongo/db/storage/encryption_hooks.cpp b/src/mongo/db/storage/encryption_hooks.cpp new file mode 100644 index 00000000000..9ee9a317dbe --- /dev/null +++ b/src/mongo/db/storage/encryption_hooks.cpp @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/storage/encryption_hooks.h" + +#include <boost/filesystem/path.hpp> + +#include "mongo/base/init.h" +#include "mongo/db/service_context.h" +#include "mongo/db/storage/data_protector.h" +#include "mongo/stdx/memory.h" + +namespace mongo { + +/* Make a EncryptionHooks pointer a decoration on the global ServiceContext */ +MONGO_INITIALIZER_WITH_PREREQUISITES(SetEncryptionHooks, ("SetGlobalEnvironment")) +(InitializerContext* context) { + auto encryptionHooks = stdx::make_unique<EncryptionHooks>(); + EncryptionHooks::set(getGlobalServiceContext(), std::move(encryptionHooks)); + + return Status::OK(); +} + +namespace { +const auto getEncryptionHooks = + ServiceContext::declareDecoration<std::unique_ptr<EncryptionHooks>>(); +} // namespace + +void EncryptionHooks::set(ServiceContext* service, std::unique_ptr<EncryptionHooks> custHooks) { + auto& hooks = getEncryptionHooks(service); + invariant(custHooks); + hooks = std::move(custHooks); +} + +EncryptionHooks* EncryptionHooks::get(ServiceContext* service) { + return getEncryptionHooks(service).get(); +} + +EncryptionHooks::~EncryptionHooks() {} + +bool EncryptionHooks::enabled() const { + return false; +} + +bool EncryptionHooks::restartRequired() { + return false; +} + +std::unique_ptr<DataProtector> EncryptionHooks::getDataProtector() { + return std::unique_ptr<DataProtector>(); +} + +boost::filesystem::path EncryptionHooks::getProtectedPathSuffix() { + return ""; +} + +Status EncryptionHooks::protectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { + return Status(ErrorCodes::InternalError, + "Encryption hooks must be enabled to use preprocessTmpData."); +} + +Status EncryptionHooks::unprotectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { + return Status(ErrorCodes::InternalError, + "Encryption hooks must be enabled to use postprocessTmpData."); +} +} // namespace mongo diff --git a/src/mongo/db/storage/encryption_hooks.h b/src/mongo/db/storage/encryption_hooks.h new file mode 100644 index 00000000000..e1c9d553a10 --- /dev/null +++ b/src/mongo/db/storage/encryption_hooks.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2017 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include <memory> +#include <string> + +#include "mongo/base/disallow_copying.h" +#include "mongo/db/jsobj.h" + +namespace boost { +namespace filesystem { +class path; +} // namespace filesystem +} // namespace boost + +namespace mongo { +class DataProtector; +class ServiceContext; + +class EncryptionHooks { +public: + static void set(ServiceContext* service, std::unique_ptr<EncryptionHooks> custHooks); + + static EncryptionHooks* get(ServiceContext* service); + + virtual ~EncryptionHooks(); + + /** + * Returns true if the encryption hooks are enabled. + */ + virtual bool enabled() const; + + /** + * Perform any encryption engine initialization/sanity checking that needs to happen after + * storage engine initialization but before the server starts accepting incoming connections. + * + * Returns true if the server needs to be rebooted because of configuration changes. + */ + virtual bool restartRequired(); + + /** + * Returns the maximum size addition when doing transforming temp data. + */ + size_t additionalBytesForProtectedBuffer() { + return 33; + } + + /** + * Get the data protector object + */ + virtual std::unique_ptr<DataProtector> getDataProtector(); + + /** + * Get an implementation specific path suffix to tag files with + */ + virtual boost::filesystem::path getProtectedPathSuffix(); + + /** + * Transform temp data to non-readable form before writing it to disk. + */ + virtual Status protectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen); + + /** + * Tranforms temp data back to readable form, after reading from disk. + */ + virtual Status unprotectTmpData( + const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen); +}; + +} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp index d14b33de5f7..e40d3a58edb 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.cpp @@ -31,12 +31,9 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" -#include <boost/filesystem/path.hpp> - #include "mongo/base/init.h" #include "mongo/base/string_data.h" #include "mongo/db/service_context.h" -#include "mongo/db/storage/data_protector.h" #include "mongo/stdx/memory.h" namespace mongo { @@ -44,7 +41,7 @@ namespace mongo { /* Make a WiredTigerCustomizationHooks pointer a decoration on the global ServiceContext */ MONGO_INITIALIZER_WITH_PREREQUISITES(SetWiredTigerCustomizationHooks, ("SetGlobalEnvironment")) (InitializerContext* context) { - auto customizationHooks = stdx::make_unique<EmptyWiredTigerCustomizationHooks>(); + auto customizationHooks = stdx::make_unique<WiredTigerCustomizationHooks>(); WiredTigerCustomizationHooks::set(getGlobalServiceContext(), std::move(customizationHooks)); return Status::OK(); @@ -66,37 +63,14 @@ WiredTigerCustomizationHooks* WiredTigerCustomizationHooks::get(ServiceContext* return getCustomizationHooks(service).get(); } -EmptyWiredTigerCustomizationHooks::~EmptyWiredTigerCustomizationHooks() {} - -bool EmptyWiredTigerCustomizationHooks::enabled() const { - return false; -} +WiredTigerCustomizationHooks::~WiredTigerCustomizationHooks() {} -bool EmptyWiredTigerCustomizationHooks::restartRequired() { +bool WiredTigerCustomizationHooks::enabled() const { return false; } -std::string EmptyWiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) { - return ""; -} - -std::unique_ptr<DataProtector> EmptyWiredTigerCustomizationHooks::getDataProtector() { - return std::unique_ptr<DataProtector>(); -} - -boost::filesystem::path EmptyWiredTigerCustomizationHooks::getProtectedPathSuffix() { +std::string WiredTigerCustomizationHooks::getTableCreateConfig(StringData tableName) { return ""; } -Status EmptyWiredTigerCustomizationHooks::protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { - return Status(ErrorCodes::InternalError, - "Customization hooks must be enabled to use preprocessTmpData."); -} - -Status EmptyWiredTigerCustomizationHooks::unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) { - return Status(ErrorCodes::InternalError, - "Customization hooks must be enabled to use postprocessTmpData."); -} } // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h index e78995481bd..1ff86a8799e 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h @@ -32,20 +32,11 @@ #include <memory> #include <string> -#include "mongo/base/disallow_copying.h" -#include "mongo/db/jsobj.h" - -namespace boost { -namespace filesystem { -class path; -} // namespace filesystem -} // namespace boost - namespace mongo { -class DataProtector; class StringData; class ServiceContext; +// Interface and default implementation for WiredTiger customization hooks class WiredTigerCustomizationHooks { public: static void set(ServiceContext* service, @@ -53,76 +44,18 @@ public: static WiredTigerCustomizationHooks* get(ServiceContext* service); - virtual ~WiredTigerCustomizationHooks() = default; + virtual ~WiredTigerCustomizationHooks(); /** * Returns true if the customization hooks are enabled. */ - virtual bool enabled() const = 0; - - /** - * Perform any encryption engine initialization/sanity checking that needs to happen after - * storage engine initialization but before the server starts accepting incoming connections. - * - * Returns true if the server needs to be rebooted because of configuration changes. - */ - virtual bool restartRequired() = 0; + virtual bool enabled() const; /** * Gets an additional configuration string for the provided table name on a * `WT_SESSION::create` call. */ - virtual std::string getTableCreateConfig(StringData tableName) = 0; - - /** - * Returns the maximum size addition when doing transforming temp data. - */ - size_t additionalBytesForProtectedBuffer() { - return 33; - } - - /** - * Get the data protector object - */ - virtual std::unique_ptr<DataProtector> getDataProtector() = 0; - - /** - * Get an implementation specific path suffix to tag files with - */ - virtual boost::filesystem::path getProtectedPathSuffix() = 0; - - /** - * Transform temp data to non-readable form before writing it to disk. - */ - virtual Status protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0; - - /** - * Tranforms temp data back to readable form, after reading from disk. - */ - virtual Status unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) = 0; + virtual std::string getTableCreateConfig(StringData tableName); }; -// Empty default implementation of the abstract class WiredTigerCustomizationHooks -class EmptyWiredTigerCustomizationHooks : public WiredTigerCustomizationHooks { -public: - ~EmptyWiredTigerCustomizationHooks() override; - - bool enabled() const override; - - bool restartRequired() override; - - std::string getTableCreateConfig(StringData tableName) override; - - std::unique_ptr<DataProtector> getDataProtector() override; - - boost::filesystem::path getProtectedPathSuffix() override; - - Status protectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override; - - Status unprotectTmpData( - const uint8_t* in, size_t inLen, uint8_t* out, size_t outLen, size_t* resultLen) override; -}; } // namespace mongo |