summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Cameron <alex.cameron@10gen.com>2019-08-05 11:11:43 +0000
committerAlex Cameron <alex.cameron@10gen.com>2019-08-05 12:06:42 +0000
commitb571e1a50d9990690f86539127cc2d119ec0edae (patch)
tree6f22e81746e8cffd48cfe9cf4b9be21a8251de99
parent74f9c3bbd17e2e057f2def58adf341628a94f83a (diff)
downloadmongo-b571e1a50d9990690f86539127cc2d119ec0edae.tar.gz
SERVER-39004 Introduce a quota mechanism for the overflow file
(cherry picked from commit b29c22ad4b89dccca63bd2a279c48f47f76093d1) (cherry picked from commit b6f1010451de1a10e12c4b6d865476cce284afce)
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp28
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h4
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp5
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp2
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h3
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp1
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp67
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h13
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp1
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp3
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp1
11 files changed, 126 insertions, 2 deletions
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
index 342220766dc..b4995e56dc4 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
@@ -79,6 +79,13 @@ Status WiredTigerGlobalOptions::add(moe::OptionSection* options) {
"WiredTiger storage engine custom "
"configuration settings")
.hidden();
+ wiredTigerOptions
+ .addOptionChaining("storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB",
+ "wiredTigerMaxCacheOverflowFileSizeGB",
+ moe::Double,
+ "Maximum amount of disk space to use for cache overflow; "
+ "Defaults to 0 (unbounded)")
+ .setDefault(moe::Value(0.0));
// WiredTiger collection options
wiredTigerOptions
@@ -142,6 +149,18 @@ Status WiredTigerGlobalOptions::store(const moe::Environment& params,
params["storage.wiredTiger.engineConfig.configString"].as<std::string>();
log() << "Engine custom option: " << wiredTigerGlobalOptions.engineConfig;
}
+ if (params.count("storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB")) {
+ const auto val =
+ params["storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB"].as<double>();
+ const auto result = validateMaxCacheOverflowFileSizeGB(val);
+ if (result.isOK()) {
+ wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB = val;
+ log() << "Max cache overflow file size custom option: "
+ << wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB;
+ } else {
+ return result;
+ }
+ }
// WiredTiger collection options
if (params.count("storage.wiredTiger.collectionConfig.blockCompressor")) {
@@ -168,4 +187,13 @@ Status WiredTigerGlobalOptions::store(const moe::Environment& params,
return Status::OK();
}
+Status WiredTigerGlobalOptions::validateMaxCacheOverflowFileSizeGB(double value) {
+ if (value != 0.0 && value < 0.1) {
+ return {ErrorCodes::BadValue,
+ "MaxCacheOverflowFileSizeGB must be either 0 (unbounded) or greater than 0.1."};
+ }
+
+ return Status::OK();
+}
+
} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
index 2dda59efc14..a591b30e97e 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
@@ -46,6 +46,7 @@ public:
checkpointDelaySecs(0),
statisticsLogDelaySecs(0),
directoryForIndexes(false),
+ maxCacheOverflowFileSizeGB(0),
useCollectionPrefixCompression(false),
useIndexPrefixCompression(false){};
@@ -57,6 +58,7 @@ public:
size_t statisticsLogDelaySecs;
std::string journalCompressor;
bool directoryForIndexes;
+ double maxCacheOverflowFileSizeGB;
std::string engineConfig;
std::string collectionBlockCompressor;
@@ -65,6 +67,8 @@ public:
bool useIndexPrefixCompression;
std::string collectionConfig;
std::string indexConfig;
+
+ static Status validateMaxCacheOverflowFileSizeGB(double);
};
extern WiredTigerGlobalOptions wiredTigerGlobalOptions;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
index 8f1e9d5b8f4..cc7d5a74503 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
@@ -102,12 +102,15 @@ public:
}
}
const bool ephemeral = false;
+ const auto maxCacheOverflowMB =
+ static_cast<size_t>(1024 * wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB);
WiredTigerKVEngine* kv =
new WiredTigerKVEngine(getCanonicalName().toString(),
params.dbpath,
getGlobalServiceContext()->getFastClockSource(),
wiredTigerGlobalOptions.engineConfig,
cacheMB,
+ maxCacheOverflowMB,
params.dur,
ephemeral,
params.repair,
@@ -117,6 +120,8 @@ public:
// Intentionally leaked.
new WiredTigerServerStatusSection(kv);
new WiredTigerEngineRuntimeConfigParameter(kv);
+ new WiredTigerMaxCacheOverflowSizeGBParameter(
+ kv, wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB);
KVStorageEngineOptions options;
options.directoryPerDB = params.directoryperdb;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 8fce4a59e39..135f6462863 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -485,6 +485,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
ClockSource* cs,
const std::string& extraOpenOptions,
size_t cacheSizeMB,
+ size_t maxCacheOverflowFileSizeMB,
bool durable,
bool ephemeral,
bool repair,
@@ -517,6 +518,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
std::stringstream ss;
ss << "create,";
ss << "cache_size=" << cacheSizeMB << "M,";
+ ss << "cache_overflow=(file_max=" << maxCacheOverflowFileSizeMB << "M),";
ss << "session_max=20000,";
ss << "eviction=(threads_min=4,threads_max=4),";
ss << "config_base=false,";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 3bc5062413f..ffd7a4f8fd7 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -72,7 +72,8 @@ public:
const std::string& path,
ClockSource* cs,
const std::string& extraOpenOptions,
- size_t cacheSizeGB,
+ size_t cacheSizeMB,
+ size_t maxCacheOverflowFileSizeMB,
bool durable,
bool ephemeral,
bool repair,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
index 54656fa1fe1..5ecd65fa828 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
@@ -89,6 +89,7 @@ private:
_cs.get(),
"",
1,
+ 0,
false,
false,
_forRepair,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
index f34bbb16026..b929317f22d 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
@@ -41,6 +41,37 @@ namespace mongo {
using std::string;
+namespace {
+
+Status applyMaxCacheOverflowSizeGBParameter(WiredTigerMaxCacheOverflowSizeGBParameter& param,
+ double value) {
+ if (value != 0.0 && value < 0.1) {
+ return {ErrorCodes::BadValue,
+ "MaxCacheOverflowFileSizeGB must be either 0 (unbounded) or greater than 0.1."};
+ }
+
+ const auto valueMB = static_cast<size_t>(1024 * value);
+
+ log() << "Reconfiguring WiredTiger max cache overflow size with value: \"" << valueMB << "MB\'";
+
+ invariant(param._data.second);
+ int ret = param._data.second->reconfigure(
+ std::string(str::stream() << "cache_overflow=(file_max=" << valueMB << "M)").c_str());
+ if (ret != 0) {
+ string result =
+ (str::stream() << "WiredTiger reconfiguration failed with error code (" << ret << "): "
+ << wiredtiger_strerror(ret));
+ error() << result;
+
+ return Status(ErrorCodes::BadValue, result);
+ }
+
+ param._data.first = value;
+ return Status::OK();
+}
+
+} // namespace
+
WiredTigerEngineRuntimeConfigParameter::WiredTigerEngineRuntimeConfigParameter(
WiredTigerKVEngine* engine)
: ServerParameter(
@@ -95,4 +126,40 @@ Status WiredTigerEngineRuntimeConfigParameter::setFromString(const std::string&
_currentValue = str;
return Status::OK();
}
+
+WiredTigerMaxCacheOverflowSizeGBParameter::WiredTigerMaxCacheOverflowSizeGBParameter(
+ WiredTigerKVEngine* engine, double value)
+ : ServerParameter(
+ ServerParameterSet::getGlobal(), "wiredTigerMaxCacheOverflowSizeGB", false, true),
+ _data(value, engine) {}
+
+void WiredTigerMaxCacheOverflowSizeGBParameter::append(OperationContext* opCtx,
+ BSONObjBuilder& b,
+ const std::string& name) {
+ b << name << _data.first;
}
+
+Status WiredTigerMaxCacheOverflowSizeGBParameter::set(const BSONElement& element) {
+ if (element.type() == String) {
+ return setFromString(element.valuestrsafe());
+ }
+
+ double value;
+ if (!element.coerce(&value)) {
+ return {ErrorCodes::BadValue, "MaxCacheOverflowFileSizeGB must be a numeric value."};
+ }
+
+ return applyMaxCacheOverflowSizeGBParameter(*this, value);
+}
+
+Status WiredTigerMaxCacheOverflowSizeGBParameter::setFromString(const std::string& str) {
+ double value;
+ const auto status = parseNumberFromString(str, &value);
+ if (!status.isOK()) {
+ return status;
+ }
+
+ return applyMaxCacheOverflowSizeGBParameter(*this, value);
+}
+
+} // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
index 1c622c6ab0a..f506710354c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
@@ -59,4 +59,17 @@ private:
// starts out as the empty string.
std::string _currentValue;
};
+
+class WiredTigerMaxCacheOverflowSizeGBParameter : public ServerParameter {
+ MONGO_DISALLOW_COPYING(WiredTigerMaxCacheOverflowSizeGBParameter);
+
+public:
+ WiredTigerMaxCacheOverflowSizeGBParameter(WiredTigerKVEngine* engine, double value);
+
+ virtual void append(OperationContext* opCtx, BSONObjBuilder& b, const std::string& name);
+ virtual Status set(const BSONElement& newValueElement);
+ virtual Status setFromString(const std::string& str);
+
+ std::pair<double, WiredTigerKVEngine*> _data;
+};
}
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
index 898c4782fc6..c49103faa3b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
@@ -79,6 +79,7 @@ public:
_cs.get(),
"",
1,
+ 0,
false,
false,
false,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
index 4d480451eb7..8f7c0a8a505 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
@@ -54,7 +54,8 @@ public:
_dbpath.path(), // .path
&_cs, // .cs
"", // .extraOpenOptions
- 1, // .cacheSizeGB
+ 1, // .cacheSizeMB
+ 0, // .maxCacheOverflowFileSizeMB
false, // .durable
false, // .ephemeral
false, // .repair
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
index 0cff392042e..9abdf85c6ca 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
@@ -80,6 +80,7 @@ public:
&_cs,
extraStrings.toString(),
1,
+ 0,
false,
false,
false,