SERVER-39004 Introduce a quota mechanism for the overflow file

(cherry picked from commit b29c22ad4b89dccca63bd2a279c48f47f76093d1)
(cherry picked from commit b6f1010451de1a10e12c4b6d865476cce284afce)

11 files changed, 126 insertions, 2 deletions

diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
index 342220766dc..b4995e56dc4 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp
@@ -79,6 +79,13 @@ Status WiredTigerGlobalOptions::add(moe::OptionSection* options) {
                            "WiredTiger storage engine custom "
                            "configuration settings")
         .hidden();
+    wiredTigerOptions
+        .addOptionChaining("storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB",
+                           "wiredTigerMaxCacheOverflowFileSizeGB",
+                           moe::Double,
+                           "Maximum amount of disk space to use for cache overflow; "
+                           "Defaults to 0 (unbounded)")
+        .setDefault(moe::Value(0.0));
 
     // WiredTiger collection options
     wiredTigerOptions
@@ -142,6 +149,18 @@ Status WiredTigerGlobalOptions::store(const moe::Environment& params,
             params["storage.wiredTiger.engineConfig.configString"].as<std::string>();
         log() << "Engine custom option: " << wiredTigerGlobalOptions.engineConfig;
     }
+    if (params.count("storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB")) {
+        const auto val =
+            params["storage.wiredTiger.engineConfig.maxCacheOverflowFileSizeGB"].as<double>();
+        const auto result = validateMaxCacheOverflowFileSizeGB(val);
+        if (result.isOK()) {
+            wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB = val;
+            log() << "Max cache overflow file size custom option: "
+                  << wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB;
+        } else {
+            return result;
+        }
+    }
 
     // WiredTiger collection options
     if (params.count("storage.wiredTiger.collectionConfig.blockCompressor")) {
@@ -168,4 +187,13 @@ Status WiredTigerGlobalOptions::store(const moe::Environment& params,
     return Status::OK();
 }
 
+Status WiredTigerGlobalOptions::validateMaxCacheOverflowFileSizeGB(double value) {
+    if (value != 0.0 && value < 0.1) {
+        return {ErrorCodes::BadValue,
+                "MaxCacheOverflowFileSizeGB must be either 0 (unbounded) or greater than 0.1."};
+    }
+
+    return Status::OK();
+}
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
index 2dda59efc14..a591b30e97e 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h
@@ -46,6 +46,7 @@ public:
           checkpointDelaySecs(0),
           statisticsLogDelaySecs(0),
           directoryForIndexes(false),
+          maxCacheOverflowFileSizeGB(0),
           useCollectionPrefixCompression(false),
           useIndexPrefixCompression(false){};
 
@@ -57,6 +58,7 @@ public:
     size_t statisticsLogDelaySecs;
     std::string journalCompressor;
     bool directoryForIndexes;
+    double maxCacheOverflowFileSizeGB;
     std::string engineConfig;
 
     std::string collectionBlockCompressor;
@@ -65,6 +67,8 @@ public:
     bool useIndexPrefixCompression;
     std::string collectionConfig;
     std::string indexConfig;
+
+    static Status validateMaxCacheOverflowFileSizeGB(double);
 };
 
 extern WiredTigerGlobalOptions wiredTigerGlobalOptions;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
index 8f1e9d5b8f4..cc7d5a74503 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_init.cpp
@@ -102,12 +102,15 @@ public:
             }
         }
         const bool ephemeral = false;
+        const auto maxCacheOverflowMB =
+            static_cast<size_t>(1024 * wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB);
         WiredTigerKVEngine* kv =
             new WiredTigerKVEngine(getCanonicalName().toString(),
                                    params.dbpath,
                                    getGlobalServiceContext()->getFastClockSource(),
                                    wiredTigerGlobalOptions.engineConfig,
                                    cacheMB,
+                                   maxCacheOverflowMB,
                                    params.dur,
                                    ephemeral,
                                    params.repair,
@@ -117,6 +120,8 @@ public:
         // Intentionally leaked.
         new WiredTigerServerStatusSection(kv);
         new WiredTigerEngineRuntimeConfigParameter(kv);
+        new WiredTigerMaxCacheOverflowSizeGBParameter(
+            kv, wiredTigerGlobalOptions.maxCacheOverflowFileSizeGB);
 
         KVStorageEngineOptions options;
         options.directoryPerDB = params.directoryperdb;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 8fce4a59e39..135f6462863 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -485,6 +485,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
                                        ClockSource* cs,
                                        const std::string& extraOpenOptions,
                                        size_t cacheSizeMB,
+                                       size_t maxCacheOverflowFileSizeMB,
                                        bool durable,
                                        bool ephemeral,
                                        bool repair,
@@ -517,6 +518,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
     std::stringstream ss;
     ss << "create,";
     ss << "cache_size=" << cacheSizeMB << "M,";
+    ss << "cache_overflow=(file_max=" << maxCacheOverflowFileSizeMB << "M),";
     ss << "session_max=20000,";
     ss << "eviction=(threads_min=4,threads_max=4),";
     ss << "config_base=false,";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 3bc5062413f..ffd7a4f8fd7 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -72,7 +72,8 @@ public:
                        const std::string& path,
                        ClockSource* cs,
                        const std::string& extraOpenOptions,
-                       size_t cacheSizeGB,
+                       size_t cacheSizeMB,
+                       size_t maxCacheOverflowFileSizeMB,
                        bool durable,
                        bool ephemeral,
                        bool repair,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
index 54656fa1fe1..5ecd65fa828 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
@@ -89,6 +89,7 @@ private:
                                       _cs.get(),
                                       "",
                                       1,
+                                      0,
                                       false,
                                       false,
                                       _forRepair,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
index f34bbb16026..b929317f22d 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
@@ -41,6 +41,37 @@ namespace mongo {
 
 using std::string;
 
+namespace {
+
+Status applyMaxCacheOverflowSizeGBParameter(WiredTigerMaxCacheOverflowSizeGBParameter& param,
+                                            double value) {
+    if (value != 0.0 && value < 0.1) {
+        return {ErrorCodes::BadValue,
+                "MaxCacheOverflowFileSizeGB must be either 0 (unbounded) or greater than 0.1."};
+    }
+
+    const auto valueMB = static_cast<size_t>(1024 * value);
+
+    log() << "Reconfiguring WiredTiger max cache overflow size with value: \"" << valueMB << "MB\'";
+
+    invariant(param._data.second);
+    int ret = param._data.second->reconfigure(
+        std::string(str::stream() << "cache_overflow=(file_max=" << valueMB << "M)").c_str());
+    if (ret != 0) {
+        string result =
+            (str::stream() << "WiredTiger reconfiguration failed with error code (" << ret << "): "
+                           << wiredtiger_strerror(ret));
+        error() << result;
+
+        return Status(ErrorCodes::BadValue, result);
+    }
+
+    param._data.first = value;
+    return Status::OK();
+}
+
+}  // namespace
+
 WiredTigerEngineRuntimeConfigParameter::WiredTigerEngineRuntimeConfigParameter(
     WiredTigerKVEngine* engine)
     : ServerParameter(
@@ -95,4 +126,40 @@ Status WiredTigerEngineRuntimeConfigParameter::setFromString(const std::string&
     _currentValue = str;
     return Status::OK();
 }
+
+WiredTigerMaxCacheOverflowSizeGBParameter::WiredTigerMaxCacheOverflowSizeGBParameter(
+    WiredTigerKVEngine* engine, double value)
+    : ServerParameter(
+          ServerParameterSet::getGlobal(), "wiredTigerMaxCacheOverflowSizeGB", false, true),
+      _data(value, engine) {}
+
+void WiredTigerMaxCacheOverflowSizeGBParameter::append(OperationContext* opCtx,
+                                                       BSONObjBuilder& b,
+                                                       const std::string& name) {
+    b << name << _data.first;
 }
+
+Status WiredTigerMaxCacheOverflowSizeGBParameter::set(const BSONElement& element) {
+    if (element.type() == String) {
+        return setFromString(element.valuestrsafe());
+    }
+
+    double value;
+    if (!element.coerce(&value)) {
+        return {ErrorCodes::BadValue, "MaxCacheOverflowFileSizeGB must be a numeric value."};
+    }
+
+    return applyMaxCacheOverflowSizeGBParameter(*this, value);
+}
+
+Status WiredTigerMaxCacheOverflowSizeGBParameter::setFromString(const std::string& str) {
+    double value;
+    const auto status = parseNumberFromString(str, &value);
+    if (!status.isOK()) {
+        return status;
+    }
+
+    return applyMaxCacheOverflowSizeGBParameter(*this, value);
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
index 1c622c6ab0a..f506710354c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.h
@@ -59,4 +59,17 @@ private:
     // starts out as the empty string.
     std::string _currentValue;
 };
+
+class WiredTigerMaxCacheOverflowSizeGBParameter : public ServerParameter {
+    MONGO_DISALLOW_COPYING(WiredTigerMaxCacheOverflowSizeGBParameter);
+
+public:
+    WiredTigerMaxCacheOverflowSizeGBParameter(WiredTigerKVEngine* engine, double value);
+
+    virtual void append(OperationContext* opCtx, BSONObjBuilder& b, const std::string& name);
+    virtual Status set(const BSONElement& newValueElement);
+    virtual Status setFromString(const std::string& str);
+
+    std::pair<double, WiredTigerKVEngine*> _data;
+};
 }
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
index 898c4782fc6..c49103faa3b 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_prefixed_record_store_test.cpp
@@ -79,6 +79,7 @@ public:
                                          _cs.get(),
                                          "",
                                          1,
+                                         0,
                                          false,
                                          false,
                                          false,
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
index 4d480451eb7..8f7c0a8a505 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
@@ -54,7 +54,8 @@ public:
                   _dbpath.path(),         // .path
                   &_cs,                   // .cs
                   "",                     // .extraOpenOptions
-                  1,                      // .cacheSizeGB
+                  1,                      // .cacheSizeMB
+                  0,                      // .maxCacheOverflowFileSizeMB
                   false,                  // .durable
                   false,                  // .ephemeral
                   false,                  // .repair
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
index 0cff392042e..9abdf85c6ca 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp
@@ -80,6 +80,7 @@ public:
                   &_cs,
                   extraStrings.toString(),
                   1,
+                  0,
                   false,
                   false,
                   false,