From a84cdf9178c4d64c556ab49bb15abf4e866e668f Mon Sep 17 00:00:00 2001 From: Max Hirschhorn Date: Wed, 16 Jan 2019 22:08:41 -0500 Subject: SERVER-39007 Use rhel62-large distro for InMemory's concurrency* tasks. * Changes resmoke.py to propagate --storageEngineCacheSizeGB as --inMemorySizeGB when running with the InMemory storage engine. * Changes the mongo shell to propagate TestData.storageEngineCacheSizeGB as --inMemorySizeGB when running with the InMemory storage engine. * Sets --storageEngineCacheSizeGB=4 when running with the InMemory storage engine in Evergreen. --- buildscripts/resmokelib/core/programs.py | 4 ++- etc/evergreen.yml | 37 ++++++++++++++++++++++----- jstests/noPassthrough/command_line_parsing.js | 2 ++ jstests/serial_run/memory.js | 16 ++++++++---- src/mongo/shell/servers.js | 6 +++++ 5 files changed, 53 insertions(+), 12 deletions(-) diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index f3518c75e8c..61805008452 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -107,7 +107,9 @@ def mongod_program( # pylint: disable=too-many-branches "wiredTigerIndexConfigString": config.WT_INDEX_CONFIG, } - if config.STORAGE_ENGINE == "rocksdb": + if config.STORAGE_ENGINE == "inMemory": + shortcut_opts["inMemorySizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE + elif config.STORAGE_ENGINE == "rocksdb": shortcut_opts["rocksdbCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE elif config.STORAGE_ENGINE == "wiredTiger" or config.STORAGE_ENGINE is None: shortcut_opts["wiredTigerCacheSizeGB"] = config.STORAGE_ENGINE_CACHE_SIZE diff --git a/etc/evergreen.yml b/etc/evergreen.yml index c5b2461f7a0..a26137de341 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1191,12 +1191,21 @@ functions: extra_args="$extra_args --continueOnFailure" fi - # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. - # resmoke will assign to the appropriate parameter on storage engines that support it. + # We reduce the storage engine's cache size to reduce the likelihood of a mongod process + # being killed by the OOM killer. The --storageEngineCacheSizeGB command line option is only + # filled in with a default value here if one hasn't already been specified in the task's + # definition or build variant's definition. set +o errexit - echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB + echo "${resmoke_args} ${test_flags}" | grep -q storageEngineCacheSizeGB if [ $? -eq 1 ]; then - extra_args="$extra_args --storageEngineCacheSizeGB=1" + echo "${resmoke_args} ${test_flags}" | grep -q "\-\-storageEngine=inMemory" + if [ $? -eq 0 ]; then + # We use a default of 4GB for the InMemory storage engine. + extra_args="$extra_args --storageEngineCacheSizeGB=4" + else + # We use a default of 1GB for all other storage engines. + extra_args="$extra_args --storageEngineCacheSizeGB=1" + fi fi set -o errexit @@ -13073,19 +13082,35 @@ buildvariants: distros: - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_replication + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_replication_causal_consistency distros: - - rhel62-large + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_replication + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_replication_with_balancer + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_causal_consistency + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_causal_consistency_and_balancer + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_with_stepdowns + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_sharded_with_stepdowns_and_balancer + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_simultaneous + distros: + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: concurrency_simultaneous_replication distros: - - rhel62-large + - rhel62-large # Some workloads require a lot of memory, use a bigger machine for this suite. - name: failpoints - name: failpoints_auth - name: gle_auth diff --git a/jstests/noPassthrough/command_line_parsing.js b/jstests/noPassthrough/command_line_parsing.js index c68b050a676..9e96dcab01a 100644 --- a/jstests/noPassthrough/command_line_parsing.js +++ b/jstests/noPassthrough/command_line_parsing.js @@ -32,6 +32,7 @@ delete m2result.parsed.net.transportLayer; delete m2result.parsed.storage.mmapv1; delete m2result.parsed.setParameter; delete m2result.parsed.storage.engine; +delete m2result.parsed.storage.inMemory; delete m2result.parsed.storage.journal; delete m2result.parsed.storage.rocksdb; delete m2result.parsed.storage.wiredTiger; @@ -60,6 +61,7 @@ delete m3result.parsed.net.transportLayer; delete m3result.parsed.storage.mmapv1; delete m3result.parsed.setParameter; delete m3result.parsed.storage.engine; +delete m3result.parsed.storage.inMemory; delete m3result.parsed.storage.journal; delete m3result.parsed.storage.rocksdb; delete m3result.parsed.storage.wiredTiger; diff --git a/jstests/serial_run/memory.js b/jstests/serial_run/memory.js index f9b125b24a9..e117f572dc2 100644 --- a/jstests/serial_run/memory.js +++ b/jstests/serial_run/memory.js @@ -1,12 +1,18 @@ var col = db.memoryTest; -var buildInfo = db.adminCommand("buildInfo"); +var buildInfo = assert.commandWorked(db.adminCommand("buildInfo")); +var serverStatus = assert.commandWorked(db.adminCommand("serverStatus")); + +// If mongod was compiled with the code coverage flag, then we reduce the length of some of the +// tests as they take an excessive amount of time. If the mongod is running with an in-memory +// storage engine, then we reduce the length of some of the tests to avoid an OOM due to the number +// of documents inserted. var codeCoverageVariant = buildInfo.buildEnvironment.ccflags.includes("-ftest-coverage"); -// If mongod was compiled with the code coverage flag, reduce some tests, as they take excessive -// time. +var inMemoryStorageEngine = !serverStatus.storageEngine.persistent; +var reduceNumLoops = codeCoverageVariant || inMemoryStorageEngine; // test creating many collections to make sure no internal cache goes OOM -var loopNum = codeCoverageVariant ? 100 : 10000; +var loopNum = reduceNumLoops ? 100 : 10000; for (var i = 0; i < loopNum; ++i) { name = "memoryTest" + i; if ((i % 1000) == 0) @@ -35,7 +41,7 @@ assert.throws(function() { doWhereTest(1000000000); }); -loopNum = codeCoverageVariant ? 10000 : 1000000; +loopNum = reduceNumLoops ? 10000 : 1000000; doWhereTest(loopNum); doWhereTest(loopNum); doWhereTest(loopNum); diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js index 61cb853173d..d636bdb470d 100644 --- a/src/mongo/shell/servers.js +++ b/src/mongo/shell/servers.js @@ -1177,6 +1177,12 @@ var MongoRunner, _startMongod, startMongoProgram, runMongoProgram, startMongoPro argArray.push(...['--rocksdbCacheSizeGB', jsTest.options().storageEngineCacheSizeGB]); } + } else if (jsTest.options().storageEngine === "inMemory") { + if (jsTest.options().storageEngineCacheSizeGB && + !argArrayContains("--inMemorySizeGB")) { + argArray.push( + ...["--inMemorySizeGB", jsTest.options().storageEngineCacheSizeGB]); + } } // apply setParameters for mongod. The 'setParameters' field should be given as // a plain JavaScript object, where each key is a parameter name and the value -- cgit v1.2.1