summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMax Hirschhorn <max.hirschhorn@mongodb.com>2018-07-10 01:39:36 -0400
committerMax Hirschhorn <max.hirschhorn@mongodb.com>2018-07-10 01:39:36 -0400
commita750bf210f70dd6e15cd65a15d50aeb8cd75fa3f (patch)
tree63b3d3213ce60f08625981f087392e0fd314e957
parent4c725a11acf11e41e8200500a03d3cec97a25dbe (diff)
downloadmongo-a750bf210f70dd6e15cd65a15d50aeb8cd75fa3f.tar.gz
SERVER-35537 Create mongoebench for running benchRun against mobile.
It take a JSON config file with a "pre" section for any setup logic and an "ops" section for the operations that the benchRun workers should execute repeatedly.
Diffstat
-rw-r--r--buildscripts/bypass_compile_and_fetch_binaries.py1
-rw-r--r--etc/evergreen.yml7
-rw-r--r--jstests/libs/mongoebench.js49
-rw-r--r--jstests/noPassthrough/mongoebench_test.js42
-rw-r--r--src/mongo/SConscript2
-rw-r--r--src/mongo/embedded/SConscript12
-rw-r--r--src/mongo/embedded/embedded_options_helpers.cpp62
-rw-r--r--src/mongo/embedded/embedded_options_helpers.h60
-rw-r--r--src/mongo/embedded/mongoed_main.cpp20
-rw-r--r--src/mongo/shell/SConscript13
-rw-r--r--src/mongo/shell/bench.cpp881
-rw-r--r--src/mongo/shell/bench.h33
-rw-r--r--src/mongo/shell/shell_utils.cpp11
-rw-r--r--src/mongo/tools/SConscript26
-rw-r--r--src/mongo/tools/mongoebench_main.cpp161
-rw-r--r--src/mongo/tools/mongoebench_options.cpp137
-rw-r--r--src/mongo/tools/mongoebench_options.h73
-rw-r--r--src/mongo/tools/mongoebench_options_init.cpp54
18 files changed, 1124 insertions, 520 deletions
diff --git a/buildscripts/bypass_compile_and_fetch_binaries.py b/buildscripts/bypass_compile_and_fetch_binaries.py
index 4bf307c8feb..831d864a582 100644
--- a/buildscripts/bypass_compile_and_fetch_binaries.py
+++ b/buildscripts/bypass_compile_and_fetch_binaries.py
@@ -269,6 +269,7 @@ def main(): # pylint: disable=too-many-locals,too-many-statements
extract_files = [
executable_name("dbtest"),
executable_name("mongobridge"),
+ executable_name("mongoebench"),
executable_name("mongoed"),
"build/integration_tests.txt",
]
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index c0c1977bcd1..f0a0dc05a19 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3257,6 +3257,7 @@ tasks:
- "./test*"
- "./dbtest*"
- "./mongobridge*"
+ - "./mongoebench*"
- "./mongoed*"
- "buildscripts/**"
- "*Example"
@@ -12201,7 +12202,7 @@ buildvariants:
scons_cache_scope: shared
gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go'
build_mongoreplay: true
- additional_targets: mongoed
+ additional_targets: mongoebench mongoed
display_tasks:
- *unittests
tasks:
@@ -12260,7 +12261,7 @@ buildvariants:
gorootvars: 'PATH="/opt/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/go1.8/go'
tooltags: -gccgoflags "$(pkg-config --libs)"
build_mongoreplay: true
- additional_targets: mongoed
+ additional_targets: mongoebench mongoed
display_tasks:
- *unittests
tasks:
@@ -12344,7 +12345,7 @@ buildvariants:
num_jobs_available: 1
gorootvars: 'PATH="/usr/local/go1.8/go/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/usr/local/go1.8/go CGO_CPPFLAGS=-I/opt/mongodbtoolchain/v2/include CGO_CFLAGS=-mmacosx-version-min=10.10 CGO_LDFLAGS=-mmacosx-version-min=10.10'
build_mongoreplay: true
- additional_targets: mongoed
+ additional_targets: mongoebench mongoed
display_tasks:
- *unittests
tasks:
diff --git a/jstests/libs/mongoebench.js b/jstests/libs/mongoebench.js
new file mode 100644
index 00000000000..f6feb4eb9f0
--- /dev/null
+++ b/jstests/libs/mongoebench.js
@@ -0,0 +1,49 @@
+"use strict";
+
+var {runMongoeBench} = (function() {
+
+ /**
+ * Spawns a mongoebench process with the specified options.
+ *
+ * If a plain JavaScript object is specified as the 'config' parameter, then it is serialized to
+ * a file as a JSON string which is then specified as the config file for the mongoebench
+ * process.
+ */
+ function runMongoeBench(config, options = {}) {
+ const args = ["mongoebench"];
+
+ if (typeof config === "object") {
+ const filename = MongoRunner.dataPath + "mongoebench_config.json";
+ writeFile(filename, tojson(config));
+ args.push(filename);
+ } else if (typeof config === "string") {
+ args.push(config);
+ } else {
+ throw new Error("'config' parameter must be a string or an object");
+ }
+
+ if (!options.hasOwnProperty("dbpath")) {
+ options.dbpath = MongoRunner.dataDir;
+ }
+
+ for (let key of Object.keys(options)) {
+ const value = options[key];
+ if (value === null || value === undefined) {
+ throw new Error(
+ "Value '" + value + "' for '" + key +
+ "' option is ambiguous; specify {flag: ''} to add --flag command line" +
+ " options'");
+ }
+
+ args.push("--" + key);
+ if (value !== "") {
+ args.push(value.toString());
+ }
+ }
+
+ const exitCode = _runMongoProgram(...args);
+ assert.eq(0, exitCode, "encountered an error in mongoebench");
+ }
+
+ return {runMongoeBench};
+})();
diff --git a/jstests/noPassthrough/mongoebench_test.js b/jstests/noPassthrough/mongoebench_test.js
new file mode 100644
index 00000000000..4e9c1fdc14e
--- /dev/null
+++ b/jstests/noPassthrough/mongoebench_test.js
@@ -0,0 +1,42 @@
+/**
+ * Tests for the mongoebench executable.
+ */
+(function() {
+ "use strict";
+
+ load("jstests/libs/mongoebench.js"); // for runMongoeBench
+
+ if (jsTest.options().storageEngine !== "mobile") {
+ print("Skipping test because storage engine isn't mobile");
+ return;
+ }
+
+ const dbpath = MongoRunner.dataPath + "mongoebench_test";
+ resetDbpath(dbpath);
+
+ // Test that the operations in the "pre" section of the configuration are run exactly once.
+ runMongoeBench( // Force clang-format to break this line.
+ {
+ pre: [{
+ op: "insert",
+ ns: "test.mongoebench_test",
+ doc: {pre: {"#SEQ_INT": {seq_id: 0, start: 0, step: 1, unique: true}}}
+ }],
+ ops: [{
+ op: "update",
+ ns: "test.mongoebench_test",
+ update: {$inc: {ops: 1}},
+ multi: true,
+ }]
+ },
+ {dbpath});
+
+ const conn = MongoRunner.runMongod({dbpath, noCleanData: true});
+ assert.neq(null, conn, "failed to start mongod after running mongoebench");
+
+ const db = conn.getDB("test");
+ const count = db.mongoebench_test.find().itcount();
+ assert.eq(1, count, "ops in 'pre' section ran more than once or didn't run at all");
+
+ MongoRunner.stopMongod(conn);
+})();
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index f00f59720b7..66e0eb45995 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -463,7 +463,6 @@ if not has_option('noshell') and usemozjs:
shell_core_env.Append(CPPDEFINES=["MONGO_SAFE_SHELL"])
shell_core_env.Library("shell_core",
source=[
- "shell/bench.cpp",
"shell/linenoise.cpp",
"shell/mk_wcwidth.cpp",
"shell/mongo-server.cpp",
@@ -482,6 +481,7 @@ if not has_option('noshell') and usemozjs:
'linenoise_utf8',
'rpc/protocol',
'scripting/scripting',
+ 'shell/benchrun',
'shell/mongojs',
'transport/message_compressor',
'transport/transport_layer_manager',
diff --git a/src/mongo/embedded/SConscript b/src/mongo/embedded/SConscript
index 73dc960d1c3..e90f08d6023 100644
--- a/src/mongo/embedded/SConscript
+++ b/src/mongo/embedded/SConscript
@@ -59,6 +59,17 @@ env.Library(
)
env.Library(
+ target='embedded_integration_helpers',
+ source=[
+ 'embedded_options_helpers.cpp',
+ ],
+ LIBDEPS=[
+ '$BUILD_DIR/mongo/base',
+ '$BUILD_DIR/mongo/util/options_parser/options_parser',
+ ],
+)
+
+env.Library(
target='mongo_embedded_capi',
source=[
'capi.cpp',
@@ -119,6 +130,7 @@ mongoed = yamlEnv.Program(
'$BUILD_DIR/mongo/transport/transport_layer_manager',
'$BUILD_DIR/mongo/util/signal_handlers',
'embedded',
+ 'embedded_integration_helpers',
],
INSTALL_ALIAS=[
'mobile-test',
diff --git a/src/mongo/embedded/embedded_options_helpers.cpp b/src/mongo/embedded/embedded_options_helpers.cpp
new file mode 100644
index 00000000000..c58f920bee1
--- /dev/null
+++ b/src/mongo/embedded/embedded_options_helpers.cpp
@@ -0,0 +1,62 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/embedded/embedded_options_helpers.h"
+
+#include "mongo/util/options_parser/options_parser.h"
+#include "mongo/util/options_parser/startup_options.h"
+
+#include <iterator>
+#include <map>
+#include <vector>
+
+namespace mongo {
+namespace embedded_integration_helpers {
+
+Status parseCommandLineOptions(int argc,
+ char* argv[],
+ const optionenvironment::OptionSection& startupOptions) {
+ // We manually run the options parsing code that's equivalent to the logic in the
+ // MONGO_INITIALIZERs for mongod. We cannot do this in initializers because embedded uses a
+ // different options format and we therefore need to have parsed the command line options before
+ // embedded::initialize() is called. However, as long as we store the options in the same place
+ // they will be valid for embedded too.
+ std::vector<std::string> args;
+ std::map<std::string, std::string> env;
+
+ args.reserve(argc);
+ std::copy(argv, argv + argc, std::back_inserter(args));
+
+ optionenvironment::OptionsParser parser;
+ return parser.run(startupOptions, args, env, &optionenvironment::startupOptionsParsed);
+}
+
+} // namespace embedded_integration_helpers
+} // namepsace mongo
diff --git a/src/mongo/embedded/embedded_options_helpers.h b/src/mongo/embedded/embedded_options_helpers.h
new file mode 100644
index 00000000000..fa46bd59050
--- /dev/null
+++ b/src/mongo/embedded/embedded_options_helpers.h
@@ -0,0 +1,60 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/status.h"
+
+namespace mongo {
+
+namespace optionenvironment {
+
+class OptionSection;
+
+} // namespace optionenvironment
+
+namespace embedded_integration_helpers {
+
+/**
+ * Parses the command line options represented by 'argc' and 'argv' into
+ * optionenvironment::startupOptionsParsed.
+ *
+ * In order to faciliate adding options which are supported by both embedded and mongod
+ * (e.g. dbpath), the caller is responsible for adding the options accepted by embedded prior to
+ * calling this function.
+ *
+ * The caller should also take care to ensure 'startupOptions' lives as long as the call to
+ * embedded::initialize() because optionenvironment::startupOptionsParsed holds references to
+ * Constraint instances owned by 'startupOptions'.
+ */
+Status parseCommandLineOptions(int argc,
+ char* argv[],
+ const optionenvironment::OptionSection& startupOptions);
+
+} // namespace embedded_integration_helpers
+} // namespace mongo
diff --git a/src/mongo/embedded/mongoed_main.cpp b/src/mongo/embedded/mongoed_main.cpp
index c43e95e09ef..09ea579fa5f 100644
--- a/src/mongo/embedded/mongoed_main.cpp
+++ b/src/mongo/embedded/mongoed_main.cpp
@@ -34,6 +34,8 @@
#include "mongo/db/mongod_options.h"
#include "mongo/db/service_context.h"
#include "mongo/embedded/embedded.h"
+#include "mongo/embedded/embedded_options.h"
+#include "mongo/embedded/embedded_options_helpers.h"
#include "mongo/embedded/service_entry_point_embedded.h"
#include "mongo/transport/service_entry_point_impl.h"
#include "mongo/transport/transport_layer.h"
@@ -102,22 +104,12 @@ int mongoedMain(int argc, char* argv[], char** envp) {
try {
optionenvironment::OptionSection startupOptions("Options");
+ // Adding all options mongod we don't have to maintain a separate set for this executable,
+ // some will be unused but that's fine as this is just an executable for testing purposes
+ // anyway.
uassertStatusOK(addMongodOptions(&startupOptions));
-
- // Manually run the code that's equivalent to the MONGO_INITIALIZERs for mongod. We can't do
- // this in initializers because embedded uses a different options format. However as long as
- // we store the options in the same place it will be valid for embedded too. Adding all
- // options mongod we don't have to maintain a separate set for this executable, some will be
- // unused but that's fine as this is just an executable for testing purposes anyway.
- std::vector<std::string> args;
- std::map<std::string, std::string> env;
-
- args.reserve(argc);
- std::copy(argv, argv + argc, std::back_inserter(args));
-
- optionenvironment::OptionsParser parser;
uassertStatusOK(
- parser.run(startupOptions, args, env, &optionenvironment::startupOptionsParsed));
+ embedded_integration_helpers::parseCommandLineOptions(argc, argv, startupOptions));
uassertStatusOK(storeMongodOptions(optionenvironment::startupOptionsParsed));
// Add embedded specific options that's not available in mongod here.
diff --git a/src/mongo/shell/SConscript b/src/mongo/shell/SConscript
index 3db44b18a52..7bdc2b580a2 100644
--- a/src/mongo/shell/SConscript
+++ b/src/mongo/shell/SConscript
@@ -4,6 +4,19 @@ Import("env")
env = env.Clone()
+env.Library(
+ target='benchrun',
+ source=[
+ 'bench.cpp',
+ ],
+ LIBDEPS=[
+ '$BUILD_DIR/mongo/base',
+ '$BUILD_DIR/mongo/client/clientdriver_minimal',
+ '$BUILD_DIR/mongo/db/logical_session_id',
+ '$BUILD_DIR/mongo/scripting/bson_template_evaluator',
+ ]
+)
+
generateJSErrorCodes = env.Command(
target=['error_codes.js'],
source=[
diff --git a/src/mongo/shell/bench.cpp b/src/mongo/shell/bench.cpp
index 8eef4653730..728c0abd94d 100644
--- a/src/mongo/shell/bench.cpp
+++ b/src/mongo/shell/bench.cpp
@@ -40,7 +40,6 @@
#include "mongo/db/query/getmore_request.h"
#include "mongo/db/query/query_request.h"
#include "mongo/scripting/bson_template_evaluator.h"
-#include "mongo/scripting/engine.h"
#include "mongo/stdx/thread.h"
#include "mongo/util/log.h"
#include "mongo/util/md5.h"
@@ -397,18 +396,6 @@ BenchRunOp opFromBson(const BSONObj& op) {
<< opType,
(opType == "find") || (opType == "query"));
myOp.batchSize = arg.numberInt();
- } else if (name == "check") {
- // check function gets thrown into a scoped function. Leaving that parsing in main loop.
- myOp.useCheck = true;
- myOp.check = arg;
- uassert(
- 34420,
- str::stream()
- << "Check field requires type CodeWScoe, Code, or String, instead its type is: "
- << typeName(myOp.check.type()),
- (myOp.check.type() == CodeWScope || myOp.check.type() == Code ||
- myOp.check.type() == String));
-
} else if (name == "command") {
// type needs to be command
uassert(34398,
@@ -754,14 +741,10 @@ void BenchRunConfig::initializeFromBson(const BSONObj& args) {
}
}
-std::unique_ptr<DBClientBase> BenchRunConfig::createConnection() const {
- const ConnectionString connectionString = uassertStatusOK(ConnectionString::parse(host));
-
- std::string errorMessage;
- std::unique_ptr<DBClientBase> connection(connectionString.connect("BenchRun", errorMessage));
- uassert(16158, errorMessage, connection);
+MONGO_DEFINE_SHIM(BenchRunConfig::createConnectionImpl);
- return connection;
+std::unique_ptr<DBClientBase> BenchRunConfig::createConnection() const {
+ return BenchRunConfig::createConnectionImpl(*this);
}
BenchRunState::BenchRunState(unsigned numWorkers)
@@ -885,498 +868,25 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) {
LogicalSessionIdToClient::parse(IDLParserErrorContext("lsid"), result["id"].Obj()));
}
- TxnNumber txnNumber = 0;
- bool inProgressMultiStatementTxn = false;
+ BenchRunOp::State opState(&_rng, &bsonTemplateEvaluator, &_statsBlackHole);
ON_BLOCK_EXIT([&] {
// Executing the transaction with a new txnNumber would end the previous transaction
// automatically, but we have to end the last transaction manually with an abort command.
- if (inProgressMultiStatementTxn) {
- abortTransaction(conn, lsid, txnNumber);
+ if (opState.inProgressMultiStatementTxn) {
+ abortTransaction(conn, lsid, opState.txnNumber);
}
});
- std::unique_ptr<Scope> scope{getGlobalScriptEngine()->newScopeForCurrentThread()};
- verify(scope.get());
-
while (!shouldStop()) {
for (const auto& op : _config->ops) {
if (shouldStop())
break;
- auto& stats = shouldCollectStats() ? _stats : _statsBlackHole;
- ScriptingFunction scopeFunc = 0;
- BSONObj scopeObj;
- if (op.useCheck) {
- auto check = op.check;
-
- if (check.type() == CodeWScope) {
- scopeFunc = scope->createFunction(check.codeWScopeCode());
- scopeObj = BSONObj(check.codeWScopeScopeDataUnsafe());
- } else {
- scopeFunc = scope->createFunction(check.valuestr());
- }
-
- scope->init(&scopeObj);
- invariant(scopeFunc);
- }
+ opState.stats = shouldCollectStats() ? &_stats : &_statsBlackHole;
try {
- switch (op.op) {
- case OpType::NOP:
- break;
- case OpType::CPULOAD: {
- // perform a tight multiplication loop. The
- // performance of this loop should be
- // predictable, and this operation can be used
- // to test underlying system variability.
- long long limit = 10000 * op.cpuFactor;
- // volatile used to ensure that loop is not optimized away
- volatile uint64_t result = 0; // NOLINT
- uint64_t x = 100;
- for (long long i = 0; i < limit; i++) {
- x *= 13;
- }
- result = x;
- } break;
- case OpType::FINDONE: {
- BSONObj fixedQuery = fixQuery(op.query, bsonTemplateEvaluator);
- BSONObj result;
- if (op.useReadCmd) {
- auto qr = stdx::make_unique<QueryRequest>(NamespaceString(op.ns));
- qr->setFilter(fixedQuery);
- qr->setProj(op.projection);
- qr->setLimit(1LL);
- qr->setWantMore(false);
- if (_config->useSnapshotReads) {
- qr->setReadConcern(readConcernSnapshot);
- }
- invariant(qr->validate());
-
- BenchRunEventTrace _bret(&stats.findOneCounter);
- boost::optional<TxnNumber> txnNumberForOp;
- if (_config->useSnapshotReads) {
- ++txnNumber;
- txnNumberForOp = txnNumber;
- inProgressMultiStatementTxn = true;
- }
- runQueryWithReadCommands(conn,
- lsid,
- txnNumberForOp,
- std::move(qr),
- Milliseconds(0),
- &result);
- } else {
- BenchRunEventTrace _bret(&stats.findOneCounter);
- result = conn->findOne(op.ns,
- fixedQuery,
- nullptr,
- DBClientCursor::QueryOptionLocal_forceOpQuery);
- }
-
- if (op.useCheck) {
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [findOne]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [findOne] : " << result;
- } break;
- case OpType::COMMAND: {
- bool ok;
- BSONObj result;
- {
- BenchRunEventTrace _bret(&stats.commandCounter);
- ok = runCommandWithSession(conn,
- op.ns,
- fixQuery(op.command, bsonTemplateEvaluator),
- op.options,
- lsid,
- &result);
- }
- if (!ok) {
- stats.errCount++;
- }
-
- if (!result["cursor"].eoo()) {
- // The command returned a cursor, so iterate all results.
- auto cursorResponse =
- uassertStatusOK(CursorResponse::parseFromBSON(result));
- int count = cursorResponse.getBatch().size();
- while (cursorResponse.getCursorId() != 0) {
- GetMoreRequest getMoreRequest(
- cursorResponse.getNSS(),
- cursorResponse.getCursorId(),
- boost::none, // batchSize
- boost::none, // maxTimeMS
- boost::none, // term
- boost::none); // lastKnownCommittedOpTime
- BSONObj getMoreCommandResult;
- uassert(ErrorCodes::CommandFailed,
- str::stream() << "getMore command failed; reply was: "
- << getMoreCommandResult,
- runCommandWithSession(conn,
- op.ns,
- getMoreRequest.toBSON(),
- kNoOptions,
- lsid,
- &getMoreCommandResult));
- cursorResponse = uassertStatusOK(
- CursorResponse::parseFromBSON(getMoreCommandResult));
- count += cursorResponse.getBatch().size();
- }
- // Just give the count to the check function.
- result = BSON("count" << count << "context" << op.context);
- }
-
- if (op.useCheck) {
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [command]"
- << causedBy(scope->getError());
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [command]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [command] : " << result;
- }
- } break;
- case OpType::FIND: {
- int count;
-
- BSONObj fixedQuery = fixQuery(op.query, bsonTemplateEvaluator);
-
- if (op.useReadCmd) {
- uassert(28824,
- "cannot use 'options' in combination with read commands",
- !op.options);
-
- auto qr = stdx::make_unique<QueryRequest>(NamespaceString(op.ns));
- qr->setFilter(fixedQuery);
- qr->setProj(op.projection);
- if (op.skip) {
- qr->setSkip(op.skip);
- }
- if (op.limit) {
- qr->setLimit(op.limit);
- }
- if (op.batchSize) {
- qr->setBatchSize(op.batchSize);
- }
- BSONObjBuilder readConcernBuilder;
- if (_config->useSnapshotReads) {
- readConcernBuilder.append("level", "snapshot");
- }
- if (op.useAClusterTimeWithinPastSeconds > 0) {
- invariant(_config->useSnapshotReads);
- // Get a random cluster time between the latest time and
- // 'useAClusterTimeWithinPastSeconds' in the past.
- Timestamp atClusterTime = getAClusterTimeSecondsInThePast(
- conn, _rng.nextInt32(op.useAClusterTimeWithinPastSeconds));
- readConcernBuilder.append("atClusterTime", atClusterTime);
- }
- qr->setReadConcern(readConcernBuilder.obj());
-
- invariant(qr->validate());
-
- BenchRunEventTrace _bret(&stats.queryCounter);
- boost::optional<TxnNumber> txnNumberForOp;
- if (_config->useSnapshotReads) {
- ++txnNumber;
- txnNumberForOp = txnNumber;
- inProgressMultiStatementTxn = true;
- }
-
- int delayBeforeGetMore = op.maxRandomMillisecondDelayBeforeGetMore
- ? _rng.nextInt32(op.maxRandomMillisecondDelayBeforeGetMore)
- : 0;
-
- count = runQueryWithReadCommands(conn,
- lsid,
- txnNumberForOp,
- std::move(qr),
- Milliseconds(delayBeforeGetMore),
- nullptr);
- } else {
- // Use special query function for exhaust query option.
- if (op.options & QueryOption_Exhaust) {
- BenchRunEventTrace _bret(&stats.queryCounter);
- stdx::function<void(const BSONObj&)> castedDoNothing(doNothing);
- count = conn->query(
- castedDoNothing,
- op.ns,
- fixedQuery,
- &op.projection,
- op.options | DBClientCursor::QueryOptionLocal_forceOpQuery);
- } else {
- BenchRunEventTrace _bret(&stats.queryCounter);
- std::unique_ptr<DBClientCursor> cursor(conn->query(
- op.ns,
- fixedQuery,
- op.limit,
- op.skip,
- &op.projection,
- op.options | DBClientCursor::QueryOptionLocal_forceOpQuery,
- op.batchSize));
- count = cursor->itcount();
- }
- }
-
- if (op.expected >= 0 && count != op.expected) {
- log() << "bench query on: " << op.ns << " expected: " << op.expected
- << " got: " << count;
- verify(false);
- }
-
- if (op.useCheck) {
- BSONObj thisValue = BSON("count" << count << "context" << op.context);
- int err = scope->invoke(scopeFunc, 0, &thisValue, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [find]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [query] : " << count;
- } break;
- case OpType::UPDATE: {
- BSONObj result;
- {
- BenchRunEventTrace _bret(&stats.updateCounter);
- BSONObj query = fixQuery(op.query, bsonTemplateEvaluator);
- BSONObj update = fixQuery(op.update, bsonTemplateEvaluator);
-
- if (op.useWriteCmd) {
- BSONObjBuilder builder;
- builder.append("update", nsToCollectionSubstring(op.ns));
- BSONArrayBuilder docBuilder(builder.subarrayStart("updates"));
- docBuilder.append(BSON(
- "q" << query << "u" << update << "multi" << op.multi << "upsert"
- << op.upsert));
- docBuilder.done();
- builder.append("writeConcern", op.writeConcern);
-
- boost::optional<TxnNumber> txnNumberForOp;
- if (_config->useIdempotentWrites) {
- ++txnNumber;
- txnNumberForOp = txnNumber;
- }
- runCommandWithSession(conn,
- nsToDatabaseSubstring(op.ns).toString(),
- builder.done(),
- kNoOptions,
- lsid,
- txnNumberForOp,
- &result);
- } else {
- auto toSend =
- makeUpdateMessage(op.ns,
- query,
- update,
- (op.upsert ? UpdateOption_Upsert : 0) |
- (op.multi ? UpdateOption_Multi : 0));
- conn->say(toSend);
- if (op.safe)
- result = conn->getLastErrorDetailed();
- }
- }
-
- if (op.safe) {
- if (op.useCheck) {
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [update]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [safe update] : " << result;
-
- if (!result["err"].eoo() && result["err"].type() == String &&
- (_config->throwGLE || op.throwGLE))
- uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
- (std::string) "From benchRun GLE" +
- causedBy(result["err"].String()));
- }
- } break;
- case OpType::INSERT: {
- BSONObj result;
- {
- BenchRunEventTrace _bret(&stats.insertCounter);
-
- BSONObj insertDoc;
- if (op.useWriteCmd) {
- BSONObjBuilder builder;
- builder.append("insert", nsToCollectionSubstring(op.ns));
- BSONArrayBuilder docBuilder(builder.subarrayStart("documents"));
- if (op.isDocAnArray) {
- for (const auto& element : op.doc) {
- insertDoc = fixQuery(element.Obj(), bsonTemplateEvaluator);
- docBuilder.append(insertDoc);
- }
- } else {
- insertDoc = fixQuery(op.doc, bsonTemplateEvaluator);
- docBuilder.append(insertDoc);
- }
- docBuilder.done();
- builder.append("writeConcern", op.writeConcern);
-
- boost::optional<TxnNumber> txnNumberForOp;
- if (_config->useIdempotentWrites) {
- ++txnNumber;
- txnNumberForOp = txnNumber;
- }
- runCommandWithSession(conn,
- nsToDatabaseSubstring(op.ns).toString(),
- builder.done(),
- kNoOptions,
- lsid,
- txnNumberForOp,
- &result);
- } else {
- std::vector<BSONObj> insertArray;
- if (op.isDocAnArray) {
- for (const auto& element : op.doc) {
- BSONObj e = fixQuery(element.Obj(), bsonTemplateEvaluator);
- insertArray.push_back(e);
- }
- } else {
- insertArray.push_back(fixQuery(op.doc, bsonTemplateEvaluator));
- }
-
- auto toSend = makeInsertMessage(
- op.ns, insertArray.data(), insertArray.size());
- conn->say(toSend);
-
- if (op.safe)
- result = conn->getLastErrorDetailed();
- }
- }
-
- if (op.safe) {
- if (op.useCheck) {
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [insert]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [safe insert] : " << result;
-
- if (!result["err"].eoo() && result["err"].type() == String &&
- (_config->throwGLE || op.throwGLE))
- uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
- (std::string) "From benchRun GLE" +
- causedBy(result["err"].String()));
- }
- } break;
- case OpType::REMOVE: {
- BSONObj result;
- {
- BenchRunEventTrace _bret(&stats.deleteCounter);
- BSONObj predicate = fixQuery(op.query, bsonTemplateEvaluator);
- if (op.useWriteCmd) {
- BSONObjBuilder builder;
- builder.append("delete", nsToCollectionSubstring(op.ns));
- BSONArrayBuilder docBuilder(builder.subarrayStart("deletes"));
- int limit = (op.multi == true) ? 0 : 1;
- docBuilder.append(BSON("q" << predicate << "limit" << limit));
- docBuilder.done();
- builder.append("writeConcern", op.writeConcern);
-
- boost::optional<TxnNumber> txnNumberForOp;
- if (_config->useIdempotentWrites) {
- ++txnNumber;
- txnNumberForOp = txnNumber;
- }
- runCommandWithSession(conn,
- nsToDatabaseSubstring(op.ns).toString(),
- builder.done(),
- kNoOptions,
- lsid,
- txnNumberForOp,
- &result);
- } else {
- auto toSend = makeRemoveMessage(
- op.ns, predicate, op.multi ? 0 : RemoveOption_JustOne);
- conn->say(toSend);
- if (op.safe)
- result = conn->getLastErrorDetailed();
- }
- }
-
- if (op.safe) {
- if (op.useCheck) {
- int err = scope->invoke(scopeFunc, 0, &result, 1000 * 60, false);
- if (err) {
- log() << "Error checking in benchRun thread [delete]"
- << causedBy(scope->getError());
-
- stats.errCount++;
-
- return;
- }
- }
-
- if (!_config->hideResults || op.showResult)
- log() << "Result from benchRun thread [safe remove] : " << result;
-
- if (!result["err"].eoo() && result["err"].type() == String &&
- (_config->throwGLE || op.throwGLE))
- uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
- (std::string) "From benchRun GLE " +
- causedBy(result["err"].String()));
- }
- } break;
- case OpType::CREATEINDEX:
- conn->createIndex(op.ns, op.key);
- break;
- case OpType::DROPINDEX:
- conn->dropIndex(op.ns, op.key);
- break;
- case OpType::LET: {
- BSONObjBuilder templateBuilder;
- bsonTemplateEvaluator.evaluate(op.value, templateBuilder);
- bsonTemplateEvaluator.setVariable(op.target,
- templateBuilder.done().firstElement());
- } break;
- default:
- uassert(34397, "In benchRun loop and got unknown op type", false);
- }
-
- // Count 1 for total ops. Successfully got through the try phrase
- stats.opCount++;
+ op.executeOnce(conn, lsid, *_config, &opState);
} catch (const DBException& ex) {
if (!_config->hideErrors || op.showError) {
bool yesWatch =
@@ -1401,7 +911,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) {
(!_config->noTrapPattern && _config->trapPattern && yesTrap) ||
(_config->trapPattern && _config->noTrapPattern && yesTrap && !noTrap)) {
{
- stats.trappedErrors.push_back(
+ opState.stats->trappedErrors.push_back(
BSON("error" << ex.what() << "op" << kOpTypeNames.find(op.op)->second
<< "count"
<< count));
@@ -1414,7 +924,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) {
sleepFor(_config->delayMillisOnFailedOperation);
- stats.errCount++;
+ ++opState.stats->errCount;
} catch (...) {
if (!_config->hideErrors || op.showError)
log() << "Error in benchRun thread caused by unknown error for op "
@@ -1422,7 +932,7 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) {
if (!_config->handleErrors && !op.handleError)
return;
- stats.errCount++;
+ ++opState.stats->errCount;
}
if (++count % 100 == 0 && !op.useWriteCmd) {
@@ -1437,6 +947,375 @@ void BenchRunWorker::generateLoadOnConnection(DBClientBase* conn) {
conn->getLastError();
}
+void BenchRunOp::executeOnce(DBClientBase* conn,
+ const boost::optional<LogicalSessionIdToClient>& lsid,
+ const BenchRunConfig& config,
+ State* state) const {
+ switch (this->op) {
+ case OpType::NOP:
+ break;
+ case OpType::CPULOAD: {
+ // perform a tight multiplication loop. The
+ // performance of this loop should be
+ // predictable, and this operation can be used
+ // to test underlying system variability.
+ long long limit = 10000 * this->cpuFactor;
+ // volatile used to ensure that loop is not optimized away
+ volatile uint64_t result = 0; // NOLINT
+ uint64_t x = 100;
+ for (long long i = 0; i < limit; i++) {
+ x *= 13;
+ }
+ result = x;
+ } break;
+ case OpType::FINDONE: {
+ BSONObj fixedQuery = fixQuery(this->query, *state->bsonTemplateEvaluator);
+ BSONObj result;
+ if (this->useReadCmd) {
+ auto qr = stdx::make_unique<QueryRequest>(NamespaceString(this->ns));
+ qr->setFilter(fixedQuery);
+ qr->setProj(this->projection);
+ qr->setLimit(1LL);
+ qr->setWantMore(false);
+ if (config.useSnapshotReads) {
+ qr->setReadConcern(readConcernSnapshot);
+ }
+ invariant(qr->validate());
+
+ BenchRunEventTrace _bret(&state->stats->findOneCounter);
+ boost::optional<TxnNumber> txnNumberForOp;
+ if (config.useSnapshotReads) {
+ ++state->txnNumber;
+ txnNumberForOp = state->txnNumber;
+ state->inProgressMultiStatementTxn = true;
+ }
+ runQueryWithReadCommands(
+ conn, lsid, txnNumberForOp, std::move(qr), Milliseconds(0), &result);
+ } else {
+ BenchRunEventTrace _bret(&state->stats->findOneCounter);
+ result = conn->findOne(
+ this->ns, fixedQuery, nullptr, DBClientCursor::QueryOptionLocal_forceOpQuery);
+ }
+
+ if (!config.hideResults || this->showResult)
+ log() << "Result from benchRun thread [findOne] : " << result;
+ } break;
+ case OpType::COMMAND: {
+ bool ok;
+ BSONObj result;
+ {
+ BenchRunEventTrace _bret(&state->stats->commandCounter);
+ ok = runCommandWithSession(conn,
+ this->ns,
+ fixQuery(this->command, *state->bsonTemplateEvaluator),
+ this->options,
+ lsid,
+ &result);
+ }
+ if (!ok) {
+ ++state->stats->errCount;
+ }
+
+ if (!result["cursor"].eoo()) {
+ // The command returned a cursor, so iterate all results.
+ auto cursorResponse = uassertStatusOK(CursorResponse::parseFromBSON(result));
+ int count = cursorResponse.getBatch().size();
+ while (cursorResponse.getCursorId() != 0) {
+ GetMoreRequest getMoreRequest(cursorResponse.getNSS(),
+ cursorResponse.getCursorId(),
+ boost::none, // batchSize
+ boost::none, // maxTimeMS
+ boost::none, // term
+ boost::none); // lastKnownCommittedOpTime
+ BSONObj getMoreCommandResult;
+ uassert(ErrorCodes::CommandFailed,
+ str::stream() << "getMore command failed; reply was: "
+ << getMoreCommandResult,
+ runCommandWithSession(conn,
+ this->ns,
+ getMoreRequest.toBSON(),
+ kNoOptions,
+ lsid,
+ &getMoreCommandResult));
+ cursorResponse =
+ uassertStatusOK(CursorResponse::parseFromBSON(getMoreCommandResult));
+ count += cursorResponse.getBatch().size();
+ }
+ // Just give the count to the check function.
+ result = BSON("count" << count << "context" << this->context);
+ }
+ } break;
+ case OpType::FIND: {
+ int count;
+
+ BSONObj fixedQuery = fixQuery(this->query, *state->bsonTemplateEvaluator);
+
+ if (this->useReadCmd) {
+ uassert(28824,
+ "cannot use 'options' in combination with read commands",
+ !this->options);
+
+ auto qr = stdx::make_unique<QueryRequest>(NamespaceString(this->ns));
+ qr->setFilter(fixedQuery);
+ qr->setProj(this->projection);
+ if (this->skip) {
+ qr->setSkip(this->skip);
+ }
+ if (this->limit) {
+ qr->setLimit(this->limit);
+ }
+ if (this->batchSize) {
+ qr->setBatchSize(this->batchSize);
+ }
+ BSONObjBuilder readConcernBuilder;
+ if (config.useSnapshotReads) {
+ readConcernBuilder.append("level", "snapshot");
+ }
+ if (this->useAClusterTimeWithinPastSeconds > 0) {
+ invariant(config.useSnapshotReads);
+ // Get a random cluster time between the latest time and
+ // 'useAClusterTimeWithinPastSeconds' in the past.
+ Timestamp atClusterTime = getAClusterTimeSecondsInThePast(
+ conn, state->rng->nextInt32(this->useAClusterTimeWithinPastSeconds));
+ readConcernBuilder.append("atClusterTime", atClusterTime);
+ }
+ qr->setReadConcern(readConcernBuilder.obj());
+
+ invariant(qr->validate());
+
+ BenchRunEventTrace _bret(&state->stats->queryCounter);
+ boost::optional<TxnNumber> txnNumberForOp;
+ if (config.useSnapshotReads) {
+ ++state->txnNumber;
+ txnNumberForOp = state->txnNumber;
+ state->inProgressMultiStatementTxn = true;
+ }
+
+ int delayBeforeGetMore = this->maxRandomMillisecondDelayBeforeGetMore
+ ? state->rng->nextInt32(this->maxRandomMillisecondDelayBeforeGetMore)
+ : 0;
+
+ count = runQueryWithReadCommands(conn,
+ lsid,
+ txnNumberForOp,
+ std::move(qr),
+ Milliseconds(delayBeforeGetMore),
+ nullptr);
+ } else {
+ // Use special query function for exhaust query option.
+ if (this->options & QueryOption_Exhaust) {
+ BenchRunEventTrace _bret(&state->stats->queryCounter);
+ stdx::function<void(const BSONObj&)> castedDoNothing(doNothing);
+ count =
+ conn->query(castedDoNothing,
+ this->ns,
+ fixedQuery,
+ &this->projection,
+ this->options | DBClientCursor::QueryOptionLocal_forceOpQuery);
+ } else {
+ BenchRunEventTrace _bret(&state->stats->queryCounter);
+ std::unique_ptr<DBClientCursor> cursor(
+ conn->query(this->ns,
+ fixedQuery,
+ this->limit,
+ this->skip,
+ &this->projection,
+ this->options | DBClientCursor::QueryOptionLocal_forceOpQuery,
+ this->batchSize));
+ count = cursor->itcount();
+ }
+ }
+
+ if (this->expected >= 0 && count != this->expected) {
+ log() << "bench query on: " << this->ns << " expected: " << this->expected
+ << " got: " << count;
+ verify(false);
+ }
+
+ if (!config.hideResults || this->showResult)
+ log() << "Result from benchRun thread [query] : " << count;
+ } break;
+ case OpType::UPDATE: {
+ BSONObj result;
+ {
+ BenchRunEventTrace _bret(&state->stats->updateCounter);
+ BSONObj query = fixQuery(this->query, *state->bsonTemplateEvaluator);
+ BSONObj update = fixQuery(this->update, *state->bsonTemplateEvaluator);
+
+ if (this->useWriteCmd) {
+ BSONObjBuilder builder;
+ builder.append("update", nsToCollectionSubstring(this->ns));
+ BSONArrayBuilder docBuilder(builder.subarrayStart("updates"));
+ docBuilder.append(BSON("q" << query << "u" << update << "multi" << this->multi
+ << "upsert"
+ << this->upsert));
+ docBuilder.done();
+ builder.append("writeConcern", this->writeConcern);
+
+ boost::optional<TxnNumber> txnNumberForOp;
+ if (config.useIdempotentWrites) {
+ ++state->txnNumber;
+ txnNumberForOp = state->txnNumber;
+ }
+ runCommandWithSession(conn,
+ nsToDatabaseSubstring(this->ns).toString(),
+ builder.done(),
+ kNoOptions,
+ lsid,
+ txnNumberForOp,
+ &result);
+ } else {
+ auto toSend = makeUpdateMessage(this->ns,
+ query,
+ update,
+ (this->upsert ? UpdateOption_Upsert : 0) |
+ (this->multi ? UpdateOption_Multi : 0));
+ conn->say(toSend);
+ if (this->safe)
+ result = conn->getLastErrorDetailed();
+ }
+ }
+
+ if (this->safe) {
+ if (!config.hideResults || this->showResult)
+ log() << "Result from benchRun thread [safe update] : " << result;
+
+ if (!result["err"].eoo() && result["err"].type() == String &&
+ (config.throwGLE || this->throwGLE))
+ uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
+ (std::string) "From benchRun GLE" + causedBy(result["err"].String()));
+ }
+ } break;
+ case OpType::INSERT: {
+ BSONObj result;
+ {
+ BenchRunEventTrace _bret(&state->stats->insertCounter);
+
+ BSONObj insertDoc;
+ if (this->useWriteCmd) {
+ BSONObjBuilder builder;
+ builder.append("insert", nsToCollectionSubstring(this->ns));
+ BSONArrayBuilder docBuilder(builder.subarrayStart("documents"));
+ if (this->isDocAnArray) {
+ for (const auto& element : this->doc) {
+ insertDoc = fixQuery(element.Obj(), *state->bsonTemplateEvaluator);
+ docBuilder.append(insertDoc);
+ }
+ } else {
+ insertDoc = fixQuery(this->doc, *state->bsonTemplateEvaluator);
+ docBuilder.append(insertDoc);
+ }
+ docBuilder.done();
+ builder.append("writeConcern", this->writeConcern);
+
+ boost::optional<TxnNumber> txnNumberForOp;
+ if (config.useIdempotentWrites) {
+ ++state->txnNumber;
+ txnNumberForOp = state->txnNumber;
+ }
+ runCommandWithSession(conn,
+ nsToDatabaseSubstring(this->ns).toString(),
+ builder.done(),
+ kNoOptions,
+ lsid,
+ txnNumberForOp,
+ &result);
+ } else {
+ std::vector<BSONObj> insertArray;
+ if (this->isDocAnArray) {
+ for (const auto& element : this->doc) {
+ BSONObj e = fixQuery(element.Obj(), *state->bsonTemplateEvaluator);
+ insertArray.push_back(e);
+ }
+ } else {
+ insertArray.push_back(fixQuery(this->doc, *state->bsonTemplateEvaluator));
+ }
+
+ auto toSend =
+ makeInsertMessage(this->ns, insertArray.data(), insertArray.size());
+ conn->say(toSend);
+
+ if (this->safe)
+ result = conn->getLastErrorDetailed();
+ }
+ }
+
+ if (this->safe) {
+ if (!config.hideResults || this->showResult)
+ log() << "Result from benchRun thread [safe insert] : " << result;
+
+ if (!result["err"].eoo() && result["err"].type() == String &&
+ (config.throwGLE || this->throwGLE))
+ uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
+ (std::string) "From benchRun GLE" + causedBy(result["err"].String()));
+ }
+ } break;
+ case OpType::REMOVE: {
+ BSONObj result;
+ {
+ BenchRunEventTrace _bret(&state->stats->deleteCounter);
+ BSONObj predicate = fixQuery(this->query, *state->bsonTemplateEvaluator);
+ if (this->useWriteCmd) {
+ BSONObjBuilder builder;
+ builder.append("delete", nsToCollectionSubstring(this->ns));
+ BSONArrayBuilder docBuilder(builder.subarrayStart("deletes"));
+ int limit = (this->multi == true) ? 0 : 1;
+ docBuilder.append(BSON("q" << predicate << "limit" << limit));
+ docBuilder.done();
+ builder.append("writeConcern", this->writeConcern);
+
+ boost::optional<TxnNumber> txnNumberForOp;
+ if (config.useIdempotentWrites) {
+ ++state->txnNumber;
+ txnNumberForOp = state->txnNumber;
+ }
+ runCommandWithSession(conn,
+ nsToDatabaseSubstring(this->ns).toString(),
+ builder.done(),
+ kNoOptions,
+ lsid,
+ txnNumberForOp,
+ &result);
+ } else {
+ auto toSend = makeRemoveMessage(
+ this->ns, predicate, this->multi ? 0 : RemoveOption_JustOne);
+ conn->say(toSend);
+ if (this->safe)
+ result = conn->getLastErrorDetailed();
+ }
+ }
+
+ if (this->safe) {
+ if (!config.hideResults || this->showResult)
+ log() << "Result from benchRun thread [safe remove] : " << result;
+
+ if (!result["err"].eoo() && result["err"].type() == String &&
+ (config.throwGLE || this->throwGLE))
+ uasserted(result["code"].eoo() ? 0 : result["code"].Int(),
+ (std::string) "From benchRun GLE " +
+ causedBy(result["err"].String()));
+ }
+ } break;
+ case OpType::CREATEINDEX:
+ conn->createIndex(this->ns, this->key);
+ break;
+ case OpType::DROPINDEX:
+ conn->dropIndex(this->ns, this->key);
+ break;
+ case OpType::LET: {
+ BSONObjBuilder templateBuilder;
+ state->bsonTemplateEvaluator->evaluate(this->value, templateBuilder);
+ state->bsonTemplateEvaluator->setVariable(this->target,
+ templateBuilder.done().firstElement());
+ } break;
+ default:
+ uassert(34397, "In benchRun loop and got unknown op type", false);
+ }
+
+ // Count 1 for total ops. Successfully got through the try phrase
+ ++state->stats->opCount;
+}
+
void BenchRunWorker::run() {
try {
auto conn(_config->createConnection());
diff --git a/src/mongo/shell/bench.h b/src/mongo/shell/bench.h
index c335af687a1..88314da99e2 100644
--- a/src/mongo/shell/bench.h
+++ b/src/mongo/shell/bench.h
@@ -31,8 +31,10 @@
#include <boost/optional.hpp>
#include <string>
+#include "mongo/base/shim.h"
#include "mongo/client/dbclient_base.h"
#include "mongo/db/jsobj.h"
+#include "mongo/db/logical_session_id.h"
#include "mongo/platform/atomic_word.h"
#include "mongo/stdx/condition_variable.h"
#include "mongo/stdx/mutex.h"
@@ -59,11 +61,35 @@ enum class OpType {
CPULOAD
};
+class BenchRunConfig;
+struct BenchRunStats;
+class BsonTemplateEvaluator;
+class LogicalSessionIdToClient;
+
/**
* Object representing one operation passed to benchRun
*/
struct BenchRunOp {
-public:
+ struct State {
+ State(PseudoRandom* rng_,
+ BsonTemplateEvaluator* bsonTemplateEvaluator_,
+ BenchRunStats* stats_)
+ : rng(rng_), bsonTemplateEvaluator(bsonTemplateEvaluator_), stats(stats_) {}
+
+ PseudoRandom* rng;
+ BsonTemplateEvaluator* bsonTemplateEvaluator;
+ BenchRunStats* stats;
+
+ // Transaction state
+ TxnNumber txnNumber = 0;
+ bool inProgressMultiStatementTxn = false;
+ };
+
+ void executeOnce(DBClientBase* conn,
+ const boost::optional<LogicalSessionIdToClient>& lsid,
+ const BenchRunConfig& config,
+ State* state) const;
+
int batchSize = 0;
BSONElement check;
BSONObj command;
@@ -129,6 +155,9 @@ public:
*/
static BenchRunConfig* createFromBson(const BSONObj& args);
+ static MONGO_DECLARE_SHIM((const BenchRunConfig&)->std::unique_ptr<DBClientBase>)
+ createConnectionImpl;
+
BenchRunConfig();
void initializeFromBson(const BSONObj& args);
@@ -218,6 +247,8 @@ public:
bool breakOnTrap;
private:
+ static std::function<std::unique_ptr<DBClientBase>(const BenchRunConfig&)> _factory;
+
/// Initialize a config object to its default values.
void initializeToDefaults();
};
diff --git a/src/mongo/shell/shell_utils.cpp b/src/mongo/shell/shell_utils.cpp
index 0d4b118dc70..77b1b6d9d99 100644
--- a/src/mongo/shell/shell_utils.cpp
+++ b/src/mongo/shell/shell_utils.cpp
@@ -62,6 +62,17 @@ extern const JSFile replsettest;
extern const JSFile bridge;
}
+MONGO_REGISTER_SHIM(BenchRunConfig::createConnectionImpl)
+(const BenchRunConfig& config)->std::unique_ptr<DBClientBase> {
+ const ConnectionString connectionString = uassertStatusOK(ConnectionString::parse(config.host));
+
+ std::string errorMessage;
+ std::unique_ptr<DBClientBase> connection(connectionString.connect("BenchRun", errorMessage));
+ uassert(16158, errorMessage, connection);
+
+ return connection;
+}
+
namespace shell_utils {
std::string _dbConnect;
diff --git a/src/mongo/tools/SConscript b/src/mongo/tools/SConscript
index dc4bfe408c7..f2226009f85 100644
--- a/src/mongo/tools/SConscript
+++ b/src/mongo/tools/SConscript
@@ -5,6 +5,9 @@ Import('get_option')
env = env.Clone()
+yamlEnv = env.Clone()
+yamlEnv.InjectThirdPartyIncludePaths(libraries=['yaml'])
+
mongobridge = env.Program(
target="mongobridge",
source=[
@@ -29,6 +32,29 @@ mongobridge = env.Program(
],
)
+mongoebench = yamlEnv.Program(
+ target='mongoebench',
+ source=[
+ 'mongoebench_main.cpp',
+ 'mongoebench_options.cpp',
+ 'mongoebench_options_init.cpp',
+ ],
+ LIBDEPS=[
+ '$BUILD_DIR/mongo/db/dbdirectclient',
+ '$BUILD_DIR/mongo/embedded/embedded',
+ '$BUILD_DIR/mongo/embedded/embedded_integration_helpers',
+ '$BUILD_DIR/mongo/shell/benchrun',
+ '$BUILD_DIR/mongo/util/signal_handlers',
+ ],
+ INSTALL_ALIAS=[
+ 'mobile-test',
+ 'tools',
+ ],
+)
+
hygienic = get_option('install-mode') == 'hygienic'
if not hygienic:
env.Install("#/", mongobridge)
+ env.Install("#/", mongoebench)
+
+env.Alias('all', mongoebench) # This ensures it compiles and links, but doesn't copy it anywhere.
diff --git a/src/mongo/tools/mongoebench_main.cpp b/src/mongo/tools/mongoebench_main.cpp
new file mode 100644
index 00000000000..ef00204a0b4
--- /dev/null
+++ b/src/mongo/tools/mongoebench_main.cpp
@@ -0,0 +1,161 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kDefault
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/base/init.h"
+#include "mongo/db/dbdirectclient.h"
+#include "mongo/db/service_context.h"
+#include "mongo/embedded/embedded.h"
+#include "mongo/embedded/embedded_options.h"
+#include "mongo/embedded/embedded_options_helpers.h"
+#include "mongo/scripting/bson_template_evaluator.h"
+#include "mongo/shell/bench.h"
+#include "mongo/tools/mongoebench_options.h"
+#include "mongo/util/exit.h"
+#include "mongo/util/log.h"
+#include "mongo/util/options_parser/option_section.h"
+#include "mongo/util/signal_handlers.h"
+#include "mongo/util/text.h"
+
+namespace mongo {
+namespace {
+
+/**
+ * DBDirectClientWithOwnOpCtx is a version of DBDirectClient that owns its own OperationContext.
+ *
+ * Since benchRun originally existed only in the mongo shell and was used with only
+ * DBClientConnections, there isn't a part of the BenchRunConfig or BenchRunWorker interfaces that
+ * is aware of having an OperationContext. In particular, the mongo shell lacks a ServiceContext and
+ * therefore also lacks the rest of the Client and OperationContext hierarchy. We shove an
+ * OperationContext onto the DBDirectClient to work around this limitation for mongoebench to work.
+ */
+class DBDirectClientWithOwnOpCtx : public DBDirectClient {
+public:
+ DBDirectClientWithOwnOpCtx() : DBDirectClient(nullptr) {
+ Client::initThreadIfNotAlready();
+ _opCtx = cc().makeOperationContext();
+ setOpCtx(_opCtx.get());
+ }
+
+private:
+ ServiceContext::UniqueOperationContext _opCtx;
+};
+
+MONGO_INITIALIZER_WITH_PREREQUISITES(SignalProcessingStartup, ("ThreadNameInitializer"))
+(InitializerContext*) {
+ // Make sure we call this as soon as possible but before any other threads are started. Before
+ // embedded::initialize is too early and after is too late. So instead we hook in during the
+ // global initialization at the right place.
+ startSignalProcessingThread();
+ return Status::OK();
+}
+
+int mongoeBenchMain(int argc, char* argv[], char** envp) {
+ ServiceContext* serviceContext = nullptr;
+
+ registerShutdownTask([&]() {
+ if (serviceContext) {
+ embedded::shutdown(serviceContext);
+ }
+ });
+
+ setupSignalHandlers();
+
+ log() << "MongoDB embedded benchRun application, for testing purposes only";
+
+ try {
+ optionenvironment::OptionSection startupOptions("Options");
+ uassertStatusOK(embedded::addOptions(&startupOptions));
+ uassertStatusOK(addMongoeBenchOptions(&startupOptions));
+ uassertStatusOK(
+ embedded_integration_helpers::parseCommandLineOptions(argc, argv, startupOptions));
+ serviceContext = embedded::initialize(nullptr);
+ } catch (const std::exception& ex) {
+ error() << ex.what();
+ return EXIT_BADOPTIONS;
+ }
+
+ // If a "pre" section was present in the benchRun config file, then we run its operations once
+ // before running the operations from the "ops" section.
+ if (mongoeBenchGlobalParams.preConfig) {
+ auto conn = mongoeBenchGlobalParams.preConfig->createConnection();
+ boost::optional<LogicalSessionIdToClient> lsid;
+
+ PseudoRandom rng(mongoeBenchGlobalParams.preConfig->randomSeed);
+ BsonTemplateEvaluator bsonTemplateEvaluator(mongoeBenchGlobalParams.preConfig->randomSeed);
+ BenchRunStats stats;
+ BenchRunOp::State state(&rng, &bsonTemplateEvaluator, &stats);
+
+ for (auto&& op : mongoeBenchGlobalParams.preConfig->ops) {
+ op.executeOnce(conn.get(), lsid, *mongoeBenchGlobalParams.preConfig, &state);
+ }
+ }
+
+ // If an "ops" section was present in the benchRun config file, then we repeatedly run its
+ // operations across the configured number of threads for the configured number of seconds.
+ if (mongoeBenchGlobalParams.opsConfig) {
+ const double seconds = mongoeBenchGlobalParams.opsConfig->seconds;
+ auto runner = std::make_unique<BenchRunner>(mongoeBenchGlobalParams.opsConfig.release());
+ runner->start();
+
+ sleepmillis(static_cast<long long>(seconds * 1000));
+
+ BSONObj stats = BenchRunner::finish(runner.release());
+ log() << "stats: " << stats;
+ }
+
+ shutdown(EXIT_CLEAN);
+}
+
+} // namespace
+
+MONGO_REGISTER_SHIM(BenchRunConfig::createConnectionImpl)
+(const BenchRunConfig& config)->std::unique_ptr<DBClientBase> {
+ return std::unique_ptr<DBClientBase>(new DBDirectClientWithOwnOpCtx());
+}
+
+} // namespace mongo
+
+#if defined(_WIN32)
+// In Windows, wmain() is an alternate entry point for main(), and receives the same parameters as
+// main() but encoded in Windows Unicode (UTF-16); "wide" 16-bit wchar_t characters. The
+// WindowsCommandLine object converts these wide character strings to a UTF-8 coded equivalent and
+// makes them available through the argv() and envp() members. This enables mongoeBenchMain() to
+// process UTF-8 encoded arguments and environment variables without regard to platform.
+int wmain(int argc, wchar_t* argvW[], wchar_t* envpW[]) {
+ mongo::WindowsCommandLine wcl(argc, argvW, envpW);
+ return mongo::mongoeBenchMain(argc, wcl.argv(), wcl.envp());
+}
+#else
+int main(int argc, char* argv[], char** envp) {
+ return mongo::mongoeBenchMain(argc, argv, envp);
+}
+#endif
diff --git a/src/mongo/tools/mongoebench_options.cpp b/src/mongo/tools/mongoebench_options.cpp
new file mode 100644
index 00000000000..1cfd950fe0d
--- /dev/null
+++ b/src/mongo/tools/mongoebench_options.cpp
@@ -0,0 +1,137 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/tools/mongoebench_options.h"
+
+#include <algorithm>
+#include <fstream>
+#include <iostream>
+#include <iterator>
+
+#include "mongo/base/status.h"
+#include "mongo/db/storage/storage_options.h"
+#include "mongo/platform/random.h"
+#include "mongo/shell/bench.h"
+#include "mongo/util/mongoutils/str.h"
+#include "mongo/util/options_parser/startup_options.h"
+
+namespace mongo {
+
+MongoeBenchGlobalParams mongoeBenchGlobalParams;
+
+Status addMongoeBenchOptions(moe::OptionSection* options) {
+ options->addOptionChaining("help", "help", moe::Switch, "show this usage information");
+
+ options
+ ->addOptionChaining(
+ "benchRunConfigFile", "benchRunConfigFile", moe::String, "config file for benchRun")
+ .hidden()
+ .positional(1, 1);
+
+ options->addOptionChaining("seed", "seed", moe::Long, "random seed to use");
+
+ options
+ ->addOptionChaining(
+ "threads", "threads,t", moe::Unsigned, "number of benchRun worker threads")
+ .setDefault(moe::Value(1U));
+
+ options->addOptionChaining("time", "time,s", moe::Double, "seconds to run benchRun for")
+ .setDefault(moe::Value(1.0));
+
+ return Status::OK();
+}
+
+void printMongoeBenchHelp(std::ostream* out) {
+ *out << "Usage: mongoebench <config file> [options]" << std::endl;
+ *out << moe::startupOptions.helpString();
+ *out << std::flush;
+}
+
+bool handlePreValidationMongoeBenchOptions(const moe::Environment& params) {
+ if (params.count("help")) {
+ printMongoeBenchHelp(&std::cout);
+ return false;
+ }
+ return true;
+}
+
+namespace {
+
+BSONObj getBsonFromJsonFile(const std::string& filename) {
+ std::ifstream infile(filename.c_str());
+ std::string data((std::istreambuf_iterator<char>(infile)), std::istreambuf_iterator<char>());
+ return fromjson(data);
+}
+
+} // namespace
+
+Status storeMongoeBenchOptions(const moe::Environment& params,
+ const std::vector<std::string>& args) {
+ if (!params.count("benchRunConfigFile")) {
+ return {ErrorCodes::BadValue, "No benchRun config file was specified"};
+ }
+
+ BSONObj config = getBsonFromJsonFile(params["benchRunConfigFile"].as<std::string>());
+ for (auto&& elem : config) {
+ const auto fieldName = elem.fieldNameStringData();
+ if (fieldName == "pre") {
+ mongoeBenchGlobalParams.preConfig.reset(
+ BenchRunConfig::createFromBson(elem.wrap("ops")));
+ } else if (fieldName == "ops") {
+ mongoeBenchGlobalParams.opsConfig.reset(BenchRunConfig::createFromBson(elem.wrap()));
+ } else {
+ return {ErrorCodes::BadValue,
+ str::stream() << "Unrecognized key in benchRun config file: " << fieldName};
+ }
+ }
+
+ int64_t seed = params.count("seed") ? static_cast<int64_t>(params["seed"].as<long>())
+ : SecureRandom::create()->nextInt64();
+
+ if (mongoeBenchGlobalParams.preConfig) {
+ mongoeBenchGlobalParams.preConfig->randomSeed = seed;
+ }
+
+ if (mongoeBenchGlobalParams.opsConfig) {
+ mongoeBenchGlobalParams.opsConfig->randomSeed = seed;
+
+ if (params.count("threads")) {
+ mongoeBenchGlobalParams.opsConfig->parallel = params["threads"].as<unsigned>();
+ }
+
+ if (params.count("time")) {
+ mongoeBenchGlobalParams.opsConfig->seconds = params["time"].as<double>();
+ }
+ }
+
+ return Status::OK();
+}
+
+} // namespace mongo
diff --git a/src/mongo/tools/mongoebench_options.h b/src/mongo/tools/mongoebench_options.h
new file mode 100644
index 00000000000..5b92c4d9758
--- /dev/null
+++ b/src/mongo/tools/mongoebench_options.h
@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects
+ * for all of the code used other than as permitted herein. If you modify
+ * file(s) with this exception, you may extend this exception to your
+ * version of the file(s), but you are not obligated to do so. If you do not
+ * wish to do so, delete this exception statement from your version. If you
+ * delete this exception statement from all source files in the program,
+ * then also delete it in the license file.
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <iosfwd>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mongo/base/status.h"
+
+namespace mongo {
+
+class BenchRunConfig;
+
+namespace optionenvironment {
+
+class OptionSection;
+class Environment;
+
+} // namespace optionenvironment
+
+namespace moe = mongo::optionenvironment;
+
+struct MongoeBenchGlobalParams {
+ std::unique_ptr<BenchRunConfig> preConfig;
+ std::unique_ptr<BenchRunConfig> opsConfig;
+};
+
+extern MongoeBenchGlobalParams mongoeBenchGlobalParams;
+
+Status addMongoeBenchOptions(moe::OptionSection* options);
+
+void printMongoeBenchHelp(std::ostream* out);
+
+/**
+ * Handle options that should come before validation, such as "help".
+ *
+ * Returns false if an option was found that implies we should prematurely exit with success.
+ */
+bool handlePreValidationMongoeBenchOptions(const moe::Environment& params);
+
+Status storeMongoeBenchOptions(const moe::Environment& params,
+ const std::vector<std::string>& args);
+
+} // namespace mongo
diff --git a/src/mongo/tools/mongoebench_options_init.cpp b/src/mongo/tools/mongoebench_options_init.cpp
new file mode 100644
index 00000000000..5818f6b916c
--- /dev/null
+++ b/src/mongo/tools/mongoebench_options_init.cpp
@@ -0,0 +1,54 @@
+/**
+ * Copyright (C) 2018 MongoDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * As a special exception, the copyright holders give permission to link the
+ * code of portions of this program with the OpenSSL library under certain
+ * conditions as described in each individual source file and distribute
+ * linked combinations including the program with the OpenSSL library. You
+ * must comply with the GNU Affero General Public License in all respects for
+ * all of the code used other than as permitted herein. If you modify file(s)
+ * with this exception, you may extend this exception to your version of the
+ * file(s), but you are not obligated to do so. If you do not wish to do so,
+ * delete this exception statement from your version. If you delete this
+ * exception statement from all source files in the program, then also delete
+ * it in the license file.
+ */
+
+#include "mongo/tools/mongoebench_options.h"
+
+#include "mongo/util/options_parser/startup_option_init.h"
+#include "mongo/util/options_parser/startup_options.h"
+#include "mongo/util/quick_exit.h"
+
+namespace mongo {
+namespace {
+
+MONGO_GENERAL_STARTUP_OPTIONS_REGISTER(MongoeBenchOptions)(InitializerContext* context) {
+ return addMongoeBenchOptions(&moe::startupOptions);
+}
+
+GlobalInitializerRegisterer mongoeBenchOptionsStore(
+ "MongoeBenchOptions_Store",
+ {"BeginStartupOptionStorage"},
+ {"EndStartupOptionStorage"},
+ [](InitializerContext* context) {
+ if (!handlePreValidationMongoeBenchOptions(moe::startupOptionsParsed)) {
+ quickExit(EXIT_SUCCESS);
+ }
+ return storeMongoeBenchOptions(moe::startupOptionsParsed, context->args());
+ });
+
+} // namespace
+} // namespace mongo
View Lorry Controller Status