diff options
author | Robert Guo <robert.guo@10gen.com> | 2018-05-18 15:46:27 -0400 |
---|---|---|
committer | Robert Guo <robert.guo@10gen.com> | 2018-05-24 09:29:18 -0400 |
commit | 3ae0777cfe3e467df6edafd1af6d8d64c4bc93db (patch) | |
tree | 2b1b33e5f007cf35812aac48d3a4d34b0e5d0ae8 | |
parent | 73dd3841a643b6e5b3b7c6f683d99d406dc1d2a8 (diff) | |
download | mongo-3ae0777cfe3e467df6edafd1af6d8d64c4bc93db.tar.gz |
SERVER-33738 Create a runCommand() override method to perform operations inside a transaction
(cherry picked from commit 29c8f66e4396c7b68535d644638f2a81592e3081)
-rw-r--r-- | buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml | 305 | ||||
-rw-r--r-- | buildscripts/resmokelib/selector.py | 1 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py | 29 | ||||
-rw-r--r-- | etc/evergreen.yml | 25 | ||||
-rw-r--r-- | jstests/core/all.js | 14 | ||||
-rw-r--r-- | jstests/core/dbhash2.js | 7 | ||||
-rw-r--r-- | jstests/core/eval0.js | 8 | ||||
-rw-r--r-- | jstests/libs/override_methods/enable_sessions.js | 55 | ||||
-rw-r--r-- | jstests/libs/txns/txn_override.js | 240 | ||||
-rw-r--r-- | jstests/libs/txns/txn_passthrough_runner.js | 16 | ||||
-rw-r--r-- | jstests/libs/txns/txn_passthrough_runner_selftest.js | 30 |
11 files changed, 681 insertions, 49 deletions
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml new file mode 100644 index 00000000000..37daeaa3f63 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/replica_sets_multi_stmt_txn_jscore_passthrough.yml @@ -0,0 +1,305 @@ +test_kind: multi_stmt_txn_passthrough + +selector: + roots: + - jstests/core/**/*.js + # TODO: SERVER-35089 + # - jstests/libs/txns/txn_passthrough_runner_selftest.js + exclude_files: + # TODO: SERVER-35089 + - jstests/core/geo_allowedcomparisons.js + - jstests/core/geo_big_polygon2.js + - jstests/core/in.js + - jstests/core/orj.js + - jstests/core/insert1.js + + # These tests already run with transactions. + - jstests/core/txns/*.js + + # These tests are not expected to pass with replica-sets: + - jstests/core/capped_update.js + - jstests/core/dbadmin.js + - jstests/core/opcounters_write_cmd.js + - jstests/core/read_after_optime.js + + ## + ## Limitations with the way the runner file injects transactions. + ## + + # These tests expects some statements to error, which will cause txns to abort entirely. + - jstests/core/bulk_api_ordered.js + - jstests/core/bulk_api_unordered.js + - jstests/core/bulk_legacy_enforce_gle.js + - jstests/core/capped5.js + - jstests/core/commands_with_uuid.js + - jstests/core/explain_execution_error.js + - jstests/core/expr.js + - jstests/core/find_getmore_bsonsize.js + - jstests/core/find_getmore_cmd.js + - jstests/core/find9.js + - jstests/core/index_big1.js + - jstests/core/index_bigkeys.js + - jstests/core/index_decimal.js + - jstests/core/index_multiple_compatibility.js + - jstests/core/index_partial_write_ops.js + - jstests/core/index8.js # No explicit check for failed command. + - jstests/core/indexa.js # No explicit check for failed command. + - jstests/core/indexes_multiple_commands.js + - jstests/core/insert_long_index_key.js + - jstests/core/js2.js + - jstests/core/json_schema/json_schema.js + - jstests/core/mr_bigobject.js + - jstests/core/not2.js + - jstests/core/notablescan.js + - jstests/core/or1.js + - jstests/core/or2.js + - jstests/core/or3.js + - jstests/core/ork.js + - jstests/core/ref4.js + - jstests/core/regex_limit.js + - jstests/core/remove_undefined.js + - jstests/core/set7.js + - jstests/core/sortb.js + - jstests/core/sortf.js + - jstests/core/sortg.js + - jstests/core/sortj.js + - jstests/core/tailable_skip_limit.js + - jstests/core/type_array.js + - jstests/core/uniqueness.js + - jstests/core/unset2.js + - jstests/core/update_addToSet.js + - jstests/core/update_arrayFilters.js + - jstests/core/update_find_and_modify_id.js + - jstests/core/update_modifier_pop.js + - jstests/core/updateh.js + - jstests/core/updatej.js + - jstests/core/ref.js + + # Consecutive writes totalling more than 16MB will cause the txn to abort with + # a TransactionTooLarge error. + - jstests/core/batch_size.js + - jstests/core/single_batch.js + + ## + ## Some aggregation stages don't support snapshot readconcern. + ## + + # $explain (requires read concern local) + - jstests/core/agg_hint.js + - jstests/core/and.js + - jstests/core/collation.js + - jstests/core/explain_shell_helpers.js + - jstests/core/index_partial_read_ops.js + - jstests/core/optimized_match_explain.js + - jstests/core/sort_array.js + - jstests/core/views/views_collation.js + + # $out + - jstests/core/bypass_doc_validation.js + - jstests/core/views/views_aggregation.js + + # $listSessions + - jstests/core/list_all_local_sessions.js + - jstests/core/list_all_sessions.js + - jstests/core/list_local_sessions.js + - jstests/core/list_sessions.js + + # $indexStats + - jstests/core/index_stats.js + + # $collStats + - jstests/core/operation_latency_histogram.js + - jstests/core/views/views_coll_stats.js + - jstests/core/views/views_stats.js + + ## + ## WriteErrors get converted to WriteCommandErrors if part of a txn. + ## + + # The same error code, but with ok:0. + - jstests/core/json_schema/additional_items.js + - jstests/core/json_schema/additional_properties.js + - jstests/core/json_schema/bsontype.js + - jstests/core/json_schema/dependencies.js + - jstests/core/json_schema/items.js + - jstests/core/json_schema/logical_keywords.js + - jstests/core/json_schema/min_max_items.js + - jstests/core/json_schema/min_max_properties.js + - jstests/core/json_schema/pattern_properties.js + - jstests/core/json_schema/required.js + - jstests/core/json_schema/unique_items.js + + - jstests/core/field_name_validation.js + - jstests/core/fts_array.js + - jstests/core/inc-SERVER-7446.js + - jstests/core/invalid_db_name.js + - jstests/core/push_sort.js + + # Checks for "WriteErrors" explicitly from the response of db.runCommand() + - jstests/core/max_doc_size.js + + # Calls res.getWriteError() or res.hasWriteError(). + - jstests/core/big_object1.js + - jstests/core/bulk_api_ordered.js + - jstests/core/bulk_api_unordered.js + - jstests/core/bulk_legacy_enforce_gle.js + - jstests/core/cappeda.js + - jstests/core/doc_validation.js + - jstests/core/doc_validation_options.js + - jstests/core/geo_multinest0.js + - jstests/core/insert_illegal_doc.js + - jstests/core/ns_length.js + - jstests/core/push2.js + - jstests/core/remove6.js + - jstests/core/removeb.js + - jstests/core/rename4.js + - jstests/core/shell_writeconcern.js + - jstests/core/storefunc.js + - jstests/core/update_arrayFilters.js + - jstests/core/update_dbref.js + - jstests/core/updatel.js + - jstests/core/write_result.js + + # Multiple writes in a txn, some of which fail because the collection doesn't exist. + # We create the collection and retry the last write, but previous writes would have + # still failed. + - jstests/core/dbref1.js + - jstests/core/dbref2.js + - jstests/core/ref3.js + - jstests/core/repair_database.js + - jstests/core/update3.js + - jstests/core/rename3.js + + ## + ## Error: Unable to acquire lock within a max lock request timeout of '0ms' milliseconds + ## + + # Collection drops done through applyOps are not converted to w:majority + - jstests/core/views/invalid_system_views.js + + # Operations run on the "out" collection of a MapReduce call, which is not always + # immediately available to a transaction as it is still being replicated. Transactions + # fail with "Unable to acquire lock" errors. + - jstests/core/function_string_representations.js + - jstests/core/mr_merge.js + - jstests/core/mr_merge2.js + - jstests/core/mr_replaceIntoDB.js + - jstests/core/mr_outreduce.js + - jstests/core/mr_outreduce2.js + + ## + ## Misc. reasons. + ## + + # SERVER-34868 Cannot run a legacy query on a session. + - jstests/core/exhaust.js + - jstests/core/validate_cmd_ns.js + + # SERVER-34772 Tailable Cursors are not allowed with snapshot readconcern. + - jstests/core/awaitdata_getmore_cmd.js + - jstests/core/getmore_cmd_maxtimems.js + - jstests/core/tailable_cursor_invalidation.js + - jstests/core/tailable_getmore_batch_size.js + + # SERVER-34918 The "max" option of a capped collection can be exceeded until the next insert. + # The reason is that we don't update the count of a collection until a transaction commits, + # by which point it is too late to complain that "max" has been exceeded. + - jstests/core/capped_max1.js + + # The "max" option of a capped collection can be temporarily exceeded before a + # txn is committed. + - jstests/core/bulk_insert_capped.js + + # Wrong count for top info (WriteLock) + - jstests/core/top.js + + # Expects collection to not have been created + - jstests/core/insert_id_undefined.js + + # Creates sessions explicitly, resulting in txns being run through different sessions + # using a single txnNumber. + - jstests/core/list_all_local_cursors.js + - jstests/core/json_schema/misc_validation.js + - jstests/core/views/views_all_commands.js + + # Committing a transaction when the server is fsync locked fails. + - jstests/core/fsync.js + + # Expects legacy errors ($err). + - jstests/core/constructors.js + + # txn interrupted by command outside of txn before getMore runs. + - jstests/core/commands_namespace_parsing.js + - jstests/core/drop3.js + - jstests/core/ensure_sorted.js + - jstests/core/geo_s2cursorlimitskip.js + - jstests/core/getmore_invalidated_cursors.js + - jstests/core/getmore_invalidated_documents.js + - jstests/core/kill_cursors.js + - jstests/core/list_collections1.js + - jstests/core/list_indexes.js + - jstests/core/max_time_ms.js + - jstests/core/oro.js + + # Expects certain number of operations in the system.profile collection. + - jstests/core/profile*.js + + # Parallel Shell - we do not signal the override to end a txn when a parallel shell closes. + - jstests/core/awaitdata_getmore_cmd.js + - jstests/core/compact_keeps_indexes.js + - jstests/core/count10.js + - jstests/core/count_plan_summary.js + - jstests/core/coveredIndex3.js + - jstests/core/currentop.js + - jstests/core/distinct3.js + - jstests/core/evald.js + - jstests/core/find_and_modify_concurrent_update.js + - jstests/core/fsync.js + - jstests/core/geo_update_btree.js + - jstests/core/killop_drop_collection.js + - jstests/core/loadserverscripts.js + - jstests/core/mr_killop.js + - jstests/core/queryoptimizer3.js + - jstests/core/remove9.js + - jstests/core/removeb.js + - jstests/core/removec.js + - jstests/core/shellstartparallel.js + - jstests/core/updatef.js + + # Command expects to see result from parallel operation. + # E.g. Suppose the following sequence of events: op1, join() op2 in parallel shell, op3. + # op3 will still be using the snapshot from op1, and not see op2 at all. + - jstests/core/cursora.js + - jstests/core/bench_test1.js + + exclude_with_any_tags: + # "Cowardly refusing to override read concern of command: ..." + - assumes_read_concern_unchanged + # "writeConcern is not allowed within a multi-statement transaction" + - assumes_write_concern_unchanged + +executor: + config: + shell_options: + eval: var testingReplication = true; + global_vars: + TestData: + sessionOptions: + causalConsistency: false + readMode: commands + hooks: + # The CheckReplDBHash hook waits until all operations have replicated to and have been applied + # on the secondaries, so we run the ValidateCollections hook after it to ensure we're + # validating the entire contents of the collection. + - class: CheckReplOplogs + - class: CheckReplDBHash + - class: ValidateCollections + - class: CleanEveryN + n: 20 + fixture: + class: ReplicaSetFixture + mongod_options: + set_parameters: + enableTestCommands: 1 + numInitialSyncAttempts: 1 + num_nodes: 2 diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 3014012511d..a4e6f15aa8f 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -602,6 +602,7 @@ _SELECTOR_REGISTRY = { "fsm_workload_test": (_JSTestSelectorConfig, _JSTestSelector), "json_schema_test": (_JsonSchemaTestSelectorConfig, _Selector), "js_test": (_JSTestSelectorConfig, _JSTestSelector), + "multi_stmt_txn_passthrough": (_JSTestSelectorConfig, _JSTestSelector), "py_test": (_PyTestCaseSelectorConfig, _Selector), "sleep_test": (_SleepTestCaseSelectorConfig, _SleepTestCaseSelector), } diff --git a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py new file mode 100644 index 00000000000..1e790612153 --- /dev/null +++ b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py @@ -0,0 +1,29 @@ +"""unittest.TestCase for multi-statement transaction passthrough tests.""" + +from __future__ import absolute_import + +from buildscripts.resmokelib import config +from buildscripts.resmokelib import core +from buildscripts.resmokelib import utils +from buildscripts.resmokelib.testing.testcases import jsrunnerfile + + +class MultiStmtTxnTestCase(jsrunnerfile.JSRunnerFileTestCase): + """Test case for mutli statement transactions.""" + + REGISTERED_NAME = "multi_stmt_txn_passthrough" + + def __init__(self, logger, multi_stmt_txn_test_file, shell_executable=None, shell_options=None): + """Initilize MultiStmtTxnTestCase.""" + jsrunnerfile.JSRunnerFileTestCase.__init__( + self, logger, "Multi-statement Transaction Passthrough", multi_stmt_txn_test_file, + test_runner_file="jstests/libs/txns/txn_passthrough_runner.js", + shell_executable=shell_executable, shell_options=shell_options) + + @property + def multi_stmt_txn_test_file(self): + """Return the name of the test file.""" + return self.test_name + + def _populate_test_data(self, test_data): + test_data["multiStmtTxnTestFile"] = self.multi_stmt_txn_test_file diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 3867b2174f5..8bbf16be8cb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -4197,6 +4197,17 @@ tasks: run_multiple_jobs: true - <<: *task_template + name: replica_sets_multi_stmt_txn_jscore_passthrough + depends_on: + - name: jsCore + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_args: --suites=replica_sets_multi_stmt_txn_jscore_passthrough --storageEngine=wiredTiger + run_multiple_jobs: true + +- <<: *task_template name: replica_sets_initsync_jscore_passthrough depends_on: - name: jsCore @@ -6918,6 +6929,7 @@ buildvariants: - name: replica_sets_initsync_jscore_passthrough - name: replica_sets_initsync_static_jscore_passthrough - name: replica_sets_jscore_passthrough + - name: replica_sets_multi_stmt_txn_jscore_passthrough - name: replica_sets_kill_primary_jscore_passthrough - name: replica_sets_terminate_primary_jscore_passthrough - name: replica_sets_kill_secondaries_jscore_passthrough @@ -8024,7 +8036,12 @@ buildvariants: - name: replica_sets_initsync_static_jscore_passthrough distros: - windows-64-vs2015-large + - name: replica_sets_multi_stmt_txn_jscore_passthrough + distros: + - windows-64-vs2015-large - name: replica_sets_kill_primary_jscore_passthrough + distros: + - windows-64-vs2015-large - name: replica_sets_terminate_primary_jscore_passthrough distros: - windows-64-vs2015-large @@ -9466,7 +9483,12 @@ buildvariants: - name: replica_sets_initsync_static_jscore_passthrough distros: - rhel62-large + - name: replica_sets_multi_stmt_txn_jscore_passthrough + distros: + - rhel62-large - name: replica_sets_kill_primary_jscore_passthrough + distros: + - rhel62-large - name: replica_sets_terminate_primary_jscore_passthrough distros: - rhel62-large @@ -9928,6 +9950,7 @@ buildvariants: - name: replica_sets_jscore_passthrough - name: replica_sets_initsync_jscore_passthrough - name: replica_sets_initsync_static_jscore_passthrough + - name: replica_sets_multi_stmt_txn_jscore_passthrough - name: replica_sets_kill_primary_jscore_passthrough - name: replica_sets_terminate_primary_jscore_passthrough - name: replica_sets_kill_secondaries_jscore_passthrough @@ -12003,6 +12026,7 @@ buildvariants: - name: replica_sets_jscore_passthrough - name: replica_sets_initsync_jscore_passthrough - name: replica_sets_initsync_static_jscore_passthrough + - name: replica_sets_multi_stmt_txn_jscore_passthrough - name: replica_sets_kill_primary_jscore_passthrough - name: replica_sets_terminate_primary_jscore_passthrough - name: replica_sets_kill_secondaries_jscore_passthrough @@ -12181,6 +12205,7 @@ buildvariants: - name: replica_sets_jscore_passthrough - name: replica_sets_initsync_jscore_passthrough - name: replica_sets_initsync_static_jscore_passthrough + - name: replica_sets_multi_stmt_txn_jscore_passthrough - name: replica_sets_kill_primary_jscore_passthrough - name: replica_sets_terminate_primary_jscore_passthrough - name: replica_sets_kill_secondaries_jscore_passthrough diff --git a/jstests/core/all.js b/jstests/core/all.js index 221cf1daeda..e77c0279215 100644 --- a/jstests/core/all.js +++ b/jstests/core/all.js @@ -3,13 +3,13 @@ t.drop(); doTest = function() { - t.save({a: [1, 2, 3]}); - t.save({a: [1, 2, 4]}); - t.save({a: [1, 8, 5]}); - t.save({a: [1, 8, 6]}); - t.save({a: [1, 9, 7]}); - t.save({a: []}); - t.save({}); + assert.commandWorked(t.save({a: [1, 2, 3]})); + assert.commandWorked(t.save({a: [1, 2, 4]})); + assert.commandWorked(t.save({a: [1, 8, 5]})); + assert.commandWorked(t.save({a: [1, 8, 6]})); + assert.commandWorked(t.save({a: [1, 9, 7]})); + assert.commandWorked(t.save({a: []})); + assert.commandWorked(t.save({})); assert.eq(5, t.find({a: {$all: [1]}}).count()); assert.eq(2, t.find({a: {$all: [1, 2]}}).count()); diff --git a/jstests/core/dbhash2.js b/jstests/core/dbhash2.js index 689134d2bf0..93c78a35f26 100644 --- a/jstests/core/dbhash2.js +++ b/jstests/core/dbhash2.js @@ -8,13 +8,16 @@ mydb = db.getSisterDB("config"); t = mydb.foo; t.drop(); -t.insert({x: 1}); +assert.commandWorked(t.insert({x: 1})); res1 = mydb.runCommand("dbhash"); res2 = mydb.runCommand("dbhash"); +assert.commandWorked(res1); +assert.commandWorked(res2); assert.eq(res1.collections.foo, res2.collections.foo); -t.insert({x: 2}); +assert.commandWorked(t.insert({x: 2})); res3 = mydb.runCommand("dbhash"); +assert.commandWorked(res3); assert.neq(res1.collections.foo, res3.collections.foo); // Validate dbHash with an empty database does not trigger an fassert/invariant diff --git a/jstests/core/eval0.js b/jstests/core/eval0.js index 7e075feb2d5..921c4b4b428 100644 --- a/jstests/core/eval0.js +++ b/jstests/core/eval0.js @@ -4,7 +4,10 @@ // ] assert.writeOK(db.evalprep.insert({}), "db must exist for eval to succeed"); + db.evalprep.drop(); +db.system.js.remove({}); + assert.eq(17, db.eval(function() { return 11 + 6; @@ -15,10 +18,11 @@ assert.eq(17, db.eval(function(x) { }, 7), "B"); // check that functions in system.js work -db.system.js.insert({ +assert.writeOK(db.system.js.insert({ _id: "add", value: function(x, y) { return x + y; } -}); +})); + assert.eq(20, db.eval("this.add(15, 5);"), "C"); diff --git a/jstests/libs/override_methods/enable_sessions.js b/jstests/libs/override_methods/enable_sessions.js index 2d304927c35..85bb57d7e94 100644 --- a/jstests/libs/override_methods/enable_sessions.js +++ b/jstests/libs/override_methods/enable_sessions.js @@ -6,40 +6,18 @@ load("jstests/libs/override_methods/override_helpers.js"); - var runCommandOriginal = Mongo.prototype.runCommand; - var runCommandWithMetadataOriginal = Mongo.prototype.runCommandWithMetadata; - var getDBOriginal = Mongo.prototype.getDB; - var sessionMap = new WeakMap(); + const getDBOriginal = Mongo.prototype.getDB; - let sessionOptions = {}; - if (typeof TestData !== "undefined" && TestData.hasOwnProperty("sessionOptions")) { - sessionOptions = TestData.sessionOptions; - } - - const driverSession = startSession(db.getMongo()); - db = driverSession.getDatabase(db.getName()); - sessionMap.set(db.getMongo(), driverSession); - - OverrideHelpers.prependOverrideInParallelShell( - "jstests/libs/override_methods/enable_sessions.js"); - - function startSession(conn) { - const driverSession = conn.startSession(sessionOptions); - // Override the endSession function to be a no-op so fuzzer doesn't accidentally end the - // session. - driverSession.endSession = Function.prototype; - return driverSession; - } + const sessionMap = new WeakMap(); + const sessionOptions = TestData.sessionOptions; // Override the runCommand to check for any command obj that does not contain a logical session // and throw an error. - function runCommandWithLsidCheck(conn, dbName, cmdObj, func, funcArgs) { + function runCommandWithLsidCheck(conn, dbName, cmdName, cmdObj, func, makeFuncArgs) { if (jsTest.options().disableEnableSessions) { - return func.apply(conn, funcArgs); + return func.apply(conn, makeFuncArgs(cmdObj)); } - const cmdName = Object.keys(cmdObj)[0]; - // If the command is in a wrapped form, then we look for the actual command object // inside the query/$query object. let cmdObjUnwrapped = cmdObj; @@ -56,18 +34,9 @@ throw new Error("command object does not have session id: " + tojson(cmdObj)); } } - return func.apply(conn, funcArgs); + return func.apply(conn, makeFuncArgs(cmdObj)); } - Mongo.prototype.runCommand = function(dbName, commandObj, options) { - return runCommandWithLsidCheck(this, dbName, commandObj, runCommandOriginal, arguments); - }; - - Mongo.prototype.runCommandWithMetadata = function(dbName, metadata, commandObj) { - return runCommandWithLsidCheck( - this, dbName, commandObj, runCommandWithMetadataOriginal, arguments); - }; - // Override the getDB to return a db object with the correct driverSession. We use a WeakMap // to cache the session for each connection instance so we can retrieve the same session on // subsequent calls to getDB. @@ -77,7 +46,10 @@ } if (!sessionMap.has(this)) { - const session = startSession(this); + const session = this.startSession(sessionOptions); + // Override the endSession function to be a no-op so jstestfuzz doesn't accidentally + // end the session. + session.endSession = Function.prototype; sessionMap.set(this, session); } @@ -86,4 +58,11 @@ return db; }; + // Override the global `db` object to be part of a session. + db = db.getMongo().getDB(db.getName()); + + OverrideHelpers.prependOverrideInParallelShell( + "jstests/libs/override_methods/enable_sessions.js"); + OverrideHelpers.overrideRunCommand(runCommandWithLsidCheck); + })(); diff --git a/jstests/libs/txns/txn_override.js b/jstests/libs/txns/txn_override.js new file mode 100644 index 00000000000..e82ea352974 --- /dev/null +++ b/jstests/libs/txns/txn_override.js @@ -0,0 +1,240 @@ +/** + * Override to run consecutive operations inside the same transaction. When an operation that + * cannot be run inside of a transaction is encountered, the active transaction is committed + * before running the next operation. + */ + +(function() { + 'use strict'; + + load('jstests/libs/override_methods/override_helpers.js'); + + const runCommandOriginal = Mongo.prototype.runCommand; + + const kCmdsSupportingTransactions = new Set([ + 'aggregate', + 'delete', + 'find', + 'findAndModify', + 'findandmodify', + 'getMore', + 'insert', + 'update', + ]); + + const kCmdsThatWrite = new Set([ + 'insert', + 'update', + 'findAndModify', + 'findandmodify', + 'delete', + ]); + + const kCmdsThatInsert = new Set([ + 'insert', + 'update', + 'findAndModify', + 'findandmodify', + ]); + + // Copied from ServerSession.TransactionStates. + const TransactionStates = { + kActive: 'active', + kInactive: 'inactive', + }; + + function commandSupportsTxn(dbName, cmdName, cmdObj) { + if (!kCmdsSupportingTransactions.has(cmdName)) { + return false; + } + + if (dbName === 'local' || dbName === 'config' || dbName === 'admin') { + return false; + } + + if (kCmdsThatWrite.has(cmdName)) { + if (cmdObj[cmdName].startsWith('system.')) { + return false; + } + } + return true; + } + + function getTxnOptionsForClient(conn) { + // We tack transaction options onto the client since we use one session per client. + if (!conn.hasOwnProperty('txnOverrideOptions')) { + conn.txnOverrideOptions = { + stmtId: new NumberInt(0), + autocommit: false, + txnNumber: new NumberLong(-1), + }; + conn.txnOverrideState = TransactionStates.kInactive; + } + return conn.txnOverrideOptions; + } + + function incrementStmtIdBy(cmdName, cmdObjUnwrapped) { + // Reserve the statement ids for batch writes. + try { + switch (cmdName) { + case "insert": + return cmdObjUnwrapped.documents.length; + case "update": + return cmdObjUnwrapped.updates.length; + case "delete": + return cmdObjUnwrapped.deletes.length; + default: + return 1; + } + } catch (e) { + // Malformed command objects can cause errors to be thrown. + return 1; + } + } + + function commitTransaction(conn, lsid, txnNumber) { + const res = runCommandOriginal.call(conn, + 'admin', + { + commitTransaction: 1, + autocommit: false, lsid, txnNumber, + }, + 0); + assert.commandWorked(res); + conn.txnOverrideState = TransactionStates.kInactive; + } + + function abortTransaction(conn, lsid, txnNumber) { + // If there's been an error, we abort the transaction. It doesn't matter if the + // abort call succeeds or not. + runCommandOriginal.call(conn, + 'admin', + { + abortTransaction: 1, + autocommit: false, + lsid: lsid, + txnNumber: txnNumber, + }, + 0); + + conn.txnOverrideState = TransactionStates.kInactive; + } + + function continueTransaction(conn, txnOptions, cmdName, cmdObj) { + if (conn.txnOverrideState === TransactionStates.kInactive) { + // First command in a transaction. + txnOptions.txnNumber = new NumberLong(txnOptions.txnNumber + 1); + txnOptions.stmtId = new NumberInt(0); + + cmdObj.startTransaction = true; + + if (cmdObj.readConcern && cmdObj.readConcern.level !== 'snapshot') { + throw new Error("refusing to override existing readConcern"); + } else { + cmdObj.readConcern = {level: 'snapshot'}; + } + + conn.txnOverrideState = TransactionStates.kActive; + } + + txnOptions.stmtId = new NumberInt(txnOptions.stmtId + incrementStmtIdBy(cmdName, cmdObj)); + + cmdObj.txnNumber = txnOptions.txnNumber; + cmdObj.stmtId = txnOptions.stmtId; + cmdObj.autocommit = false; + } + + function runCommandWithTransactions(conn, dbName, commandName, commandObj, func, makeFuncArgs) { + const driverSession = conn.getDB(dbName).getSession(); + if (driverSession.getSessionId() === null) { + // Sessions is explicitly disabled for this command. So we skip overriding it to + // use transactions. + return func.apply(conn, makeFuncArgs(commandObj)); + } + + let cmdObjUnwrapped = commandObj; + let cmdNameUnwrapped = commandName; + + if (commandName === "query" || commandName === "$query") { + commandObj[commandName] = Object.assign({}, cmdObjUnwrapped[commandName]); + cmdObjUnwrapped = commandObj[commandName]; + cmdNameUnwrapped = Object.keys(cmdObjUnwrapped)[0]; + } + + const commandSupportsTransaction = + commandSupportsTxn(dbName, cmdNameUnwrapped, cmdObjUnwrapped); + + const txnOptions = getTxnOptionsForClient(conn); + + if (!commandSupportsTransaction) { + if (conn.txnOverrideState === TransactionStates.kActive) { + commitTransaction(conn, commandObj.lsid, txnOptions.txnNumber); + } + + } else { + continueTransaction(conn, txnOptions, cmdNameUnwrapped, cmdObjUnwrapped); + } + + if (commandName === 'drop' || commandName === 'convertToCapped') { + // Convert all collection drops to w:majority so they won't prevent subsequent + // operations in transactions from failing when failing to acquire collection locks. + if (!cmdObjUnwrapped.writeConcern) { + cmdObjUnwrapped.writeConcern = {}; + } + cmdObjUnwrapped.writeConcern.w = 'majority'; + } + + let res = func.apply(conn, makeFuncArgs(commandObj)); + + if (res.ok !== 1) { + abortTransaction(conn, commandObj.lsid, txnOptions.txnNumber); + if (kCmdsThatInsert.has(cmdNameUnwrapped)) { + // If the command inserted data, we check if it failed because the collection did + // not exist; if so, create the collection and retry the command. Tests that + // expect collections to not exist will have to be skipped. + if (res.code === ErrorCodes.NamespaceNotFound) { + const createCmdRes = + runCommandOriginal.call(conn, + dbName, + { + create: cmdObjUnwrapped[cmdNameUnwrapped], + lsid: commandObj.lsid, + writeConcern: {w: 'majority'}, + }, + 0); + + if (createCmdRes.ok !== 1) { + if (createCmdRes.code !== ErrorCodes.NamespaceExists) { + // The collection still does not exist. So we just return the original + // response to the caller, + return res; + } + } else { + assert.commandWorked(createCmdRes); + } + } else { + // If the insert command failed for any other reason, we return the original + // response without retrying. + return res; + } + + continueTransaction(conn, txnOptions, cmdNameUnwrapped, cmdObjUnwrapped); + + res = func.apply(conn, makeFuncArgs(commandObj)); + if (res.ok !== 1) { + abortTransaction(conn, commandObj.lsid, txnOptions.txnNumber); + } + } + } + + return res; + } + + startParallelShell = function() { + throw new Error( + "Cowardly refusing to run test with transaction override enabled when it uses" + + "startParalleShell()"); + }; + + OverrideHelpers.overrideRunCommand(runCommandWithTransactions); +})();
\ No newline at end of file diff --git a/jstests/libs/txns/txn_passthrough_runner.js b/jstests/libs/txns/txn_passthrough_runner.js new file mode 100644 index 00000000000..3acc3184d54 --- /dev/null +++ b/jstests/libs/txns/txn_passthrough_runner.js @@ -0,0 +1,16 @@ +(function() { + 'use strict'; + + load('jstests/libs/override_methods/enable_sessions.js'); + load('jstests/libs/txns/txn_override.js'); + + const testFile = TestData.multiStmtTxnTestFile; + + try { + load(testFile); + } finally { + // Run a lightweight command to allow the override file to commit the last command. + // Ensure this command runs even if the test errors. + assert.commandWorked(db.runCommand({ping: 1})); + } +})(); diff --git a/jstests/libs/txns/txn_passthrough_runner_selftest.js b/jstests/libs/txns/txn_passthrough_runner_selftest.js new file mode 100644 index 00000000000..dae1d940373 --- /dev/null +++ b/jstests/libs/txns/txn_passthrough_runner_selftest.js @@ -0,0 +1,30 @@ +// Sanity test for the override logic in txn_override.js. We use the profiler to check that +// operation is not visible immediately, but is visible after the transaction commits. + +(function() { + 'use strict'; + + const testName = jsTest.name(); + + // Profile all commands. + db.setProfilingLevel(2); + + const coll = db[testName]; + + assert.commandWorked(coll.insert({x: 1})); + let commands = db.system.profile.find().toArray(); + // Check that the insert is not visible because the txn has not committed. + assert.eq(commands.length, 1); + assert.eq(commands[0].command.create, testName); + + // Use a dummy, unrelated operation to signal the txn runner to commit the transaction. + assert.commandWorked(db.runCommand({ping: 1})); + + commands = db.system.profile.find().toArray(); + // Assert the insert is now visible. + assert.eq(commands.length, 3); + assert.eq(commands[0].command.create, testName); + assert.eq(commands[1].command.insert, testName); + assert.eq(commands[2].command.find, 'system.profile'); + +})(); |