SERVER-30681 Run the concurrency suite with causal consistency enabled

7 files changed, 320 insertions, 3 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
index 008e6381a64..d7709c42789 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
@@ -3,6 +3,9 @@ test_kind: js_test
 selector:
   roots:
   - jstests/concurrency/fsm_all_sharded*.js
+  exclude_files:
+  # Skip causal consistency tests because they're run in separate suites
+  - jstests/concurrency/fsm_all_sharded_causal_consistency*.js
 
 # Concurrency tests that run against a sharded cluster start one themselves.
 executor:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml
new file mode 100644
index 00000000000..f7d51bf5a24
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency.yml
@@ -0,0 +1,12 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_all_sharded_causal_consistency.js
+
+# Concurrency tests that run against a sharded cluster start one themselves.
+executor:
+  config:
+    shell_options:
+      nodb: ''
+      readMode: commands
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml
new file mode 100644
index 00000000000..bd225c7ee5a
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_causal_consistency_and_balancer.yml
@@ -0,0 +1,12 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_all_sharded_causal_consistency_and_balancer.js
+
+# Concurrency tests that run against a sharded cluster start one themselves.
+executor:
+  config:
+    shell_options:
+      nodb: ''
+      readMode: commands
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 6d3e0857b50..12155b43694 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3188,6 +3188,42 @@ tasks:
       resmoke_args: --suites=concurrency_sharded --storageEngine=wiredTiger
 
 - <<: *task_template
+  name: concurrency_sharded_causal_consistency
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_causal_consistency --storageEngine=mmapv1
+
+- <<: *task_template
+  name: concurrency_sharded_causal_consistency_WT
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_causal_consistency --storageEngine=wiredTiger
+
+- <<: *task_template
+  name: concurrency_sharded_causal_consistency_and_balancer
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_causal_consistency_and_balancer --storageEngine=mmapv1
+
+- <<: *task_template
+  name: concurrency_sharded_causal_consistency_and_balancer_WT
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_causal_consistency_and_balancer --storageEngine=wiredTiger
+
+- <<: *task_template
   name: concurrency_simultaneous
   commands:
   - func: "do setup"
@@ -5605,6 +5641,18 @@ buildvariants:
   - name: concurrency_sharded_WT
     distros:
     - windows-64-vs2015-large
+  - name: concurrency_sharded_causal_consistency
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_causal_consistency_WT
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_causal_consistency_and_balancer
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
+    distros:
+    - windows-64-vs2015-large
   - name: concurrency_simultaneous
     distros:
     - windows-64-vs2015-large
@@ -6068,6 +6116,10 @@ buildvariants:
   - name: concurrency_replication_WT
   - name: concurrency_sharded
   - name: concurrency_sharded_WT
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_WT
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -6367,6 +6419,10 @@ buildvariants:
   - name: concurrency_replication_WT
   - name: concurrency_sharded
   - name: concurrency_sharded_WT
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_WT
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -6595,6 +6651,10 @@ buildvariants:
   - name: concurrency_replication_WT
   - name: concurrency_sharded
   - name: concurrency_sharded_WT
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_WT
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -8659,6 +8719,8 @@ buildvariants:
     - rhel62-large  # Some workloads require a lot of memory, use a bigger machine for this suite.
   - name: concurrency_replication
   - name: concurrency_sharded
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_simultaneous
   - name: dbtest
   - name: failpoints
@@ -9092,6 +9154,10 @@ buildvariants:
   - name: concurrency_replication_WT
   - name: concurrency_sharded
   - name: concurrency_sharded_WT
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_WT
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -9277,6 +9343,10 @@ buildvariants:
   - name: concurrency_replication_WT
   - name: concurrency_sharded
   - name: concurrency_sharded_WT
+  - name: concurrency_sharded_causal_consistency
+  - name: concurrency_sharded_causal_consistency_WT
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_causal_consistency_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
diff --git a/jstests/concurrency/fsm_all_sharded_causal_consistency.js b/jstests/concurrency/fsm_all_sharded_causal_consistency.js
new file mode 100644
index 00000000000..001b939f0d4
--- /dev/null
+++ b/jstests/concurrency/fsm_all_sharded_causal_consistency.js
@@ -0,0 +1,103 @@
+'use strict';
+
+load('jstests/concurrency/fsm_libs/runner.js');
+
+var dir = 'jstests/concurrency/fsm_workloads';
+
+var blacklist = [
+    // Disabled due to known bugs
+    'distinct.js',             // SERVER-13116 distinct isn't sharding aware
+    'distinct_noindex.js',     // SERVER-13116 distinct isn't sharding aware
+    'distinct_projection.js',  // SERVER-13116 distinct isn't sharding aware
+    'create_database.js',      // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'drop_database.js',        // SERVER-17397 Drops of sharded namespaces may not fully succeed
+
+    // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'.
+    // This bug is problematic for these workloads because they assert on count() values:
+    'agg_match.js',
+    'count.js',
+    'count_limit_skip.js',
+
+    // $lookup and $graphLookup are not supported on sharded collections.
+    'agg_graph_lookup.js',
+    'view_catalog_cycle_lookup.js',
+
+    // Disabled due to SERVER-20057, 'Concurrent, sharded mapReduces can fail when temporary
+    // namespaces collide across mongos processes'
+    'map_reduce_drop.js',
+    'map_reduce_inline.js',
+    'map_reduce_merge.js',
+    'map_reduce_merge_nonatomic.js',
+    'map_reduce_reduce.js',
+    'map_reduce_reduce_nonatomic.js',
+    'map_reduce_replace.js',
+    'map_reduce_replace_nonexistent.js',
+    'map_reduce_replace_remove.js',
+
+    // Disabled due to MongoDB restrictions and/or workload restrictions
+
+    // These workloads sometimes trigger 'Could not lock auth data update lock'
+    // errors because the AuthorizationManager currently waits for only five
+    // seconds to acquire the lock for authorization documents
+    'auth_create_role.js',
+    'auth_create_user.js',
+    'auth_drop_role.js',
+    'auth_drop_user.js',
+
+    'agg_group_external.js',                  // uses >100MB of data, which can overwhelm test hosts
+    'agg_sort_external.js',                   // uses >100MB of data, which can overwhelm test hosts
+    'compact.js',                             // compact can only be run against a standalone mongod
+    'compact_simultaneous_padding_bytes.js',  // compact can only be run against a mongod
+    'convert_to_capped_collection.js',        // convertToCapped can't be run on mongos processes
+    'convert_to_capped_collection_index.js',  // convertToCapped can't be run on mongos processes
+    'findAndModify_mixed_queue_unindexed.js',   // findAndModify requires a shard key
+    'findAndModify_remove_queue.js',            // remove cannot be {} for findAndModify
+    'findAndModify_remove_queue_unindexed.js',  // findAndModify requires a shard key
+    'findAndModify_update_collscan.js',         // findAndModify requires a shard key
+    'findAndModify_update_grow.js',             // can cause OOM kills on test hosts
+    'findAndModify_update_queue.js',            // findAndModify requires a shard key
+    'findAndModify_update_queue_unindexed.js',  // findAndModify requires a shard key
+    'group.js',                // the group command cannot be issued against a sharded cluster
+    'group_cond.js',           // the group command cannot be issued against a sharded cluster
+    'indexed_insert_eval.js',  // eval doesn't work with sharded collections
+    'indexed_insert_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // These workloads sometimes triggers an 'unable to target write op for collection ... caused by
+    // ... database not found' error. Further investigation still needs to be done, but these
+    // failures may be due to SERVER-17397 'drops in a sharded cluster may not fully succeed'
+    // because it drops and reuses the same namespaces.
+    'kill_multicollection_aggregation.js',
+    'invalidated_cursors.js',
+
+    'plan_cache_drop_database.js',  // cannot ensureIndex after dropDatabase without sharding first
+    'remove_single_document.js',    // our .remove(query, {justOne: true}) calls lack shard keys
+    'remove_single_document_eval.js',         // eval doesn't work with sharded collections
+    'remove_single_document_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // The rename_* workloads are disabled since renameCollection doesn't work with sharded
+    // collections
+    'rename_capped_collection_chain.js',
+    'rename_capped_collection_dbname_chain.js',
+    'rename_capped_collection_dbname_droptarget.js',
+    'rename_capped_collection_droptarget.js',
+    'rename_collection_chain.js',
+    'rename_collection_dbname_chain.js',
+    'rename_collection_dbname_droptarget.js',
+    'rename_collection_droptarget.js',
+
+    'update_simple_eval.js',           // eval doesn't work with sharded collections
+    'update_simple_eval_nolock.js',    // eval doesn't work with sharded collections
+    'update_upsert_multi.js',          // our update queries lack shard keys
+    'update_upsert_multi_noindex.js',  // our update queries lack shard keys
+    'upsert_where.js',      // cannot use upsert command with $where with sharded collections
+    'yield_and_hashed.js',  // stagedebug can only be run against a standalone mongod
+    'yield_and_sorted.js',  // stagedebug can only be run against a standalone mongod
+].map(function(file) {
+    return dir + '/' + file;
+});
+
+runWorkloadsSerially(ls(dir).filter(function(file) {
+    return !Array.contains(blacklist, file);
+}),
+                     {sharded: {enabled: true}, replication: {enabled: true}},
+                     {sessionOptions: {causallyConsistentReads: true}});
diff --git a/jstests/concurrency/fsm_all_sharded_causal_consistency_and_balancer.js b/jstests/concurrency/fsm_all_sharded_causal_consistency_and_balancer.js
new file mode 100644
index 00000000000..2114183ce8a
--- /dev/null
+++ b/jstests/concurrency/fsm_all_sharded_causal_consistency_and_balancer.js
@@ -0,0 +1,109 @@
+'use strict';
+
+load('jstests/concurrency/fsm_libs/runner.js');
+
+var dir = 'jstests/concurrency/fsm_workloads';
+
+var blacklist = [
+    // Disabled due to known bugs
+    'distinct.js',             // SERVER-13116 distinct isn't sharding aware
+    'distinct_noindex.js',     // SERVER-13116 distinct isn't sharding aware
+    'distinct_projection.js',  // SERVER-13116 distinct isn't sharding aware
+    'create_database.js',      // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'drop_database.js',        // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'remove_where.js',  // SERVER-14669 Multi-removes that use $where miscount removed documents
+
+    // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'.
+    // This bug is problematic for these workloads because they assert on count() values:
+    'agg_match.js',
+    'count.js',
+    'count_limit_skip.js',
+    'count_noindex.js',
+
+    // $lookup and $graphLookup are not supported on sharded collections.
+    'agg_graph_lookup.js',
+    'view_catalog_cycle_lookup.js',
+
+    // Disabled due to SERVER-20057, 'Concurrent, sharded mapReduces can fail when temporary
+    // namespaces collide across mongos processes'
+    'map_reduce_drop.js',
+    'map_reduce_inline.js',
+    'map_reduce_merge.js',
+    'map_reduce_merge_nonatomic.js',
+    'map_reduce_reduce.js',
+    'map_reduce_reduce_nonatomic.js',
+    'map_reduce_replace.js',
+    'map_reduce_replace_nonexistent.js',
+    'map_reduce_replace_remove.js',
+
+    // Disabled due to SERVER-13364, 'The geoNear command doesn't handle shard versioning, so a
+    // concurrent chunk migration may cause duplicate or missing results'
+    'yield_geo_near_dedup.js',
+
+    // Disabled due to MongoDB restrictions and/or workload restrictions
+
+    // These workloads sometimes trigger 'Could not lock auth data update lock'
+    // errors because the AuthorizationManager currently waits for only five
+    // seconds to acquire the lock for authorization documents
+    'auth_create_role.js',
+    'auth_create_user.js',
+    'auth_drop_role.js',
+    'auth_drop_user.js',
+
+    'agg_group_external.js',                  // uses >100MB of data, which can overwhelm test hosts
+    'agg_sort_external.js',                   // uses >100MB of data, which can overwhelm test hosts
+    'compact.js',                             // compact can only be run against a standalone mongod
+    'compact_simultaneous_padding_bytes.js',  // compact can only be run against a mongod
+    'convert_to_capped_collection.js',        // convertToCapped can't be run on mongos processes
+    'convert_to_capped_collection_index.js',  // convertToCapped can't be run on mongos processes
+    'findAndModify_mixed_queue_unindexed.js',   // findAndModify requires a shard key
+    'findAndModify_remove_queue.js',            // remove cannot be {} for findAndModify
+    'findAndModify_remove_queue_unindexed.js',  // findAndModify requires a shard key
+    'findAndModify_update_collscan.js',         // findAndModify requires a shard key
+    'findAndModify_update_grow.js',             // can cause OOM kills on test hosts
+    'findAndModify_update_queue.js',            // findAndModify requires a shard key
+    'findAndModify_update_queue_unindexed.js',  // findAndModify requires a shard key
+    'group.js',                // the group command cannot be issued against a sharded cluster
+    'group_cond.js',           // the group command cannot be issued against a sharded cluster
+    'indexed_insert_eval.js',  // eval doesn't work with sharded collections
+    'indexed_insert_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // These workloads sometimes triggers an 'unable to target write op for collection ... caused by
+    // ... database not found' error. Further investigation still needs to be done, but these
+    // failures may be due to SERVER-17397 'drops in a sharded cluster may not fully succeed'
+    // because it drops and reuses the same namespaces.
+    'kill_multicollection_aggregation.js',
+    'invalidated_cursors.js',
+
+    'plan_cache_drop_database.js',  // cannot ensureIndex after dropDatabase without sharding first
+    'remove_single_document.js',    // our .remove(query, {justOne: true}) calls lack shard keys
+    'remove_single_document_eval.js',         // eval doesn't work with sharded collections
+    'remove_single_document_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // The rename_* workloads are disabled since renameCollection doesn't work with sharded
+    // collections
+    'rename_capped_collection_chain.js',
+    'rename_capped_collection_dbname_chain.js',
+    'rename_capped_collection_dbname_droptarget.js',
+    'rename_capped_collection_droptarget.js',
+    'rename_collection_chain.js',
+    'rename_collection_dbname_chain.js',
+    'rename_collection_dbname_droptarget.js',
+    'rename_collection_droptarget.js',
+
+    'update_simple_eval.js',           // eval doesn't work with sharded collections
+    'update_simple_eval_nolock.js',    // eval doesn't work with sharded collections
+    'update_upsert_multi.js',          // our update queries lack shard keys
+    'update_upsert_multi_noindex.js',  // our update queries lack shard keys
+    'upsert_where.js',      // cannot use upsert command with $where with sharded collections
+    'yield_and_hashed.js',  // stagedebug can only be run against a standalone mongod
+    'yield_and_sorted.js',  // stagedebug can only be run against a standalone mongod
+].map(function(file) {
+    return dir + '/' + file;
+});
+
+runWorkloadsSerially(ls(dir).filter(function(file) {
+    return !Array.contains(blacklist, file);
+}),
+                     {sharded: {enabled: true, enableBalancer: true}, replication: {enabled: true}},
+                     {sessionOptions: {causallyConsistentReads: true}});
diff --git a/jstests/concurrency/fsm_libs/worker_thread.js b/jstests/concurrency/fsm_libs/worker_thread.js
index 44bdb870d0c..d592ffb688b 100644
--- a/jstests/concurrency/fsm_libs/worker_thread.js
+++ b/jstests/concurrency/fsm_libs/worker_thread.js
@@ -38,9 +38,17 @@ var workerThread = (function() {
                 }
 
                 if (typeof args.sessionOptions !== 'undefined') {
-                    myDB = new Mongo(args.host)
-                               .startSession(args.sessionOptions)
-                               .getDatabase(args.dbName);
+                    // TODO SERVER-30912: the shardCollection command hangs when run under a
+                    // session, so for now we don't start a session to enable testing of causal
+                    // consistency.
+                    myDB = new Mongo(args.host).getDB(args.dbName);
+
+                    if (args.sessionOptions.causallyConsistentReads) {
+                        // TODO SERVER-30679: We manually enable causal consistency on the
+                        // connection object so that "afterClusterTime" is injected into the
+                        // readConcern of any command requests through this connection.
+                        myDB.getMongo().setCausalConsistency();
+                    }
                 } else {
                     myDB = new Mongo(args.host).getDB(args.dbName);
                 }