SERVER-30677 Run the concurrency suite with CSRS primary stepdowns

6 files changed, 345 insertions, 1 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
index d7709c42789..894ddadd725 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded.yml
@@ -4,8 +4,9 @@ selector:
   roots:
   - jstests/concurrency/fsm_all_sharded*.js
   exclude_files:
-  # Skip causal consistency tests because they're run in separate suites
+  # Skip causal consistency and continuous stepdown tests because they're run in separate suites
   - jstests/concurrency/fsm_all_sharded_causal_consistency*.js
+  - jstests/concurrency/fsm_all_sharded_with_stepdowns*.js
 
 # Concurrency tests that run against a sharded cluster start one themselves.
 executor:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
new file mode 100644
index 00000000000..541a49dda06
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
@@ -0,0 +1,12 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_all_sharded_with_stepdowns.js
+
+# Concurrency tests that run against a sharded cluster start one themselves.
+executor:
+  config:
+    shell_options:
+      nodb: ''
+      readMode: commands
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
new file mode 100644
index 00000000000..d098731df2f
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
@@ -0,0 +1,12 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_all_sharded_with_stepdowns_and_balancer.js
+
+# Concurrency tests that run against a sharded cluster start one themselves.
+executor:
+  config:
+    shell_options:
+      nodb: ''
+      readMode: commands
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index ad435432c26..6b9838632dc 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -3224,6 +3224,42 @@ tasks:
       resmoke_args: --suites=concurrency_sharded_causal_consistency_and_balancer --storageEngine=wiredTiger
 
 - <<: *task_template
+  name: concurrency_sharded_with_stepdowns
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_with_stepdowns --storageEngine=mmapv1
+
+- <<: *task_template
+  name: concurrency_sharded_with_stepdowns_WT
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_with_stepdowns --storageEngine=wiredTiger
+
+- <<: *task_template
+  name: concurrency_sharded_with_stepdowns_and_balancer
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_with_stepdowns_and_balancer --storageEngine=mmapv1
+
+- <<: *task_template
+  name: concurrency_sharded_with_stepdowns_and_balancer_WT
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    timeout_secs: 21600 # 6 hour timeout for each test
+    vars:
+      resmoke_args: --suites=concurrency_sharded_with_stepdowns_and_balancer --storageEngine=wiredTiger
+
+- <<: *task_template
   name: concurrency_simultaneous
   commands:
   - func: "do setup"
@@ -5653,6 +5689,18 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
     distros:
     - windows-64-vs2015-large
+  - name: concurrency_sharded_with_stepdowns
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_with_stepdowns_WT
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+    distros:
+    - windows-64-vs2015-large
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
+    distros:
+    - windows-64-vs2015-large
   - name: concurrency_simultaneous
     distros:
     - windows-64-vs2015-large
@@ -6120,6 +6168,10 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_WT
   - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_WT
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -6423,6 +6475,10 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_WT
   - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_WT
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -6655,6 +6711,10 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_WT
   - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_WT
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -8721,6 +8781,8 @@ buildvariants:
   - name: concurrency_sharded
   - name: concurrency_sharded_causal_consistency
   - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
   - name: dbtest
   - name: failpoints
@@ -9158,6 +9220,10 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_WT
   - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_WT
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
@@ -9347,6 +9413,10 @@ buildvariants:
   - name: concurrency_sharded_causal_consistency_WT
   - name: concurrency_sharded_causal_consistency_and_balancer
   - name: concurrency_sharded_causal_consistency_and_balancer_WT
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_WT
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_sharded_with_stepdowns_and_balancer_WT
   - name: concurrency_simultaneous
   - name: concurrency_simultaneous_WT
   - name: dbtest
diff --git a/jstests/concurrency/fsm_all_sharded_with_stepdowns.js b/jstests/concurrency/fsm_all_sharded_with_stepdowns.js
new file mode 100644
index 00000000000..a5e8b6af7f6
--- /dev/null
+++ b/jstests/concurrency/fsm_all_sharded_with_stepdowns.js
@@ -0,0 +1,120 @@
+'use strict';
+
+load('jstests/concurrency/fsm_libs/runner.js');
+
+var dir = 'jstests/concurrency/fsm_workloads';
+
+var blacklist = [
+    // Disabled due to known bugs
+    'distinct.js',             // SERVER-13116 distinct isn't sharding aware
+    'distinct_noindex.js',     // SERVER-13116 distinct isn't sharding aware
+    'distinct_projection.js',  // SERVER-13116 distinct isn't sharding aware
+    'create_database.js',      // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'drop_database.js',        // SERVER-17397 Drops of sharded namespaces may not fully succeed
+
+    // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'.
+    // This bug is problematic for these workloads because they assert on count() values:
+    'agg_match.js',
+    'count.js',
+    'count_limit_skip.js',
+
+    // $lookup and $graphLookup are not supported on sharded collections.
+    'agg_graph_lookup.js',
+    'view_catalog_cycle_lookup.js',
+
+    // Disabled due to SERVER-20057, 'Concurrent, sharded mapReduces can fail when temporary
+    // namespaces collide across mongos processes'
+    'map_reduce_drop.js',
+    'map_reduce_inline.js',
+    'map_reduce_merge.js',
+    'map_reduce_merge_nonatomic.js',
+    'map_reduce_reduce.js',
+    'map_reduce_reduce_nonatomic.js',
+    'map_reduce_replace.js',
+    'map_reduce_replace_nonexistent.js',
+    'map_reduce_replace_remove.js',
+
+    // Disabled due to MongoDB restrictions and/or workload restrictions
+
+    // These workloads sometimes trigger 'Could not lock auth data update lock'
+    // errors because the AuthorizationManager currently waits for only five
+    // seconds to acquire the lock for authorization documents
+    'auth_create_role.js',
+    'auth_create_user.js',
+    'auth_drop_role.js',
+    'auth_drop_user.js',
+
+    'agg_group_external.js',                  // uses >100MB of data, which can overwhelm test hosts
+    'agg_sort_external.js',                   // uses >100MB of data, which can overwhelm test hosts
+    'compact.js',                             // compact can only be run against a standalone mongod
+    'compact_simultaneous_padding_bytes.js',  // compact can only be run against a mongod
+    'convert_to_capped_collection.js',        // convertToCapped can't be run on mongos processes
+    'convert_to_capped_collection_index.js',  // convertToCapped can't be run on mongos processes
+    'findAndModify_mixed_queue_unindexed.js',   // findAndModify requires a shard key
+    'findAndModify_remove_queue.js',            // remove cannot be {} for findAndModify
+    'findAndModify_remove_queue_unindexed.js',  // findAndModify requires a shard key
+    'findAndModify_update_collscan.js',         // findAndModify requires a shard key
+    'findAndModify_update_grow.js',             // can cause OOM kills on test hosts
+    'findAndModify_update_queue.js',            // findAndModify requires a shard key
+    'findAndModify_update_queue_unindexed.js',  // findAndModify requires a shard key
+    'group.js',                // the group command cannot be issued against a sharded cluster
+    'group_cond.js',           // the group command cannot be issued against a sharded cluster
+    'indexed_insert_eval.js',  // eval doesn't work with sharded collections
+    'indexed_insert_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // These workloads sometimes triggers an 'unable to target write op for collection ... caused by
+    // ... database not found' error. Further investigation still needs to be done, but these
+    // failures may be due to SERVER-17397 'drops in a sharded cluster may not fully succeed'
+    // because it drops and reuses the same namespaces.
+    'kill_multicollection_aggregation.js',
+    'invalidated_cursors.js',
+
+    'plan_cache_drop_database.js',  // cannot ensureIndex after dropDatabase without sharding first
+    'remove_single_document.js',    // our .remove(query, {justOne: true}) calls lack shard keys
+    'remove_single_document_eval.js',         // eval doesn't work with sharded collections
+    'remove_single_document_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // The rename_* workloads are disabled since renameCollection doesn't work with sharded
+    // collections
+    'rename_capped_collection_chain.js',
+    'rename_capped_collection_dbname_chain.js',
+    'rename_capped_collection_dbname_droptarget.js',
+    'rename_capped_collection_droptarget.js',
+    'rename_collection_chain.js',
+    'rename_collection_dbname_chain.js',
+    'rename_collection_dbname_droptarget.js',
+    'rename_collection_droptarget.js',
+
+    'update_simple_eval.js',           // eval doesn't work with sharded collections
+    'update_simple_eval_nolock.js',    // eval doesn't work with sharded collections
+    'update_upsert_multi.js',          // our update queries lack shard keys
+    'update_upsert_multi_noindex.js',  // our update queries lack shard keys
+    'upsert_where.js',      // cannot use upsert command with $where with sharded collections
+    'yield_and_hashed.js',  // stagedebug can only be run against a standalone mongod
+    'yield_and_sorted.js',  // stagedebug can only be run against a standalone mongod
+
+    // ChunkHelper directly talks to the config servers and doesn't support retries for network
+    // errors
+    'sharded_base_partitioned.js',
+    'sharded_mergeChunks_partitioned.js',
+    'sharded_moveChunk_drop_shard_key_index.js',
+    'sharded_moveChunk_partitioned.js',
+    'sharded_splitChunk_partitioned.js',
+
+    // These workloads frequently time out waiting for the distributed lock to drop a sharded
+    // collection.
+    'kill_aggregation.js',
+    'kill_rooted_or.js',
+    'view_catalog_cycle_with_drop.js',
+].map(function(file) {
+    return dir + '/' + file;
+});
+
+runWorkloadsSerially(
+    ls(dir).filter(function(file) {
+        return !Array.contains(blacklist, file);
+    }),
+    {
+      sharded: {enabled: true, stepdownOptions: {configStepdown: true, shardStepdown: false}},
+      replication: {enabled: true}
+    });
diff --git a/jstests/concurrency/fsm_all_sharded_with_stepdowns_and_balancer.js b/jstests/concurrency/fsm_all_sharded_with_stepdowns_and_balancer.js
new file mode 100644
index 00000000000..791ded5fa3c
--- /dev/null
+++ b/jstests/concurrency/fsm_all_sharded_with_stepdowns_and_balancer.js
@@ -0,0 +1,129 @@
+'use strict';
+
+load('jstests/concurrency/fsm_libs/runner.js');
+
+var dir = 'jstests/concurrency/fsm_workloads';
+
+var blacklist = [
+    // Disabled due to known bugs
+    'distinct.js',             // SERVER-13116 distinct isn't sharding aware
+    'distinct_noindex.js',     // SERVER-13116 distinct isn't sharding aware
+    'distinct_projection.js',  // SERVER-13116 distinct isn't sharding aware
+    'create_database.js',      // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'drop_database.js',        // SERVER-17397 Drops of sharded namespaces may not fully succeed
+    'remove_where.js',  // SERVER-14669 Multi-removes that use $where miscount removed documents
+
+    // Disabled due to SERVER-3645, '.count() can be wrong on sharded collections'.
+    // This bug is problematic for these workloads because they assert on count() values:
+    'agg_match.js',
+    'count.js',
+    'count_limit_skip.js',
+    'count_noindex.js',
+
+    // $lookup and $graphLookup are not supported on sharded collections.
+    'agg_graph_lookup.js',
+    'view_catalog_cycle_lookup.js',
+
+    // Disabled due to SERVER-20057, 'Concurrent, sharded mapReduces can fail when temporary
+    // namespaces collide across mongos processes'
+    'map_reduce_drop.js',
+    'map_reduce_inline.js',
+    'map_reduce_merge.js',
+    'map_reduce_merge_nonatomic.js',
+    'map_reduce_reduce.js',
+    'map_reduce_reduce_nonatomic.js',
+    'map_reduce_replace.js',
+    'map_reduce_replace_nonexistent.js',
+    'map_reduce_replace_remove.js',
+
+    // Disabled due to SERVER-13364, 'The geoNear command doesn't handle shard versioning, so a
+    // concurrent chunk migration may cause duplicate or missing results'
+    'yield_geo_near_dedup.js',
+
+    // Disabled due to MongoDB restrictions and/or workload restrictions
+
+    // These workloads sometimes trigger 'Could not lock auth data update lock'
+    // errors because the AuthorizationManager currently waits for only five
+    // seconds to acquire the lock for authorization documents
+    'auth_create_role.js',
+    'auth_create_user.js',
+    'auth_drop_role.js',
+    'auth_drop_user.js',
+
+    'agg_group_external.js',                  // uses >100MB of data, which can overwhelm test hosts
+    'agg_sort_external.js',                   // uses >100MB of data, which can overwhelm test hosts
+    'compact.js',                             // compact can only be run against a standalone mongod
+    'compact_simultaneous_padding_bytes.js',  // compact can only be run against a mongod
+    'convert_to_capped_collection.js',        // convertToCapped can't be run on mongos processes
+    'convert_to_capped_collection_index.js',  // convertToCapped can't be run on mongos processes
+    'findAndModify_mixed_queue_unindexed.js',   // findAndModify requires a shard key
+    'findAndModify_remove_queue.js',            // remove cannot be {} for findAndModify
+    'findAndModify_remove_queue_unindexed.js',  // findAndModify requires a shard key
+    'findAndModify_update_collscan.js',         // findAndModify requires a shard key
+    'findAndModify_update_grow.js',             // can cause OOM kills on test hosts
+    'findAndModify_update_queue.js',            // findAndModify requires a shard key
+    'findAndModify_update_queue_unindexed.js',  // findAndModify requires a shard key
+    'group.js',                // the group command cannot be issued against a sharded cluster
+    'group_cond.js',           // the group command cannot be issued against a sharded cluster
+    'indexed_insert_eval.js',  // eval doesn't work with sharded collections
+    'indexed_insert_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // These workloads sometimes triggers an 'unable to target write op for collection ... caused by
+    // ... database not found' error. Further investigation still needs to be done, but these
+    // failures may be due to SERVER-17397 'drops in a sharded cluster may not fully succeed'
+    // because it drops and reuses the same namespaces.
+    'kill_multicollection_aggregation.js',
+    'invalidated_cursors.js',
+
+    'plan_cache_drop_database.js',  // cannot ensureIndex after dropDatabase without sharding first
+    'remove_single_document.js',    // our .remove(query, {justOne: true}) calls lack shard keys
+    'remove_single_document_eval.js',         // eval doesn't work with sharded collections
+    'remove_single_document_eval_nolock.js',  // eval doesn't work with sharded collections
+
+    // The rename_* workloads are disabled since renameCollection doesn't work with sharded
+    // collections
+    'rename_capped_collection_chain.js',
+    'rename_capped_collection_dbname_chain.js',
+    'rename_capped_collection_dbname_droptarget.js',
+    'rename_capped_collection_droptarget.js',
+    'rename_collection_chain.js',
+    'rename_collection_dbname_chain.js',
+    'rename_collection_dbname_droptarget.js',
+    'rename_collection_droptarget.js',
+
+    'update_simple_eval.js',           // eval doesn't work with sharded collections
+    'update_simple_eval_nolock.js',    // eval doesn't work with sharded collections
+    'update_upsert_multi.js',          // our update queries lack shard keys
+    'update_upsert_multi_noindex.js',  // our update queries lack shard keys
+    'upsert_where.js',      // cannot use upsert command with $where with sharded collections
+    'yield_and_hashed.js',  // stagedebug can only be run against a standalone mongod
+    'yield_and_sorted.js',  // stagedebug can only be run against a standalone mongod
+
+    // ChunkHelper directly talks to the config servers and doesn't support retries for network
+    // errors
+    'sharded_base_partitioned.js',
+    'sharded_mergeChunks_partitioned.js',
+    'sharded_moveChunk_drop_shard_key_index.js',
+    'sharded_moveChunk_partitioned.js',
+    'sharded_splitChunk_partitioned.js',
+
+    // These workloads frequently time out waiting for the distributed lock to drop a sharded
+    // collection.
+    'kill_aggregation.js',
+    'kill_rooted_or.js',
+    'view_catalog_cycle_with_drop.js',
+].map(function(file) {
+    return dir + '/' + file;
+});
+
+runWorkloadsSerially(ls(dir).filter(function(file) {
+    return !Array.contains(blacklist, file);
+}),
+                     {
+                       sharded: {
+                           enabled: true,
+                           enableBalancer: true,
+                           stepdownOptions: {configStepdown: true, shardStepdown: false}
+                       },
+                       replication: {enabled: true}
+                     });