SERVER-40823 Enable Flow Control in testing

10 files changed, 311 insertions, 1 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 7705682cce1..bc245b2e682 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -54,6 +54,7 @@ DEFAULTS = {
     "dbtest_executable": None,
     "dry_run": None,
     "exclude_with_any_tags": None,
+    "flow_control": None,
     "genny_executable": None,
     "include_with_any_tags": None,
     "jobs": 1,
@@ -372,6 +373,9 @@ STAGGER_JOBS = None
 # If set to true, it enables read concern majority. Else, read concern majority is disabled.
 MAJORITY_READ_CONCERN = None
 
+# If set to true, it enables flow control. Else, flow control is disabled.
+FLOW_CONTROL = None
+
 # If set, then all mongod's started by resmoke.py and by the mongo shell will use the specified
 # storage engine.
 STORAGE_ENGINE = None
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 6ed11e8d28d..881c096d173 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -119,6 +119,9 @@ def mongod_program(  # pylint: disable=too-many-branches
     if "replSet" in kwargs and "waitForStepDownOnNonCommandShutdown" not in suite_set_parameters:
         suite_set_parameters["waitForStepDownOnNonCommandShutdown"] = False
 
+    if "enableFlowControl" not in suite_set_parameters:
+        suite_set_parameters["enableFlowControl"] = config.FLOW_CONTROL
+
     _apply_set_parameters(args, suite_set_parameters)
 
     shortcut_opts = {
@@ -271,6 +274,10 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     mongod_set_parameters.setdefault("logComponentVerbosity",
                                      default_mongod_log_component_verbosity())
 
+    # If the 'enableFlowControl' setParameter for mongod was not already specified, we set its value
+    # to a default.
+    mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL)
+
     # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its
     # value to a default.
     mongos_set_parameters.setdefault("logComponentVerbosity",
@@ -382,6 +389,8 @@ def dbtest_program(logger, executable=None, suites=None, process_kwargs=None, **
     if config.STORAGE_ENGINE is not None:
         kwargs["storageEngine"] = config.STORAGE_ENGINE
 
+    kwargs["flowControl"] = config.FLOW_CONTROL
+
     return generic_program(logger, args, process_kwargs=process_kwargs, **kwargs)
 
 
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index d03c7704a81..3d3bf63cb5b 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -248,6 +248,11 @@ def _make_parser():  # pylint: disable=too-many-statements
                  "off"), metavar="ON|OFF", help=("Enable or disable majority read concern support."
                                                  " Defaults to %default."))
 
+    parser.add_option("--flowControl", type="choice", action="store", dest="flow_control",
+                      choices=("on",
+                               "off"), metavar="ON|OFF", help=("Enable or disable flow control."
+                                                               " Defaults to %default."))
+
     parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE",
                       help="The storage engine used by dbtests and jstests.")
 
@@ -377,7 +382,8 @@ def _make_parser():  # pylint: disable=too-many-statements
 
     parser.set_defaults(benchrun_device="Desktop", dry_run="off", find_suites=False,
                         list_suites=False, logger_file="console", shuffle="auto",
-                        stagger_jobs="off", suite_files="with_server", majority_read_concern="on")
+                        stagger_jobs="off", suite_files="with_server", majority_read_concern="on",
+                        flow_control="on")
     return parser
 
 
@@ -565,6 +571,7 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements
     _config.EXCLUDE_WITH_ANY_TAGS.extend(
         utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
     _config.FAIL_FAST = not config.pop("continue_on_failure")
+    _config.FLOW_CONTROL = config.pop("flow_control") == "on"
     _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags"))
     _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable"))
     _config.JOBS = config.pop("jobs")
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index cb12e7117e8..4200036eb3c 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -388,6 +388,7 @@ variables:
   - enterprise-rhel-62-64-bit
   - enterprise-rhel-62-64-bit-coverage
   - enterprise-rhel-62-64-bit-inmem
+  - enterprise-rhel-62-64-bit-flow-control-off
   - enterprise-rhel-62-64-bit-majority-read-concern-off
   - enterprise-rhel-62-64-bit-required-inmem
   - enterprise-rhel-62-64-bit-required-majority-read-concern-off
@@ -11100,6 +11101,247 @@ buildvariants:
     - rhel62-large
   - name: secondary_reads_passthrough_gen
 
+- name: enterprise-rhel-62-64-bit-flow-control-off
+  display_name: "Enterprise RHEL 6.2 (flow control off)"
+  modules:
+  - enterprise
+  run_on:
+  - rhel62-small
+  expansions: &enterprise-rhel-62-64-bit-flow-control-off
+    test_flags: >-
+      --flowControl=off
+    compile_flags: >-
+      -j$(grep -c ^processor /proc/cpuinfo)
+      --ssl
+      --release
+      --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+      MONGO_DISTMOD=rhel62
+    multiversion_platform: rhel62
+    multiversion_edition: enterprise
+    repo_edition: enterprise
+    scons_cache_scope: shared
+    tooltags: "ssl sasl gssapi"
+    build_mongoreplay: true
+    large_distro_name: rhel62-large
+  display_tasks:
+  - *dbtest
+  tasks:
+  - name: compile_TG
+    distros:
+    - rhel62-large
+  - name: dbtest_TG
+    distros:
+    - rhel62-large
+  - name: rollback_fuzzer_gen
+  - name: rollback_fuzzer_clean_shutdowns_gen
+  - name: rollback_fuzzer_unclean_shutdowns_gen
+  - name: aggregation
+  - name: aggregation_ese
+  - name: aggregation_ese_gcm
+  - name: aggregation_auth
+  - name: aggregation_facet_unwind_passthrough
+  - name: aggregation_mongos_passthrough
+  - name: aggregation_one_shard_sharded_collections
+  - name: aggregation_read_concern_majority_passthrough
+  - name: aggregation_sharded_collections_passthrough
+  - name: audit
+  - name: auth_gen
+  - name: auth_audit_gen
+  - name: bulk_gle_passthrough
+  - name: causally_consistent_jscore_passthrough_gen
+  - name: causally_consistent_jscore_passthrough_auth_gen
+  - name: sharded_causally_consistent_jscore_passthrough_gen
+  - name: change_streams
+  - name: change_streams_mongos_passthrough
+  - name: change_streams_mongos_sessions_passthrough
+  - name: change_streams_secondary_reads
+  - name: change_streams_sharded_collections_passthrough
+  - name: change_streams_whole_db_passthrough
+  - name: change_streams_whole_db_mongos_passthrough
+  - name: change_streams_whole_db_secondary_reads_passthrough
+  - name: change_streams_whole_db_sharded_collections_passthrough
+  - name: change_streams_whole_cluster_passthrough
+  - name: change_streams_whole_cluster_mongos_passthrough
+  - name: change_streams_whole_cluster_secondary_reads_passthrough
+  - name: change_streams_whole_cluster_sharded_collections_passthrough
+  - name: concurrency
+  - name: concurrency_replication
+  - name: concurrency_replication_causal_consistency
+    distros:
+    - rhel62-large
+  - name: concurrency_replication_multi_stmt_txn
+  - name: concurrency_sharded_replication
+  - name: concurrency_sharded_replication_with_balancer
+  - name: concurrency_sharded_causal_consistency_gen
+  - name: concurrency_sharded_causal_consistency_and_balancer
+  - name: concurrency_sharded_local_read_write_multi_stmt_txn
+  - name: concurrency_sharded_local_read_write_multi_stmt_txn_with_balancer
+  - name: concurrency_sharded_multi_stmt_txn
+  - name: concurrency_sharded_multi_stmt_txn_with_balancer
+  - name: concurrency_sharded_multi_stmt_txn_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns
+  - name: concurrency_sharded_with_stepdowns_and_balancer
+  - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
+  - name: disk_wiredtiger
+  - name: ese
+  - name: failpoints
+  - name: failpoints_auth
+  - name: fle
+  - name: gle_auth
+  - name: gle_auth_basics_passthrough
+  - name: gle_auth_basics_passthrough_write_cmd
+  - name: gle_auth_write_cmd
+  - name: initial_sync_fuzzer_gen
+  - name: integration_tests_replset
+    distros:
+    - rhel62-large
+  - name: integration_tests_sharded
+    distros:
+    - rhel62-large
+  - name: integration_tests_standalone
+    distros:
+    - rhel62-large
+  - name: integration_tests_standalone_audit
+    distros:
+    - rhel62-large
+  - name: jsCore
+  - name: jsCore_ese
+  - name: jsCore_ese_gcm
+  - name: jsCore_auth
+  - name: jsCore_compatibility
+  - name: jsCore_decimal
+  - name: jsCore_minimum_batch_size
+  - name: jsCore_op_query
+  - name: jsCore_txns
+  - name: jsCore_txns_multi_oplog_entries
+  - name: causally_consistent_jscore_txns_passthrough
+  - name: jsonSchema
+  - name: aggregation_multiversion_fuzzer_gen
+  - name: aggregation_expression_multiversion_fuzzer_gen
+  - name: aggregation_wildcard_fuzzer_gen
+  - name: jstestfuzz_gen
+  - name: jstestfuzz_concurrent_gen
+  - name: jstestfuzz_concurrent_replication_gen
+  - name: jstestfuzz_concurrent_replication_session_gen
+  - name: jstestfuzz_concurrent_sharded_gen
+  - name: jstestfuzz_concurrent_sharded_causal_consistency_gen
+  - name: jstestfuzz_concurrent_sharded_continuous_stepdown_gen
+  - name: jstestfuzz_concurrent_sharded_session_gen
+  - name: jstestfuzz_interrupt_gen
+  - name: jstestfuzz_interrupt_replication_gen
+  - name: jstestfuzz_replication_gen
+  - name: jstestfuzz_replication_initsync_gen
+  - name: jstestfuzz_replication_session_gen
+  - name: jstestfuzz_sharded_gen
+  - name: jstestfuzz_sharded_causal_consistency_gen
+  - name: jstestfuzz_sharded_continuous_stepdown_gen
+  - name: jstestfuzz_sharded_session_gen
+  - name: logical_session_cache_replication_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen
+  - name: mongosTest
+  - name: multi_shard_local_read_write_multi_stmt_txn_jscore_passthrough_gen
+  - name: multi_shard_multi_stmt_txn_jscore_passthrough_gen
+  - name: multi_stmt_txn_jscore_passthrough_with_migration_gen
+  - name: multi_shard_multi_stmt_txn_kill_primary_jscore_passthrough_gen
+  - name: multiversion_auth
+  - name: multiversion
+  - name: noPassthroughWithMongod_gen
+  - name: noPassthrough_gen
+  - name: parallel_gen
+  - name: parallel_compatibility
+  - name: read_concern_linearizable_passthrough
+    distros:
+    - rhel62-large
+  - name: read_concern_majority_passthrough_gen
+  - name: read_only
+  - name: read_only_sharded
+  - name: replica_sets
+    distros:
+    - rhel62-large
+  - name: replica_sets_auth_gen
+  - name: replica_sets_ese_gen
+  - name: replica_sets_ese_gcm_gen
+  - name: replica_sets_jscore_passthrough
+  - name: replica_sets_initsync_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_initsync_static_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_multi_oplog_txns_gen
+  - name: replica_sets_multi_oplog_txns_jscore_passthrough
+  - name: replica_sets_multi_stmt_txn_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_multi_stmt_txn_stepdown_jscore_passthrough_gen
+    distros:
+    - rhel62-large
+  - name: replica_sets_multi_stmt_txn_kill_primary_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_multi_stmt_txn_terminate_primary_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_kill_primary_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_terminate_primary_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: replica_sets_kill_secondaries_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: retryable_writes_jscore_passthrough_gen
+  - name: retryable_writes_jscore_stepdown_passthrough
+    distros:
+    - rhel62-large
+  - name: sasl
+  - name: session_jscore_passthrough
+  - name: sharded_causally_consistent_jscore_txns_passthrough_gen
+  - name: sharded_collections_causally_consistent_jscore_txns_passthrough
+  - name: sharded_collections_jscore_passthrough
+  - name: sharded_jscore_txns
+  - name: sharded_jscore_txns_sharded_collections
+  - name: sharded_multi_stmt_txn_jscore_passthrough
+    distros:
+    - rhel62-large
+  - name: sharding_gen
+  - name: sharding_auth_gen
+  - name: sharding_auth_audit_gen
+  - name: sharding_ese_gen
+  - name: sharding_ese_gcm_gen
+  - name: sharding_gle_auth_basics_passthrough
+  - name: sharding_gle_auth_basics_passthrough_write_cmd
+  - name: sharding_jscore_passthrough
+  - name: sharding_jscore_op_query_passthrough
+  - name: sharding_jscore_passthrough_wire_ops_gen
+  - name: sharding_last_stable_mongos_and_mixed_shards_gen
+  - name: sharding_csrs_continuous_config_stepdown_gen
+  - name: sharding_op_query_gen
+  - name: slow1_gen
+  - name: serial_run
+  - name: snmp
+  - name: ssl_gen
+  - name: sslSpecial_gen
+  - name: tool
+  - name: update_fuzzer_gen
+  - name: update_fuzzer_replication_gen
+  - name: write_concern_majority_passthrough
+    distros:
+    - rhel62-large
+  - name: secondary_reads_passthrough_gen
+
 - name: enterprise-rhel-62-64-bit-required-majority-read-concern-off
   display_name: "! Enterprise RHEL 6.2 (majority read concern off)"
   modules:
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index c36878f6d82..ab6111bd83b 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -1070,6 +1070,36 @@ buildvariants:
     - name: linkbench
     - name: big_update
 
+- name: linux-3-node-replSet-flowcontrol
+  display_name: Linux 3-Node ReplSet with Flow Control
+  batchtime: 1440  # 24 hours
+  modules: *modules
+  expansions:
+    setup: replica-flowcontrol
+    cluster: replica
+    platform: linux
+    project_dir: *project_dir
+    authentication: enabled
+    storageEngine: wiredTiger
+  run_on:
+      - "rhel70-perf-replset"
+  depends_on:
+      - name: compile
+        variant: compile-linux-64-amzn
+  tasks:
+    - name: industry_benchmarks
+    - name: industry_benchmarks_secondary_reads
+    - name: crud_workloads
+    - name: service_architecture_workloads
+    - name: mixed_workloads
+    - name: smoke_test
+    - name: industry_benchmarks_wmajority
+    - name: change_streams_throughput
+    - name: change_streams_latency
+    - name: tpcc
+    - name: linkbench
+    - name: big_update
+
 - name: linux-3-node-replSet-ssl
   display_name: Linux 3-Node ReplSet (SSL)
   batchtime: 10080 # 7 days
diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index 8cf44ff1562..a070a881f0a 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -393,6 +393,7 @@ mongod = env.Program(
         'db/storage/devnull/storage_devnull',
         'db/storage/ephemeral_for_test/storage_ephemeral_for_test',
         'db/storage/flow_control',
+        'db/storage/flow_control_parameters',
         'db/storage/storage_engine_lock_file',
         'db/storage/storage_engine_metadata',
         'db/storage/storage_init_d',
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 474c05ac8ab..94ec16df197 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -127,6 +127,7 @@
 #include "mongo/db/stats/counters.h"
 #include "mongo/db/storage/backup_cursor_hooks.h"
 #include "mongo/db/storage/encryption_hooks.h"
+#include "mongo/db/storage/flow_control_parameters_gen.h"
 #include "mongo/db/storage/storage_engine.h"
 #include "mongo/db/storage/storage_engine_init.h"
 #include "mongo/db/storage/storage_engine_lock_file.h"
@@ -436,6 +437,10 @@ ExitCode _initAndListen(int listenPort) {
         invariant(serverGlobalParams.featureCompatibility.isVersionInitialized());
     }
 
+    if (gFlowControlEnabled.load()) {
+        log() << "Flow Control is enabled on this deployment.";
+    }
+
     if (storageGlobalParams.upgrade) {
         log() << "finished checking dbs";
         exitCleanly(EXIT_CLEAN);
diff --git a/src/mongo/dbtests/SConscript b/src/mongo/dbtests/SConscript
index ef20fb6a986..8f9f1286a5a 100644
--- a/src/mongo/dbtests/SConscript
+++ b/src/mongo/dbtests/SConscript
@@ -16,6 +16,7 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/db/server_options_core',
+        '$BUILD_DIR/mongo/db/storage/flow_control_parameters',
         '$BUILD_DIR/mongo/db/storage/storage_options',
         '$BUILD_DIR/mongo/util/options_parser/options_parser_init',
         '$BUILD_DIR/mongo/unittest/unittest',
diff --git a/src/mongo/dbtests/framework_options.cpp b/src/mongo/dbtests/framework_options.cpp
index c8940e3df68..ea4f54b65d8 100644
--- a/src/mongo/dbtests/framework_options.cpp
+++ b/src/mongo/dbtests/framework_options.cpp
@@ -40,6 +40,7 @@
 #include "mongo/base/status.h"
 #include "mongo/bson/util/builder.h"
 #include "mongo/db/query/find.h"
+#include "mongo/db/storage/flow_control_parameters_gen.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/dbtests/dbtests.h"
 #include "mongo/unittest/unittest.h"
@@ -123,6 +124,11 @@ Status storeTestFrameworkOptions(const moe::Environment& params,
     storageGlobalParams.dbpath = dbpathString.c_str();
 
     storageGlobalParams.engine = params["storage.engine"].as<string>();
+    gFlowControlEnabled.store(params["enableFlowControl"].as<bool>());
+
+    if (gFlowControlEnabled.load()) {
+        log() << "Flow Control enabled" << endl;
+    }
 
     if (storageGlobalParams.engine == "wiredTiger" &&
         params.count("replication.enableMajorityReadConcern")) {
diff --git a/src/mongo/dbtests/framework_options.idl b/src/mongo/dbtests/framework_options.idl
index 29311cae79a..a4be3951ee7 100644
--- a/src/mongo/dbtests/framework_options.idl
+++ b/src/mongo/dbtests/framework_options.idl
@@ -93,6 +93,11 @@ configs:
         arg_vartype: String
         short_name: storageEngine
         default: wiredTiger
+    enableFlowControl:
+        description: 'Whether Flow Control is enabled'
+        arg_vartype: Bool
+        short_name: flowControl
+        default: true
     suites:
         description: 'Test suites to run'
         arg_vartype: StringVector