# This suite is intended to exercise transaction behavior with failovers by running existing
# concurrency workloads with state functions wrapped in transactions while shard and config
# primaries are being killed. State functions that use operations not supported in a transaction
# are not wrapped in one.

test_kind: fsm_workload_test

selector:
  roots:
  - jstests/concurrency/fsm_workloads/**/*.js
  exclude_files:
  ##
  # Blacklists from concurrency_sharded_replication
  ##

  # SERVER-13116 distinct isn't sharding aware
  - jstests/concurrency/fsm_workloads/distinct.js
  - jstests/concurrency/fsm_workloads/distinct_noindex.js
  - jstests/concurrency/fsm_workloads/distinct_projection.js

  # SERVER-17397 Drops of sharded namespaces may not fully succeed
  - jstests/concurrency/fsm_workloads/create_database.js
  - jstests/concurrency/fsm_workloads/drop_database.js

  # Disabled due to SERVER-33753, '.count() without a predicate can be wrong on sharded
  # collections'. This bug is problematic for these workloads because they assert on count()
  # values:
  - jstests/concurrency/fsm_workloads/agg_match.js

  # $lookup and $graphLookup are not supported on sharded collections.
  - jstests/concurrency/fsm_workloads/agg_graph_lookup.js
  - jstests/concurrency/fsm_workloads/view_catalog_cycle_lookup.js

  # Disabled due to SERVER-20057, 'Concurrent, sharded mapReduces can fail when temporary
  # namespaces collide across mongos processes'
  - jstests/concurrency/fsm_workloads/map_reduce_drop.js
  - jstests/concurrency/fsm_workloads/map_reduce_inline.js
  - jstests/concurrency/fsm_workloads/map_reduce_interrupt.js
  - jstests/concurrency/fsm_workloads/map_reduce_merge.js
  - jstests/concurrency/fsm_workloads/map_reduce_merge_nonatomic.js
  - jstests/concurrency/fsm_workloads/map_reduce_reduce.js
  - jstests/concurrency/fsm_workloads/map_reduce_reduce_nonatomic.js
  - jstests/concurrency/fsm_workloads/map_reduce_replace.js
  - jstests/concurrency/fsm_workloads/map_reduce_replace_nonexistent.js
  - jstests/concurrency/fsm_workloads/map_reduce_replace_remove.js

  # Disabled due to MongoDB restrictions and/or workload restrictions

  # These workloads sometimes trigger 'Could not lock auth data update lock'
  # errors because the AuthorizationManager currently waits for only five
  # seconds to acquire the lock for authorization documents
  - jstests/concurrency/fsm_workloads/auth_create_role.js
  - jstests/concurrency/fsm_workloads/auth_create_user.js
  - jstests/concurrency/fsm_workloads/auth_drop_role.js
  - jstests/concurrency/fsm_workloads/auth_drop_user.js

  # uses >100MB of data, which can overwhelm test hosts
  - jstests/concurrency/fsm_workloads/agg_group_external.js
  - jstests/concurrency/fsm_workloads/agg_sort_external.js

  # compact can only be run against a standalone mongod
  - jstests/concurrency/fsm_workloads/compact.js
  - jstests/concurrency/fsm_workloads/compact_while_creating_indexes.js

  # convertToCapped can't be run on mongos processes
  - jstests/concurrency/fsm_workloads/convert_to_capped_collection.js
  - jstests/concurrency/fsm_workloads/convert_to_capped_collection_index.js

  # findAndModify requires a shard key
  - jstests/concurrency/fsm_workloads/findAndModify_mixed_queue_unindexed.js
  - jstests/concurrency/fsm_workloads/findAndModify_remove_queue_unindexed.js
  - jstests/concurrency/fsm_workloads/findAndModify_update_collscan.js
  - jstests/concurrency/fsm_workloads/findAndModify_update_queue.js
  - jstests/concurrency/fsm_workloads/findAndModify_update_queue_unindexed.js

  # remove cannot be {} for findAndModify
  - jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js

  # can cause OOM kills on test hosts
  - jstests/concurrency/fsm_workloads/findAndModify_update_grow.js

  # cannot ensureIndex after dropDatabase without sharding first
  - jstests/concurrency/fsm_workloads/plan_cache_drop_database.js

  # reIndex is not supported in mongos.
  - jstests/concurrency/fsm_workloads/reindex.js
  - jstests/concurrency/fsm_workloads/reindex_background.js

  # The WTWriteConflictException failpoint is not supported on mongos.
  - jstests/concurrency/fsm_workloads/collmod_writeconflict.js
  - jstests/concurrency/fsm_workloads/reindex_writeconflict.js

  # our .remove(query, {justOne: true}) calls lack shard keys
  - jstests/concurrency/fsm_workloads/remove_single_document.js

  # The rename_* workloads are disabled since renameCollection doesn't work with sharded
  # collections
  - jstests/concurrency/fsm_workloads/rename_capped_collection_chain.js
  - jstests/concurrency/fsm_workloads/rename_capped_collection_dbname_chain.js
  - jstests/concurrency/fsm_workloads/rename_capped_collection_dbname_droptarget.js
  - jstests/concurrency/fsm_workloads/rename_capped_collection_droptarget.js
  - jstests/concurrency/fsm_workloads/rename_collection_chain.js
  - jstests/concurrency/fsm_workloads/rename_collection_dbname_chain.js
  - jstests/concurrency/fsm_workloads/rename_collection_dbname_droptarget.js
  - jstests/concurrency/fsm_workloads/rename_collection_droptarget.js

  # cannot use upsert command with $where with sharded collections
  - jstests/concurrency/fsm_workloads/upsert_where.js

  # stagedebug can only be run against a standalone mongod
  - jstests/concurrency/fsm_workloads/yield_and_hashed.js
  - jstests/concurrency/fsm_workloads/yield_and_sorted.js

  # TODO Unblacklist (SERVER-35538).
  - jstests/concurrency/fsm_workloads/database_versioning.js

  # TODO Unblacklist (SERVER-38852).
  - jstests/concurrency/fsm_workloads/agg_out_interrupt_cleanup.js

  # serverStatus does not include transaction metrics on mongos.
  - jstests/concurrency/fsm_workloads/multi_statement_transaction_atomicity_isolation_metrics_test.js

  # Uses the same transaction id across different routers, which is not allowed because when either
  # router tries to commit, it may not know the full participant list.
  - jstests/concurrency/fsm_workloads/multi_statement_transaction_all_commands_same_session.js

  ##
  # Blacklists from concurrency_replication_multi_stmt_txn
  ##

  # Relies on having one thread observe writes from the other threads, which won't become visible
  # once a transaction in the thread is started because it'll keep reading from the same snapshot.
  - jstests/concurrency/fsm_workloads/create_index_background.js
  - jstests/concurrency/fsm_workloads/create_index_background_partial_filter.js
  - jstests/concurrency/fsm_workloads/create_index_background_wildcard.js

  # Expects reads to die with a particular error, but other errors are possible if the read is part
  # of a transaction (e.g. ErrorCodes.LockTimeout).
  - jstests/concurrency/fsm_workloads/drop_index_during_replan.js

  # Performs direct writes to system.views
  - jstests/concurrency/fsm_workloads/view_catalog_direct_system_writes.js

  ##
  # Blacklists from concurrency_sharded_multi_stmt_txn
  ##

  # Use updates that do not contain the shard key, so they are rejected before any commands are
  # sent to participant shards, but these workloads do not fail, so the auto retry transaction logic
  # attempts to commit, which fails because no participants have been contacted.
  - jstests/concurrency/fsm_workloads/update_rename.js
  - jstests/concurrency/fsm_workloads/update_rename_noindex.js

  # Expects DocumentValidationFailure which causes transaction to abort and retry indefinitely.
  - jstests/concurrency/fsm_workloads/schema_validator_with_expr_variables.js

  ##
  # Blacklists from concurrency_sharded_with_stepdowns
  ##

  # ChunkHelper directly talks to the config servers and doesn't support retries for network errors
  - jstests/concurrency/fsm_workloads/cleanupOrphanedWhileMigrating.js
  - jstests/concurrency/fsm_workloads/sharded_base_partitioned.js
  - jstests/concurrency/fsm_workloads/sharded_mergeChunks_partitioned.js
  - jstests/concurrency/fsm_workloads/sharded_moveChunk_drop_shard_key_index.js
  - jstests/concurrency/fsm_workloads/sharded_moveChunk_partitioned.js
  - jstests/concurrency/fsm_workloads/sharded_splitChunk_partitioned.js

  # These workloads frequently time out waiting for the distributed lock to drop a sharded
  # collection.
  - jstests/concurrency/fsm_workloads/kill_aggregation.js
  - jstests/concurrency/fsm_workloads/kill_rooted_or.js

  # Uses non retryable commands.
  - jstests/concurrency/fsm_workloads/agg_out.js
  - jstests/concurrency/fsm_workloads/agg_out_interrupt_cleanup.js
  - jstests/concurrency/fsm_workloads/agg_sort.js
  - jstests/concurrency/fsm_workloads/collmod.js
  - jstests/concurrency/fsm_workloads/collmod_separate_collections.js
  - jstests/concurrency/fsm_workloads/invalidated_cursors.js
  - jstests/concurrency/fsm_workloads/kill_multicollection_aggregation.js
  - jstests/concurrency/fsm_workloads/view_catalog.js
  - jstests/concurrency/fsm_workloads/view_catalog_cycle_with_drop.js
  - jstests/concurrency/fsm_workloads/view_catalog_direct_system_writes.js

  ##
  # Blacklists specific to this suite
  ##

  # Uses getMore in the same state function as a command not supported in a transaction.
  - jstests/concurrency/fsm_workloads/list_indexes.js
  - jstests/concurrency/fsm_workloads/create_capped_collection.js
  - jstests/concurrency/fsm_workloads/create_capped_collection_maxdocs.js

  # Uses non-retryable commands in the same state function as a command not supported in a
  # transaction.
  - jstests/concurrency/fsm_workloads/agg_merge_when_not_matched_insert.js
  - jstests/concurrency/fsm_workloads/agg_merge_when_matched_replace_with_new.js

  # TODO SERVER-40713 moveChunk is not considered retryable by the network retry override.
  - jstests/concurrency/fsm_workloads/agg_with_chunk_migrations.js
  - jstests/concurrency/fsm_workloads/random_moveChunk_broadcast_delete_transaction.js
  - jstests/concurrency/fsm_workloads/random_moveChunk_broadcast_update_transaction.js

  # TODO SERVER-40493: JS engine interruptions on mongos return ErrorCodes::Interrupted, which isn't
  # considered a retryable or transient transaction error.
  - jstests/concurrency/fsm_workloads/indexed_insert_where.js
  - jstests/concurrency/fsm_workloads/remove_where.js
  - jstests/concurrency/fsm_workloads/update_where.js

  # TODO SERVER-40705: Stepdown with prepared transaction that performed text search triggers
  # invariant
  - jstests/concurrency/fsm_workloads/indexed_insert_text.js
  - jstests/concurrency/fsm_workloads/indexed_insert_text_multikey.js

  exclude_with_any_tags:
  - requires_replication
  # The touch command does not exist on mongos, which is enforced before session information is
  # parsed, causing state functions in these workloads to fail before starting a transaction despite
  # sending a command with startTransaction=true, leading to an infinite loop in the auto
  # transaction retry override.
  - requires_touch
  # Sharing cursors between state functions will fail in this suite because it will attempt to use
  # the same cursor in multiple transactions.
  - state_functions_share_cursor
  # These start a transaction in one state function and use it in other state functions. This suite
  # would instead execute each state function as its own transaction.
  - state_functions_share_transaction
  # Note that "requires_non_retryable_writes" does not need to be blacklisted because some writes
  # that are not individually retryable can be retried if they are part of a transaction.

executor:
  archive:
    hooks:
      - CheckReplDBHash
      - ValidateCollections
    tests: true
  config:
    shell_options:
      readMode: commands
      global_vars:
        TestData:
          runInsideTransaction: true
          runningWithAutoSplit: false
          runningWithBalancer: false
          runningWithConfigStepdowns: true
          runningWithSessions: true
          runningWithShardStepdowns: true
          traceExceptions: false
          useStepdownPermittedFile: true
          usingReplicaSetShards: true
  hooks:
  # We use a stepdown interval of 15 seconds because we will retry all commands in a transaction
  # so we need to allow time for at most 10 operations to be re-run and then re-committed. If
  # too many network errors occur when re-running a transaction we will run out of retries.
  - class: ContinuousStepdown
    config_stepdown: true
    kill: true
    shard_stepdown: true
    stepdown_interval_ms: 15000
    use_stepdown_permitted_file: true
    wait_for_mongos_retarget: true
  - class: CheckReplDBHash
  - class: ValidateCollections
  - class: CleanupConcurrencyWorkloads
  fixture:
    class: ShardedClusterFixture
    enable_balancer: false
    enable_autosplit: false
    mongos_options:
      set_parameters:
        enableTestCommands: 1
    configsvr_options:
      num_nodes: 3
      all_nodes_electable: true
      replset_config_options:
        settings:
          catchUpTimeoutMillis: 0
      voting_secondaries: true
    shard_options:
      all_nodes_electable: true
      mongod_options:
        oplogSize: 1024
      replset_config_options:
        settings:
          catchUpTimeoutMillis: 0
      voting_secondaries: true
    mongod_options:
      set_parameters:
        enableTestCommands: 1
        enableElectionHandoff: 0
    num_rs_nodes_per_shard: 3
    num_shards: 2
    num_mongos: 2