SERVER-56164 system-perf.yml and perf.yml follow finalized DSI api

1 files changed, 180 insertions, 195 deletions

diff --git a/etc/perf.yml b/etc/perf.yml
index 3d07b51ce62..dcb3eebc8bb 100644
--- a/etc/perf.yml
+++ b/etc/perf.yml
@@ -20,16 +20,18 @@ variables:
 
   ###
   # **Or**: Leave this section uncommented to bypass/skip compile.
-  # # This file ↓ came from a microbenchmarks waterfall run.
-  # # https://evergreen.mongodb.com/version/performance_996dcdc3d96346d71f012388eccc79c691619340
-  # # Artifacts eventually expire. If this fails, grab the compile artifacts url and update this.
-  # _skip_remote_file: &_remote_file
-  #     perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
-  # _skip_compile: &_compile
-  #     []
-  # _skip_expansions: &_expansion_updates
-  #   - key: mdb_binary_for_server
-  #     value: https://mciuploads.s3.amazonaws.com/perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
+  # This file ↓ came from a microbenchmarks waterfall run.
+  # https://evergreen.mongodb.com/version/performance_996dcdc3d96346d71f012388eccc79c691619340
+  # Artifacts eventually expire. If this fails, grab the compile artifacts url and update this.
+#  _skip_remote_file: &_remote_file
+#      perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
+#  _skip_compile: &_compile
+#      []
+#  _skip_expansions: &_expansion_updates
+#    - key: mdb_binary_for_server
+#      value: https://mciuploads.s3.amazonaws.com/perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
+#    - key: mdb_binary_for_client
+#      value: https://mciuploads.s3.amazonaws.com/perf/5f6ca2392fbabe40badf39c4/c39af144b2370be0537410d9bc79be66a1a5f3c7/linux/mongodb-5f6ca2392fbabe40badf39c4.tar.gz
   ###
 
   _src_dir: &src_dir src/mongo
@@ -47,93 +49,96 @@ variables:
     - benchmarks
     - py-tpcc
 
-
 modules:
-###
-# Same in every DSI project
-- name: dsi
-  repo: git@github.com:10gen/dsi.git
-  prefix: ../../src
-  branch: master
-- name: genny
-  repo: git@github.com:10gen/genny.git
-  prefix: ../../src
-  branch: master
-- name: signal-processing
-  repo: git@github.com:10gen/signal-processing.git
-  prefix: ../../src
-  branch: master
-- name: workloads
-  repo: git@github.com:10gen/workloads.git
-  prefix: workloads
-  branch: master
-- name: linkbench
-  repo: git@github.com:10gen/linkbench.git
-  prefix: ../../src
-  branch: master
-- name: linkbench2
-  repo: git@github.com:10gen/linkbench2.git
-  prefix: ../../src
-  branch: master
-- name: mongo-perf
-  repo: git@github.com:mongodb/mongo-perf.git
-  prefix: ../../src
-  branch: master
-- name: YCSB
-  repo: git@github.com:mongodb-labs/YCSB.git
-  prefix: ../../src
-  branch: master
-  ref: 4e7287880c04514cad2df5761b9511c940a33059
-- name: benchmarks
-  repo: git@github.com:mongodb-labs/benchmarks.git
-  prefix: ../../src
-  branch: master
-- name: py-tpcc
-  repo: git@github.com:mongodb-labs/py-tpcc.git
-  prefix: ../../src
-  branch: master
-  ref: 2d19705337a40e24831a904266a648b85df5be84
+  ###
+  # Same in every DSI project. Ensure that this block is synchronized with
+  # evergreen-dsitest.yml, atlas/system_perf_atlas.yml, and src/dsi/onboarding.py
+  # (search update-repos-here) in this repo, and etc/system_perf.yml and
+  # etc/perf.yml in mongodb/mongo
+  - name: dsi
+    repo: git@github.com:10gen/dsi.git
+    prefix: ../../src
+    branch: master
+  - name: genny
+    repo: git@github.com:10gen/genny.git
+    prefix: ../../src
+    branch: master
+  - name: signal-processing
+    repo: git@github.com:10gen/signal-processing.git
+    prefix: ../../src
+    branch: master
+  - name: workloads
+    repo: git@github.com:10gen/workloads.git
+    prefix: ../../src
+    branch: master
+  - name: linkbench
+    repo: git@github.com:10gen/linkbench.git
+    prefix: ../../src
+    branch: master
+  - name: linkbench2
+    repo: git@github.com:10gen/linkbench2.git
+    prefix: ../../src
+    branch: master
+  - name: mongo-perf
+    repo: git@github.com:mongodb/mongo-perf.git
+    prefix: ../../src
+    branch: master
+  - name: YCSB
+    repo: git@github.com:mongodb-labs/YCSB.git
+    prefix: ../../src
+    branch: master
+    ref: 4e7287880c04514cad2df5761b9511c940a33059
+  - name: benchmarks
+    repo: git@github.com:mongodb-labs/benchmarks.git
+    prefix: ../../src
+    branch: master
+  - name: py-tpcc
+    repo: git@github.com:mongodb-labs/py-tpcc.git
+    prefix: ../../src
+    branch: master
+    ref: 2d19705337a40e24831a904266a648b85df5be84
 
-###
-- name: enterprise
-  repo: git@github.com:10gen/mongo-enterprise-modules.git
-  prefix: src/mongo/db/modules
-  branch: master
-- name: mongo-tools
-  repo: git@github.com:mongodb/mongo-tools.git
-  prefix: mongo-tools/src/github.com/mongodb
-  branch: master
+  ###
+#  - name: mongo
+#    repo: git@github.com:mongodb/mongo.git
+#    prefix: ../../src
+#    branch: master
+  - name: enterprise
+    repo: git@github.com:10gen/mongo-enterprise-modules.git
+    prefix: src/mongo/db/modules
+    branch: master
+  - name: mongo-tools
+    repo: git@github.com:mongodb/mongo-tools.git
+    prefix: mongo-tools/src/github.com/mongodb
+    branch: master
 
 
 ###
 # Same in every DSI project
 pre:
-    - func: "f_other_pre_ops"
-    - func: "f_dsi_pre_run"
+  - func: f_other_pre_ops
+  - func: f_dsi_pre_run
 post:
-    - func: "f_dsi_post_run"
-    - func: "f_other_post_ops"
+  - func: f_dsi_post_run
+  - func: f_other_post_ops
+timeout:
+  - func: f_dsi_timeout
+  - func: f_other_timeout
 ###
 
 functions:
   ###
   # Same in every DSI project
-  "f_dsi_pre_run":
+  f_dsi_pre_run:
     - command: manifest.load
     - command: expansions.update
       params:
         updates: *_expansion_updates
-  "f_run_dsi_workload":
-    - command: shell.exec
-      params:
-        script: |
-          rm -rf ./*
-          mkdir src
-    - command: manifest.load
+  f_run_dsi_workload:
     - command: git.get_project
       params:
         directory: *src_dir
-        revisions: &revisions_list
+        revisions:
           dsi: ${dsi_rev}
           genny: ${genny_rev}
           signal-processing: ${signal-processing_rev}
@@ -149,7 +154,6 @@ functions:
         file: ./expansions.yml
     - command: shell.exec
       params:
-        continue_on_err: true
         script: ./src/dsi/run-dsi run_workload
     - command: shell.exec
       type: system
@@ -163,89 +167,44 @@ functions:
       type: test
       params:
         script: ./src/dsi/run-dsi determine_failure -m TEST
+  f_dsi_post_run:
     - command: shell.exec
-      type: test
       params:
-        script: |
-          ./src/dsi/run-dsi analysis
-          # detect outliers needs to run, so defer the post_run_check exit status to later
-          echo $? > post_run_check.status
+        script: ./src/dsi/run-dsi post_run
+    - command: attach.results
+      params:
+        file_location: ./build/EvergreenResultsJson/results.json
     - command: json.send
       params:
-        name: "perf"
-        file: "./build/LegacyPerfJson/perf.json"
-    - command: shell.exec
+        name: perf
+        file: ./build/LegacyPerfJson/perf.json
+    - command: s3.put
       params:
-        script: |
-          set -o errexit
-          is_patch=${is_patch}
-          task_id=${task_id}
-          perf_jira_user=${perf_jira_user}
-          perf_jira_pw=${perf_jira_pw}
-          analysis_user=${dsi_analysis_atlas_user}
-          analysis_password=${dsi_analysis_atlas_pw}
-          evergreen_api_key=${evergreen_api_key}
-          evergreen_api_user=${evergreen_api_user}
-          source ./src/dsi/src/signal_processing_setup.sh
-    - command: shell.exec
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ./build/Artifacts/DSIArtifacts.tgz
+        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/x-gzip
+        display_name: DSI Artifacts - Execution ${execution}
+    - command: s3.put
       params:
-        script: |
-          set -o verbose
-          source ./signal_processing_venv/bin/activate
-          detect-changes --config .signal-processing.yml --mongo-repo=./src/mongo -v
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ./build/Documentation/index.html
+        remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-index.html
+        bucket: mciuploads
+        permissions: public-read
+        content_type: text/html
+        display_name: Documentation
+  f_dsi_timeout:
     - command: shell.exec
-      type: test
       params:
-        script: |
-          set -o verbose
-          exit $(cat post_run_check.status)
-  "f_dsi_post_run":
-      - command: shell.exec
-        params:
-          script: ./src/dsi/run-dsi infrastructure_teardown
-      - command: shell.exec
-        params:
-          script: ./src/dsi/run-dsi ./src/dsi/src/dsi/make_artifact.sh
-      - command: s3.put
-        params:
-          aws_key: ${aws_key}
-          aws_secret: ${aws_secret}
-          local_file: dsi-artifacts.tgz
-          remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/dsi-artifacts-${task_name}-${build_id}-${execution}.${ext|tgz}
-          bucket: mciuploads
-          permissions: public-read
-          content_type: ${content_type|application/x-gzip}
-          display_name: Dsi Artifacts - Execution ${execution}
-      - command: s3.put
-        params:
-          aws_key: ${aws_key}
-          aws_secret: ${aws_secret}
-          local_file: src/mongo/workloads/workloads/jsdoc/jsdocs-redirect.html
-          remote_file: ${project_dir}/${build_variant}/${revision}/${task_id}/${version_id}/logs/workloads-${task_name}-${build_id}.html
-          bucket: mciuploads
-          permissions: public-read
-          content_type: text/html
-          display_name: workloads documentation
-      - command: attach.results
-        params:
-          file_location: report.json
-      - command: json.send
-        params:
-          name: "perf"
-          file: "./build/LegacyPerfJson/perf.json"
-      - command: s3.put
-        params:
-          aws_key: ${aws_key}
-          aws_secret: ${aws_secret}
-          local_file: pip-requirements.txt
-          remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt
-          bucket: mciuploads
-          permissions: public-read
-          content_type: atext-plain
-          display_name: Pip Requirements
+        script: ./src/dsi/run-dsi on_timeout
   ###
 
-  "f_other_post_ops":
+  f_other_post_ops:
       - command: shell.exec
         params:
           working_dir: src
@@ -257,7 +216,7 @@ functions:
             if [ -d "${scons_cache_path}" ]; then
                 /opt/mongodbtoolchain/v3/bin/python3 buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
             fi
-  "f_other_pre_ops":
+  f_other_pre_ops:
     - &f_other_pre_ops
       command: shell.exec
       params:
@@ -266,10 +225,14 @@ functions:
           for PS in mongo{,d,s,import,export,dump,restore,stat,files,top,bridge} resmoke.py python{,2} lldb _test; do
               pkill -9 "$PS"
           done
+  f_other_timeout:
+    # Can't be empty so just `echo`.
+    - command: shell.exec
+      params: {script: "echo"}
 
   ###
   # Compile
-  "compile mongodb":
+  compile mongodb:
     # We create a virtual environment with the Python dependencies for compiling the server
     # installed.
     - command: shell.exec
@@ -371,77 +334,99 @@ functions:
         display_name: mongodb${compile-variant|}.tar.gz
   ###
 
-tasks:
-###
-# Same in every DSI project
-- name: genny_generate_all_tasks
-  commands:
+  ## Schedule Tasks ##
+  f_schedule_tasks:
     - command: git.get_project
       params:
         directory: *src_dir
-        revisions: *revisions_list
+        revisions:
+          dsi: ${dsi_rev}
+          genny: ${genny_rev}
+          signal-processing: ${signal-processing_rev}
+          linkbench: ${linkbench_rev}
+          linkbench2: ${linkbench2_rev}
+          workloads: ${workloads_rev}
+          mongo-perf: ${mongo-perf_rev}
+          YCSB: ${YCSB_rev}
+          benchmarks: ${benchmarks_rev}
+          py-tpcc: ${py-tpcc_rev}
     - command: expansions.write
       params:
         file: ./expansions.yml
     - command: shell.exec
       params:
-        script: ./src/genny/scripts/genny_auto_tasks.sh all_tasks
+        script: ./src/dsi/run-dsi schedule_tasks --tasks=${tasks}
     - command: generate.tasks
       params:
         files:
           - build/TaskJSON/Tasks.json
-- name: genny_auto_tasks
+
+
+tasks:
+###
+# Same in every DSI project
+- name: schedule_global_auto_tasks
+  priority: 5
   commands:
-    - command: git.get_project
-      params:
-        directory: *src_dir
-        revisions: *revisions_list
-    - command: expansions.write
-      params:
-        file: ./expansions.yml
-    - command: shell.exec
-      params:
-        script: ./src/genny/scripts/genny_auto_tasks.sh variant_tasks
-    - command: generate.tasks
-      params:
-        files:
-          - build/TaskJSON/Tasks.json
-- name: genny_patch_tasks
+    - func: f_schedule_tasks
+      vars:
+        tasks: all_tasks
+- name: schedule_variant_auto_tasks
+  priority: 5
   commands:
-    - command: git.get_project
-      params:
-        directory: *src_dir
-        revisions: *revisions_list
-    - command: expansions.write
-      params:
-        file: ./expansions.yml
-    - command: shell.exec
-      params:
-        script: ./src/genny/scripts/genny_auto_tasks.sh patch_tasks
-    - command: generate.tasks
-      params:
-        files:
-          - build/TaskJSON/Tasks.json
+    - func: f_schedule_tasks
+      vars:
+        tasks: variant_tasks
+- name: schedule_patch_auto_tasks
+  priority: 5
+  commands:
+    - func: f_schedule_tasks
+      vars:
+        tasks: patch_tasks
 - name: smoke_test
   priority: 5
   commands:
     - func: f_run_dsi_workload
       vars:
-        test_control: "short"
+        test_control: short
 - name: smoke_test_ssl
   priority: 5
   commands:
     - func: f_run_dsi_workload
       vars:
-          test_control: short
-          mongodb_setup: replica-ssl
-          infrastructure_provisioning: replica
+        test_control: short
+        mongodb_setup: replica-ssl
+        infrastructure_provisioning: replica
+- name: smoke_test_standalone_auth
+  priority: 5
+  commands:
+    - func: f_run_dsi_workload
+      vars:
+        test_control: short
+        mongodb_setup: standalone-auth
+        infrastructure_provisioning: single
+- name: smoke_test_replset_auth
+  priority: 5
+  commands:
+    - func: f_run_dsi_workload
+      vars:
+        test_control: short
+        mongodb_setup: replica-auth
+        infrastructure_provisioning: replica
+- name: smoke_test_shard_lite_auth
+  priority: 5
+  commands:
+    - func: f_run_dsi_workload
+      vars:
+        test_control: short
+        mongodb_setup: shard-lite-auth
+        infrastructure_provisioning: shard-lite
 - name: dsi_integ_test_run_command_simple
   priority: 5
   commands:
     - func: f_run_dsi_workload
       vars:
-        test_control: "run_command_simple"
+        test_control: run_command_simple
 ###
 
 - name: compile