SERVER-56769 Use tarball from arhive_dist_test task instead of git.get_project

(cherry picked from commit 0089b70cca49afd81834c8d051c986e6232d409d)

1 files changed, 210 insertions, 111 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 85c528bd55c..c20d425423f 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -19,6 +19,32 @@
 # - 'build_variant' (references the executing task's buildvariant).
 # - 'config_root' (references the root directory for the executor's configuration artifacts).
 
+
+#####################################################
+#          Setup environment in a new task          #
+#####################################################
+
+# There are several ways to setup the environment in your task.
+#
+# 1. If your task depends on 'archive_dist_test'/'archive_dist_test_debug' task you can call the function "do setup"
+# - func: "do setup"
+# Or alternatively call the functions in the sequence below, if you don't need everything else from "do setup"
+# - func: "fetch artifacts" (includes python, shell scripts, jstests etc. from the mongo and enterprise repos)
+# - func: "f_expansions_write"
+# - func: "kill processes"
+# - func: "cleanup environment"
+# - func: "set up venv"
+#
+# 2. If your task does not depend on 'archive_dist_test'/'archive_dist_test_debug' task use the following functions
+# call sequence
+# - command: manifest.load
+# - func: "git get project" (clone the entire mongo and enterprise repos)
+# - func: "f_expansions_write"
+# - func: "kill processes"
+# - func: "cleanup environment"
+# - func: "set up venv"
+
+
 stepback: true
 command_type: system
 pre_error_fails_task: true
@@ -171,12 +197,12 @@ variables:
         ${archive_file|src/archive.json}
   setup_group_can_fail_task: true
   setup_group:
-  - func: "f_expansions_write"
   - command: manifest.load
   - func: "git get project"
+  - func: "set task expansion macros"
+  - func: "f_expansions_write"
   - func: "kill processes"
   - func: "cleanup environment"
-  - func: "set task expansion macros"
   # The python virtual environment is installed in ${workdir}, which is created in
   # "set up venv".
   - func: "set up venv"
@@ -250,15 +276,17 @@ variables:
   setup_task:
     - func: "apply compile expansions"
     - func: "set task expansion macros"
+    - func: "f_expansions_write"
   teardown_task:
   - func: "attach scons logs"
   setup_group_can_fail_task: true
   setup_group:
     - command: manifest.load
     - func: "git get project"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "kill processes"
     - func: "cleanup environment"
-    - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
@@ -403,11 +431,11 @@ functions:
         workdir: ${workdir}
 
   "git get project": &git_get_project
-    - command: git.get_project
-      params:
-        directory: ${git_project_directory|src}
-        revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
-          enterprise: ${enterprise_rev}
+    command: git.get_project
+    params:
+      directory: ${git_project_directory|src}
+      revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
+        enterprise: ${enterprise_rev}
 
   "fetch artifacts": &fetch_artifacts
     command: s3.get
@@ -710,7 +738,6 @@ functions:
       exec_timeout_secs: ${exec_timeout_secs}
 
   ### Set expansion macros used in each task.
-  ### Note that this function will override task expansions set in update_bypass_expansions
   "set task expansion macros": &set_task_expansion_macros
     command: expansions.update
     params:
@@ -766,15 +793,30 @@ functions:
       name: perf
       file: src/perf.json
 
+  "cleanup environment": &cleanup_environment
+    command: subprocess.exec
+    params:
+      binary: bash
+      args:
+        - "./src/evergreen/cleanup_environment.sh"
+
+  "kill processes": &kill_processes
+    command: subprocess.exec
+    params:
+      silent: true
+      binary: bash
+      args:
+        - "./src/evergreen/kill_processes.sh"
+
   "do setup":
   - *f_expansions_write
-  - *set_up_venv
-  - *upload_pip_requirements
-  - *set_task_expansion_macros
-  - *f_expansions_write
   - *fetch_artifacts
+  - *kill_processes
+  - *cleanup_environment
   - *fetch_binaries
   - *fetch_debugsymbols_archive
+  - *set_up_venv
+  - *upload_pip_requirements
   - *extract_binaries
   - *check_binary_version
   - *get_buildnumber
@@ -785,9 +827,11 @@ functions:
   - *collect_system_resource_info
 
   "do non-compile setup":
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - *set_task_expansion_macros
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
@@ -795,9 +839,11 @@ functions:
   - *set_up_credentials
 
   "do benchmark setup":
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - *set_task_expansion_macros
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
@@ -825,7 +871,11 @@ functions:
         - "./src/evergreen/compiled_binaries_get.sh"
 
   "generate powercycle tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
     - *upload_pip_requirements
 
@@ -859,7 +909,11 @@ functions:
           - powercycle_tasks.json
 
   "run powercycle sentinel":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
     - *upload_pip_requirements
     - *configure_evergreen_api_credentials
@@ -898,9 +952,6 @@ functions:
 
   "generate selected tests":
     - *f_expansions_write
-    - command: expansions.write
-      params:
-        file: expansions.yml
     - *configure_evergreen_api_credentials
     - *configure_selected_tests_credentials
     - command: subprocess.exec
@@ -963,9 +1014,11 @@ functions:
           - src/generated_burn_in_tags_config/burn_in_tags_gen.json
 
   "generate randomized multiversion tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - *set_task_expansion_macros
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
     - *upload_pip_requirements
     - *configure_evergreen_api_credentials
@@ -1018,9 +1071,11 @@ functions:
           - src/generated_resmoke_config/*.json
 
   "generate resmoke tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - *set_task_expansion_macros
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
     - *upload_pip_requirements
     - *f_expansions_write
@@ -1214,13 +1269,12 @@ functions:
 
 
   "generate explicit multiversion tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - *set_task_expansion_macros
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
-    - command: expansions.write
-      params:
-        file: expansions.yml
     - *upload_pip_requirements
     - *f_expansions_write
     - *configure_evergreen_api_credentials
@@ -1266,15 +1320,12 @@ functions:
           - src/generated_resmoke_config/*.json
 
   "generate implicit multiversion tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - *set_task_expansion_macros
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
-    - *f_expansions_write
-    - *configure_evergreen_api_credentials
-    - command: expansions.write
-      params:
-        file: expansions.yml
     - *configure_evergreen_api_credentials
     - *f_expansions_write
     - *do_multiversion_setup
@@ -1321,13 +1372,12 @@ functions:
           - src/generated_resmoke_config/*.json
 
   "generate fuzzer tasks":
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - *set_task_expansion_macros
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - *set_up_venv
-    - command: expansions.write
-      params:
-        file: expansions.yml
     - *upload_pip_requirements
 
     - *f_expansions_write
@@ -1454,12 +1504,6 @@ functions:
         ignore_missing_file: true
         file: src/powercycle_exit.yml
     - *f_expansions_write
-
-    - command: expansions.write
-      params:
-        file: expansions.yml
-
-    - *f_expansions_write
     - command: subprocess.exec
       type: setup
       params:
@@ -1495,21 +1539,6 @@ functions:
       args:
         - "./src/evergreen/do_watchdog_setup.sh"
 
-  "cleanup environment":
-    command: subprocess.exec
-    params:
-      binary: bash
-      args:
-        - "./src/evergreen/cleanup_environment.sh"
-
-  "kill processes":
-    command: subprocess.exec
-    params:
-      silent: true
-      binary: bash
-      args:
-        - "./src/evergreen/kill_processes.sh"
-
   "run kitchen":
     - *f_expansions_write
     - command: subprocess.exec
@@ -1997,12 +2026,6 @@ functions:
 
 # Pre task steps
 pre:
-  - func: "f_expansions_write"
-  - command: manifest.load
-  - func: "git get project"
-  - func: "kill processes"
-  - func: "cleanup environment"
-  - func: "set up venv"
   - func: "set task expansion macros"
   - func: "f_expansions_write"
 
@@ -2094,6 +2117,7 @@ tasks:
         include:
           - "./build/**.gcno"
           - "./etc/*san.suppressions"
+          - "./etc/expansions.default.yml"
           - "./etc/pip/**"
           - "./etc/repo_config.yaml"
           - "./etc/scons/**"
@@ -2104,7 +2128,7 @@ tasks:
           - "jstests/**"
           - "library_dependency_graph.json"
           - "patch_files.txt"
-          - "pytests/**"
+          - "evergreen/**"
           - "src/mongo/client/sdam/json_tests/sdam_tests/**"
           - "src/mongo/client/sdam/json_tests/server_selection_tests/**"
           - "src/mongo/db/modules/enterprise/docs/**"
@@ -2627,9 +2651,11 @@ tasks:
 - name: compile_benchmarks
   depends_on: []
   commands:
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - func: "set task expansion macros"
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
@@ -2669,14 +2695,16 @@ tasks:
 ## lint ##
 - name: lint_pylinters
   commands:
-    - *f_expansions_write
     - command: timeout.update
       params:
         # 40 minutes
         exec_timeout_secs: 2400
     - *f_expansions_write
-    - func: "set task expansion macros"
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "scons lint"
@@ -2685,14 +2713,16 @@ tasks:
 
 - name: lint_clang_format
   commands:
-    - *f_expansions_write
     - command: timeout.update
       params:
         # 40 minutes
         exec_timeout_secs: 2400
     - *f_expansions_write
-    - func: "set task expansion macros"
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "scons lint"
@@ -2701,14 +2731,16 @@ tasks:
 
 - name: lint_eslint
   commands:
-    - *f_expansions_write
     - command: timeout.update
       params:
         # 40 minutes
         exec_timeout_secs: 2400
     - *f_expansions_write
-    - func: "set task expansion macros"
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "scons lint"
@@ -2717,14 +2749,16 @@ tasks:
 
 - name: lint_cpplint
   commands:
-    - *f_expansions_write
     - command: timeout.update
       params:
         # 40 minutes
         exec_timeout_secs: 2400
     - *f_expansions_write
-    - func: "set task expansion macros"
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "scons lint"
@@ -2734,9 +2768,11 @@ tasks:
 - name: lint_yaml
   depends_on: []
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "f_expansions_write"
@@ -2749,7 +2785,11 @@ tasks:
 
 - name: lint_shellscripts
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - command: subprocess.exec
     type: test
     params:
@@ -2759,9 +2799,11 @@ tasks:
 
 - name: lint_errorcodes
   commands:
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - func: "set task expansion macros"
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "scons lint"
@@ -2772,9 +2814,11 @@ tasks:
   depends_on:
     - name: archive_dist_test_debug
   commands:
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - func: "set task expansion macros"
-    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "do setup"
@@ -2788,9 +2832,11 @@ tasks:
 
 - name: burn_in_tests_gen
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "configure evergreen api credentials"
@@ -2825,9 +2871,11 @@ tasks:
 
 - name: burn_in_tests_multiversion_gen
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "f_expansions_write"
@@ -3748,9 +3796,12 @@ tasks:
 - name: lint_fuzzer_sanity_patch
   patch_only: true
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - *set_up_venv
   - func: "get added and modified patch files"
   - func: "setup jstestfuzz"
   - func: "lint fuzzer sanity patch"
@@ -3758,9 +3809,12 @@ tasks:
 # Check that the mutational fuzzer can parse all JS filess.
 - name: lint_fuzzer_sanity_all
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - *set_up_venv
   - func: "setup jstestfuzz"
   - func: "lint fuzzer sanity all"
 
@@ -3919,9 +3973,11 @@ tasks:
   depends_on:
     - name: archive_dist_test_debug
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "configure evergreen api credentials"
@@ -3935,9 +3991,11 @@ tasks:
 
 - name: selected_tests_gen
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "generate selected tests"
@@ -4233,6 +4291,8 @@ tasks:
   name: integration_tests_standalone
   tags: ["integration", "standalone"]
   commands:
+  - command: manifest.load
+  - *git_get_project
   - func: "do setup"
   - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
@@ -4250,6 +4310,8 @@ tasks:
   name: integration_tests_standalone_audit
   tags: ["integration", "audit"]
   commands:
+  - command: manifest.load
+  - *git_get_project
   - func: "do setup"
   - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
@@ -4267,6 +4329,8 @@ tasks:
   name: integration_tests_replset
   tags: ["integration"]
   commands:
+  - command: manifest.load
+  - *git_get_project
   - func: "do setup"
   - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
@@ -4284,6 +4348,8 @@ tasks:
   name: integration_tests_replset_ssl_auth
   tags: ["integration"]
   commands:
+  - command: manifest.load
+  - *git_get_project
   - func: "do setup"
   - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
@@ -4301,6 +4367,8 @@ tasks:
   name: integration_tests_sharded
   tags: ["integration", "sharded"]
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
   - func: "do setup"
   - func: "set up win mount script"
@@ -6100,9 +6168,11 @@ tasks:
   depends_on:
   - name: archive_dist_test_debug
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
   - func: "upload pip requirements"
   - func: "run idl tests"
@@ -6167,6 +6237,12 @@ tasks:
   depends_on:
   - name: package
   commands:
+    - command: manifest.load
+    - *git_get_project
+    - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
+    - *set_up_venv
     - func: "fetch packages"
     - func: "set up remote credentials"
       vars:
@@ -6180,7 +6256,12 @@ tasks:
   depends_on:
     - name: compile_dist_test
   commands:
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
+    - *kill_processes
+    - *cleanup_environment
+    - *set_up_venv
     - func: "scons compile"
       vars:
         targets: >-
@@ -6936,23 +7017,36 @@ tasks:
   exec_timeout_secs: 7200 # 2 hour timeout for the task overall
   depends_on: []
   commands:
+  - command: manifest.load
+  - *git_get_project
+  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - *set_up_venv
   - func: "shared scons cache pruning"
 
 - name: win_shared_scons_cache_pruning
   exec_timeout_secs: 7200 # 2 hour timeout for the task overall
   depends_on: []
   commands:
+  - command: manifest.load
+  - *git_get_project
+  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
+  - *set_up_venv
   - func: "set up win mount script"
   - func: "shared scons cache pruning"
 
 - name: validate_commit_message
   exec_timeout_secs: 600 # 10 minute timeout
   commands:
+  - command: manifest.load
+  - *git_get_project
   - *f_expansions_write
-  - func: "set task expansion macros"
-  - *f_expansions_write
+  - *kill_processes
+  - *cleanup_environment
   - func: "set up venv"
-  - *f_expansions_write
   - command: subprocess.exec
     type: test
     params:
@@ -6963,11 +7057,13 @@ tasks:
 - name: check_for_todos
   exec_timeout_secs: 600 # 10 minute timeout
   commands:
+    - command: manifest.load
+    - *git_get_project
     - *f_expansions_write
-    - func: "set task expansion macros"
-    - *f_expansions_write
-    - func: "configure evergreen api credentials"
+    - *kill_processes
+    - *cleanup_environment
     - func: "set up venv"
+    - func: "configure evergreen api credentials"
     - command: subprocess.exec
       type: test
       params:
@@ -7108,12 +7204,12 @@ task_groups:
   setup_group:
     - command: manifest.load
     - func: "git get project"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "kill processes"
     - func: "cleanup environment"
     - func: "set up venv"
     - func: "upload pip requirements"
-    - func: "set task expansion macros"
-    - func: "f_expansions_write"
     - func: "configure evergreen api credentials"
     - func: "get buildnumber"
     - func: "f_expansions_write"
@@ -7126,6 +7222,7 @@ task_groups:
   setup_task:
     - func: "apply compile expansions"
     - func: "set task expansion macros"
+    - func: "f_expansions_write"
   teardown_task:
   tasks:
   - clang_tidy
@@ -7136,11 +7233,11 @@ task_groups:
   setup_group:
     - command: manifest.load
     - func: "git get project"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "get buildnumber"
     - func: "set up venv"
     - func: "upload pip requirements"
-    - func: "set task expansion macros"
-    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
@@ -7149,6 +7246,7 @@ task_groups:
   setup_task:
     - func: "set task expansion macros"
     - func: "apply compile expansions"
+    - func: "f_expansions_write"
   teardown_task:
       - func: "attach scons logs"
   tasks:
@@ -7160,11 +7258,11 @@ task_groups:
   setup_group:
     - command: manifest.load
     - func: "git get project"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "get buildnumber"
     - func: "set up venv"
     - func: "upload pip requirements"
-    - func: "set task expansion macros"
-    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
@@ -7173,6 +7271,7 @@ task_groups:
   setup_task:
     - func: "set task expansion macros"
     - func: "apply compile expansions"
+    - func: "f_expansions_write"
   teardown_task:
       - func: "attach scons logs"
   tasks: