BACKPORT-5444: [v3.4] Use signal processing without DSI

3 files changed, 62 insertions, 29 deletions

diff --git a/buildscripts/signal_processing_setup.sh b/buildscripts/signal_processing_setup.sh
new file mode 100755
index 00000000000..d0d023c94c5
--- /dev/null
+++ b/buildscripts/signal_processing_setup.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+if [[ -d "signal_processing_venv" ]]; then
+    source ./signal_processing_venv/bin/activate
+    exit 0
+fi
+
+# Configure signal processing
+cat > ./analysis.yml << EOF
+mongo_uri: mongodb+srv://\${analysis_user}:\${analysis_password}@performancedata-g6tsc.mongodb.net/perf
+is_patch: ${is_patch}
+task_id: ${task_id}
+EOF
+virtualenv -p /opt/mongodbtoolchain/v3/bin/python3 signal_processing_venv
+
+# Setup pip to use our internal PyPI
+cat > ./signal_processing_venv/pip.conf << EOF
+[global]
+index-url = https://pypi.org/simple
+extra-index-url = https://${perf_jira_user}:${perf_jira_pw}@artifactory.corp.mongodb.com/artifactory/api/pypi/mongodb-dag-local/simple
+EOF
+
+# Create the Evergreen API credentials
+cat > .evergreen.yml <<END_OF_CREDS
+api_server_host: https://evergreen.mongodb.com/api
+api_key: "${evergreen_api_key}"
+user: "${evergreen_api_user}"
+END_OF_CREDS
+
+source ./signal_processing_venv/bin/activate
+pip install dag-signal-processing~=1.0
+
+export analysis_user="${dsi_analysis_atlas_user}"
+export analysis_password="${dsi_analysis_atlas_pw}"
\ No newline at end of file
diff --git a/etc/perf.yml b/etc/perf.yml
index 51b7eddb537..c45b0623717 100644
--- a/etc/perf.yml
+++ b/etc/perf.yml
@@ -162,20 +162,16 @@ functions:
         working_dir: src
         silent: true
         script: |
-          set -o errexit
-
-          # Create the Evergreen API credentials
-          cat > .evergreen.yml <<END_OF_CREDS
-          api_server_host: https://evergreen.mongodb.com/api
-          api_key: "${evergreen_api_key}"
-          user: "${evergreen_api_user}"
-          END_OF_CREDS
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
-          set -v
-          ./dsi/run-dsi detect-changes --evergreen-api-config .evergreen.yml
+          is_patch=${is_patch}
+          task_id=${task_id}
+          perf_jira_user=${perf_jira_user}
+          perf_jira_pw=${perf_jira_pw}
+          dsi_analysis_atlas_user=${dsi_analysis_atlas_user}
+          dsi_analysis_atlas_pw=${dsi_analysis_atlas_pw}
+          evergreen_api_key=${evergreen_api_key}
+          evergreen_api_user=${evergreen_api_user}
+          source ./buildscripts/signal_processing_setup.sh
+          detect-changes --evergreen-api-config .evergreen.yml
     - command: json.get_history
       params:
         task: ${task_name}
@@ -194,6 +190,7 @@ functions:
         script: |
           set -o errexit
           set -o verbose
+          ./dsi/run-dsi setup
           source ./dsi/dsi_venv/bin/activate
 
           # Any tasks that want the analysis scripts to analyze mongod.log files should pass in
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index cd501c12a3a..6980a4de467 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -265,22 +265,17 @@ functions:
     - command: shell.exec
       params:
         working_dir: work
-        silent: true
-        script: |
-          set -o errexit
-
-          # Create the Evergreen API credentials
-          cat > .evergreen.yml <<END_OF_CREDS
-          api_server_host: https://evergreen.mongodb.com/api
-          api_key: "${evergreen_api_key}"
-          user: "${evergreen_api_user}"
-          END_OF_CREDS
-
-    - command: shell.exec
-      params:
-        working_dir: work
         script: |
-          ../src/dsi/dsi/run-dsi detect-changes --evergreen-api-config .evergreen.yml
+          is_patch=${is_patch}
+          task_id=${task_id}
+          perf_jira_user=${perf_jira_user}
+          perf_jira_pw=${perf_jira_pw}
+          dsi_analysis_atlas_user=${dsi_analysis_atlas_user}
+          dsi_analysis_atlas_pw=${dsi_analysis_atlas_pw}
+          evergreen_api_key=${evergreen_api_key}
+          evergreen_api_user=${evergreen_api_user}
+          source ../src/buildscripts/signal_processing_setup.sh
+          detect-changes --evergreen-api-config .evergreen.yml
     - command: shell.exec
       type : test
       params:
@@ -291,6 +286,14 @@ functions:
           TAG="3.4.17-Baseline"
           OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json"
           python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name}
+          # detect outliers needs to run, so defer the post_run_check exit status to later
+          echo $? > post_run_check.status
+    - command: shell.exec
+      params:
+        working_dir: work
+        script: |
+          source ../src/buildscripts/signal_processing_setup.sh
+          detect-outliers --evergreen-api-config .evergreen.yml
 
 #######################################
 #               Tasks                 #