diff options
author | Alexander Costas <alexander.costas@mongodb.com> | 2019-11-13 16:48:50 +0000 |
---|---|---|
committer | evergreen <evergreen@mongodb.com> | 2019-11-13 16:48:50 +0000 |
commit | ffe79da4681589eb10dfe2d5ed922f7046b81e07 (patch) | |
tree | 6492e74a25ac8da713d70b8945021457cea75eef | |
parent | eae761f417f689aaefd276ba195527ff1df64e3e (diff) | |
download | mongo-ffe79da4681589eb10dfe2d5ed922f7046b81e07.tar.gz |
BACKPORT-5442: [v4.0] Use signal processing without DSI
-rwxr-xr-x | buildscripts/signal_processing_setup.sh | 33 | ||||
-rw-r--r-- | etc/perf.yml | 19 | ||||
-rw-r--r-- | etc/system_perf.yml | 33 |
3 files changed, 62 insertions, 23 deletions
diff --git a/buildscripts/signal_processing_setup.sh b/buildscripts/signal_processing_setup.sh new file mode 100755 index 00000000000..d0d023c94c5 --- /dev/null +++ b/buildscripts/signal_processing_setup.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +if [[ -d "signal_processing_venv" ]]; then + source ./signal_processing_venv/bin/activate + exit 0 +fi + +# Configure signal processing +cat > ./analysis.yml << EOF +mongo_uri: mongodb+srv://\${analysis_user}:\${analysis_password}@performancedata-g6tsc.mongodb.net/perf +is_patch: ${is_patch} +task_id: ${task_id} +EOF +virtualenv -p /opt/mongodbtoolchain/v3/bin/python3 signal_processing_venv + +# Setup pip to use our internal PyPI +cat > ./signal_processing_venv/pip.conf << EOF +[global] +index-url = https://pypi.org/simple +extra-index-url = https://${perf_jira_user}:${perf_jira_pw}@artifactory.corp.mongodb.com/artifactory/api/pypi/mongodb-dag-local/simple +EOF + +# Create the Evergreen API credentials +cat > .evergreen.yml <<END_OF_CREDS +api_server_host: https://evergreen.mongodb.com/api +api_key: "${evergreen_api_key}" +user: "${evergreen_api_user}" +END_OF_CREDS + +source ./signal_processing_venv/bin/activate +pip install dag-signal-processing~=1.0 + +export analysis_user="${dsi_analysis_atlas_user}" +export analysis_password="${dsi_analysis_atlas_pw}"
\ No newline at end of file diff --git a/etc/perf.yml b/etc/perf.yml index 36a51e72e78..3355c626dde 100644 --- a/etc/perf.yml +++ b/etc/perf.yml @@ -162,14 +162,16 @@ functions: working_dir: src silent: true script: | - set -o errexit - - # Create the Evergreen API credentials - cat > .evergreen.yml <<END_OF_CREDS - api_server_host: https://evergreen.mongodb.com/api - api_key: "${evergreen_api_key}" - user: "${evergreen_api_user}" - END_OF_CREDS + is_patch=${is_patch} + task_id=${task_id} + perf_jira_user=${perf_jira_user} + perf_jira_pw=${perf_jira_pw} + dsi_analysis_atlas_user=${dsi_analysis_atlas_user} + dsi_analysis_atlas_pw=${dsi_analysis_atlas_pw} + evergreen_api_key=${evergreen_api_key} + evergreen_api_user=${evergreen_api_user} + source ./buildscripts/signal_processing_setup.sh + detect-changes --evergreen-api-config .evergreen.yml - command: shell.exec params: working_dir: src @@ -194,6 +196,7 @@ functions: script: | set -o errexit set -o verbose + ./dsi/run-dsi setup source ./dsi/dsi_venv/bin/activate # Any tasks that want the analysis scripts to analyze mongod.log files should pass in diff --git a/etc/system_perf.yml b/etc/system_perf.yml index e25ba697449..df1d10b7b5b 100644 --- a/etc/system_perf.yml +++ b/etc/system_perf.yml @@ -304,22 +304,17 @@ functions: - command: shell.exec params: working_dir: work - silent: true - script: | - set -o errexit - - # Create the Evergreen API credentials - cat > .evergreen.yml <<END_OF_CREDS - api_server_host: https://evergreen.mongodb.com/api - api_key: "${evergreen_api_key}" - user: "${evergreen_api_user}" - END_OF_CREDS - - - command: shell.exec - params: - working_dir: work script: | - ../src/dsi/dsi/run-dsi detect-changes --evergreen-api-config .evergreen.yml + is_patch=${is_patch} + task_id=${task_id} + perf_jira_user=${perf_jira_user} + perf_jira_pw=${perf_jira_pw} + dsi_analysis_atlas_user=${dsi_analysis_atlas_user} + dsi_analysis_atlas_pw=${dsi_analysis_atlas_pw} + evergreen_api_key=${evergreen_api_key} + evergreen_api_user=${evergreen_api_user} + source ../src/buildscripts/signal_processing_setup.sh + detect-changes --evergreen-api-config .evergreen.yml - command: shell.exec type : test params: @@ -330,6 +325,14 @@ functions: TAG="4.0.2-Baseline" OVERRIDEFILE="../src/dsi/dsi/analysis/${branch_name}/system_perf_override.json" python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file perf.json --rev ${revision} --refTag $TAG --overrideFile $OVERRIDEFILE --project_id sys-perf --variant ${build_variant} --task ${task_name} + # detect outliers needs to run, so defer the post_run_check exit status to later + echo $? > post_run_check.status + - command: shell.exec + params: + working_dir: work + script: | + source ../src/buildscripts/signal_processing_setup.sh + detect-outliers --evergreen-api-config .evergreen.yml "upload pip requirements": command: s3.put |