SERVER-42075: Add DSI module to perf.yml

1 files changed, 39 insertions, 48 deletions

diff --git a/etc/perf.yml b/etc/perf.yml
index 4603f33f2bc..51d15877cf3 100644
--- a/etc/perf.yml
+++ b/etc/perf.yml
@@ -47,28 +47,23 @@ modules:
   repo: git@github.com:10gen/mongo-enterprise-modules.git
   prefix: src/mongo/db/modules
   branch: v4.0
-
+- name: dsi
+  repo: git@github.com:10gen/dsi.git
+  branch: master
+- name: mongo-perf
+  repo: git@github.com:mongodb/mongo-perf.git
+  branch: master
 
 functions:
-  "download analysis scripts":
-    - command: shell.exec
-      params:
-        script: |
-          set -v
-          rm -rf ./dsi
-          mkdir -p ./src
-          git clone git@github.com:10gen/dsi.git
-    # get the mongo source, note the s3.get calls put the
-    # exe files in their respective make locations.
-    - command: git.get_project
-      params:
-        directory: src
+  "git get project": &git_get_project
+    command: git.get_project
+    params:
+      directory: src
+      revisions:  # for each module include revision as <module_name> : ${<module_name>_rev}
+        enterprise: ${enterprise_rev}
+        dsi: ${dsi_rev}
+        mongo-perf: ${mongo-perf_rev}
   "start server":
-    - command: shell.exec
-      params:
-        script: |
-          rm -rf ./*
-          mkdir src
     - command: s3.get
       params:
         aws_key: ${aws_key}
@@ -91,9 +86,6 @@ functions:
           set -v
           chmod +x mongod
           chmod +x mongo
-          git clone git@github.com:mongodb/mongo-perf.git perf
-          cd perf
-          git describe --abbrev=7 --tags
     - command: shell.exec
       params:
         background: true
@@ -138,13 +130,6 @@ functions:
       params:
         working_dir: src
         script: |
-          set -e
-          set -v
-          ../dsi/run-dsi setup
-    - command: shell.exec
-      params:
-        working_dir: src
-        script: |
           cat > bootstrap.yml <<EOF
           infrastructure_provisioning: ${cluster}
           platform: ${platform}
@@ -160,7 +145,7 @@ functions:
           task_id: ${task_id}
           EOF
 
-          cp ../dsi/configurations/analysis/analysis.common.yml  analysis.yml
+          cp ./dsi/configurations/analysis/analysis.common.yml  analysis.yml
     - command: shell.exec
       params:
         working_dir: src
@@ -176,7 +161,7 @@ functions:
         working_dir: src
         script: |
           set -v
-          ../dsi/run-dsi detect-changes
+          ./dsi/run-dsi detect-changes
     - command: json.get_history
       params:
         task: ${task_name}
@@ -195,7 +180,7 @@ functions:
         script: |
           set -o errexit
           set -o verbose
-          source ./venv/bin/activate
+          source ./dsi/dsi_venv/bin/activate
 
           # Any tasks that want the analysis scripts to analyze mongod.log files should pass in
           # `reports_analysis: true` as a var to this function. The following line will select the
@@ -203,8 +188,8 @@ functions:
           reports_analysis_flags="--reports-analysis . --perf-file perf/perf.json"
           cmd_flags=$([ "${reports_analysis}" = "true" ] && echo "$reports_analysis_flags" || echo "")
           REFTAG="3.6.5-Baseline"
-          OVERRIDE="../dsi/analysis/${branch_name}/perf_override.json"
-          python ../dsi/analysis/perf_regression_check.py $cmd_flags -f history.json --rev ${revision} -t tags.json --refTag $REFTAG --overrideFile $OVERRIDE --variant ${build_variant} --task ${task_name} --threshold 0.075 --threadThreshold 0.12
+          OVERRIDE="./dsi/analysis/${branch_name}/perf_override.json"
+          python ./dsi/analysis/perf_regression_check.py $cmd_flags -f history.json --rev ${revision} -t tags.json --refTag $REFTAG --overrideFile $OVERRIDE --variant ${build_variant} --task ${task_name} --threshold 0.075 --threadThreshold 0.12
   "run perf tests":
     - command: shell.exec
       params:
@@ -223,10 +208,11 @@ functions:
           set -e
           set -v
           source ./venv/bin/activate
-          cd perf
+          cd mongo-perf
+          mkdir -p ../perf
           #  give mongod a few seconds to start up so that we can connect.
           sleep 5
-          ${perf_exec_wrapper} python benchrun.py --shell ../mongo -t ${threads} --trialCount 5 -f testcases/*.js --readCmd ${readCmd} --includeFilter ${includeFilter1}  --includeFilter ${includeFilter2} --excludeFilter ${excludeFilter} --out perf.json --exclude-testbed --username admin --password password
+          ${perf_exec_wrapper} python benchrun.py --shell ../mongo -t ${threads} --trialCount 5 -f testcases/*.js --readCmd ${readCmd} --includeFilter ${includeFilter1}  --includeFilter ${includeFilter2} --excludeFilter ${excludeFilter} --out ../perf/perf.json --exclude-testbed --username admin --password password
           echo "Oplog size at end of tests..."
           ../mongo --username admin --password password --eval "db.getSiblingDB('local').oplog.rs.totalSize()/1024/1024" admin
     - command: "json.send"
@@ -296,6 +282,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -304,7 +291,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -313,6 +299,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -321,7 +308,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: true
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -330,6 +316,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -338,7 +325,6 @@ tasks:
         excludeFilter: "none"
         threads : "1"
         readCmd: true
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         report_analysis: true
@@ -347,6 +333,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -355,7 +342,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -364,6 +350,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -372,7 +359,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -381,6 +367,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -389,7 +376,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -398,6 +384,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -406,7 +393,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -415,6 +401,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -423,7 +410,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -432,6 +418,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -440,7 +427,6 @@ tasks:
         excludeFilter : "none"
         threads : "1"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -449,6 +435,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -457,7 +444,6 @@ tasks:
         excludeFilter: "none"
         threads : "1"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         report_analysis: true
@@ -466,6 +452,7 @@ tasks:
   - variant: linux-wt-standalone
     name: compile
   commands:
+    - func: "git get project"
     - func: "start server"
     - func: "run perf tests"
       vars:
@@ -474,7 +461,6 @@ tasks:
         excludeFilter : "single_threaded"
         threads : "1 2 4 8"
         readCmd: false
-    - func: "download analysis scripts"
     - func: "analyze"
       vars:
         reports_analysis: true
@@ -485,6 +471,8 @@ buildvariants:
   batchtime: 90 # 1.5 hours
   modules:
     - enterprise
+    - dsi
+    - mongo-perf
   expansions:
     # We are explicitly tracking the rhel62 variant compile options from evergreen.yml for
     # microbenchmarks, since they run on the centos6 boxes.  If we can get proper artifacts directly
@@ -559,7 +547,10 @@ buildvariants:
 
 - name: linux-wt-repl
   display_name: 1-Node ReplSet Linux inMemory
-  batchtime: 90 # 1.5 hours
+  batchtime: 90  # 1.5 hours
+  modules:
+    - dsi
+    - mongo-perf
   expansions:
     mongod_exec_wrapper: *exec_wrapper
     perf_exec_wrapper: *perf_wrapper