SERVER-30770: system_perf.yml: Use new DSI interface and cleanup

2 files changed, 52 insertions, 171 deletions

diff --git a/etc/longevity.yml b/etc/longevity.yml
index 10b5081affc..46c2fd90f33 100644
--- a/etc/longevity.yml
+++ b/etc/longevity.yml
@@ -9,19 +9,18 @@ post:
       params:
         working_dir: work
         script: |
-          set -v
           source ./dsienv.sh
-          $DSI_PATH/bin/make_artifact.sh
+          make_artifact.sh
     - command: s3.put
       params:
         aws_key: ${aws_key}
         aws_secret: ${aws_secret}
         local_file: work/reports.tgz
-        remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}.${ext|tgz}
+        remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/${task_name}-${build_id}-${execution}.${ext|tgz}
         bucket: mciuploads
         permissions: public-read
         content_type: ${content_type|application/x-gzip}
-        display_name: test-log
+        display_name: test-log - Execution ${execution}
     - command: attach.results
       params:
         file_location: work/report.json
@@ -64,6 +63,9 @@ functions:
           storageEngine: ${storageEngine}
           test: ${test}.longevity
           production: true
+
+          # compositions of expansions
+          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
           EOF
 
           cat > runtime.yml <<EOF
@@ -85,11 +87,6 @@ functions:
           ext: ${ext}
           script_flags : ${script_flags}
           dsi_rev: ${dsi_rev}
-          compare_task: ${compare_task}
-          workloads_rev: ${workloads_rev}
-
-          # compositions of expansions
-          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
           EOF
     - command: shell.exec
       params:
@@ -112,8 +109,6 @@ functions:
     - command: shell.exec
       params:
         working_dir: work
-        # setup execution environment
-        # configure environment
         script: |
           set -e
           virtualenv ./venv
@@ -126,12 +121,11 @@ functions:
           set -v
           set -e
           source work/dsienv.sh
-          $DSI_PATH/bin/setup-dsi-env.sh
+          setup-dsi-env.sh
           ls -a work
 
-  "infrastructure provisioning":
+  "deploy cluster":
     - command: shell.exec
-      # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances.
       params:
         working_dir: work
         script: |
@@ -139,22 +133,12 @@ functions:
           set -v
           source ./dsienv.sh
           source ./venv/bin/activate
-          $DSI_PATH/bin/infrastructure_provisioning.py
-
-  "configure mongodb cluster":
-    - command: shell.exec
-      # bring up the mongod
-      params:
-        working_dir: work
-        script: |
-          set -e
-          set -o verbose
-          source ./dsienv.sh
-          source ./venv/bin/activate
-          $DSI_PATH/bin/mongodb_setup.py && echo "${setup} MongoDB Cluster STARTED."
+          infrastructure_provisioning.py
+          mongodb_setup.py
 
   "run test":
     - command: shell.exec
+      type: test
       params:
         working_dir: work
         script: |
@@ -162,9 +146,7 @@ functions:
           set -v
           source ./dsienv.sh
           source ./venv/bin/activate
-          echo "Run test for ${test}-${storageEngine} with setup ${setup}"
-          $DSI_PATH/bin/run_test.py ${storageEngine} ${test} ${cluster}
-          echo "Complete test for ${test} with setup ${setup}!"
+          test_control.py
     - command: "json.send"
       params:
           name: "perf"
@@ -178,13 +160,12 @@ functions:
         script: |
           set -e
           set -o verbose
-          source ./dsienv.sh
           # Longevity runs so rarely, we simply teardown the cluster when done.
           # Note that nowadays infrastructure_teardown.py is actually copying the terraform.tfstate into /data/infrastructure_provisioning
           # but as of this writing the rhel70-perf-longevity distro didn't actually use the teardown hook.
           source ./dsienv.sh
           source ./venv/bin/activate
-          $DSI_PATH/bin/infrastructure_teardown.py
+          infrastructure_teardown.py
           echo "Cluster DESTROYED."
           echo
           echo "All perf results"
@@ -196,9 +177,8 @@ functions:
       params:
         working_dir: work
         script: |
-          set -v
           source ./dsienv.sh
-          $DSI_PATH/bin/make_artifact.sh
+          make_artifact.sh
 
   "analyze":
     - command: json.get_history
@@ -272,12 +252,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "ycsb"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "ycsb"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -293,12 +269,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "ycsb"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "ycsb"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -318,6 +290,11 @@ modules:
   branch: master
 
 
+
+#######################################
+#         Linux Buildvariants         #
+#######################################
+
 buildvariants:
 - name: linux-wt-shard
   display_name: Linux WT Shard
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index ac29cf34480..a4ad8bec798 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -10,7 +10,7 @@ post:
         working_dir: work
         script: |
           source ./dsienv.sh
-          $DSI_PATH/bin/make_artifact.sh
+          make_artifact.sh
     - command: s3.put
       params:
         aws_key: ${aws_key}
@@ -34,13 +34,17 @@ post:
     - command: attach.results
       params:
         file_location: work/report.json
+    - command: "json.send"
+      params:
+         name: "perf"
+         file: "work/perf.json"
     - command: shell.exec
       params:
         working_dir: work
         script: |
           source ./dsienv.sh
           if [ -e /data/infrastructure_provisioning/terraform/provisioned.${cluster} ]; then
-            $DSI_PATH/bin/mark_idle.sh
+            mark_idle.sh
           fi
 
     - command: shell.exec
@@ -82,6 +86,7 @@ functions:
           storageEngine: ${storageEngine}
           test: ${test}
           production: true
+          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
           EOF
 
           cat > runtime.yml <<EOF
@@ -104,9 +109,6 @@ functions:
           script_flags : ${script_flags}
           dsi_rev: ${dsi_rev}
           workloads_rev: ${workloads_rev}
-
-          # compositions of expansions
-          mongodb_binary_archive: "https://s3.amazonaws.com/mciuploads/${project}/${version_id}/${revision}/${platform}/mongod-${version_id}.tar.gz"
           EOF
     - command: shell.exec
       params:
@@ -136,21 +138,18 @@ functions:
           virtualenv ./venv
           source ./venv/bin/activate
           pip install -r ../src/dsi/dsi/requirements.txt
-          python ../src/dsi/dsi/bin/bootstrap.py --production
-          ls
-          pwd
+          python ../src/dsi/dsi/bin/bootstrap.py
     - command: shell.exec
       params:
         script: |
           set -v
           set -e
           source work/dsienv.sh
-          $DSI_PATH/bin/setup-dsi-env.sh
+          setup-dsi-env.sh
           ls -a work
 
-  "infrastructure provisioning":
+  "deploy cluster":
     - command: shell.exec
-      # call infrastructure-provisioning.py. This will either create a cluster, or update tags on existing instances.
       params:
         working_dir: work
         script: |
@@ -158,22 +157,12 @@ functions:
           set -v
           source ./dsienv.sh
           source ./venv/bin/activate
-          $DSI_PATH/bin/infrastructure_provisioning.py
-
-  "configure mongodb cluster":
-    - command: shell.exec
-      # bring up the mongod
-      params:
-        working_dir: work
-        script: |
-          set -e
-          set -o verbose
-          source ./dsienv.sh
-          source ./venv/bin/activate
-          $DSI_PATH/bin/mongodb_setup.py && echo "${setup} MongoDB Cluster STARTED."
+          infrastructure_provisioning.py
+          mongodb_setup.py
 
   "run test":
     - command: shell.exec
+      type: test
       params:
         working_dir: work
         script: |
@@ -181,8 +170,7 @@ functions:
           set -v
           source ./dsienv.sh
           source ./venv/bin/activate
-          echo "Run test for ${test}-${storageEngine} with setup ${setup}"
-          $DSI_PATH/bin/run_test.py ${storageEngine} ${test} ${cluster}
+          test_control.py
     - command: "json.send"
       params:
          name: "perf"
@@ -194,7 +182,7 @@ functions:
         working_dir: work
         script: |
           source ./dsienv.sh
-          $DSI_PATH/bin/make_artifact.sh
+          make_artifact.sh
 
   "analyze":
     - command: json.get_history
@@ -281,14 +269,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "ycsb"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "ycsb"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -303,14 +285,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "ycsb"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "ycsb"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -325,14 +301,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "ycsb-wmajority"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "ycsb-wmajority"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -347,14 +317,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "ycsb-wmajority"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "ycsb-wmajority"
     - func: "make test log artifact"
     - func: "analyze"
       vars:
@@ -369,14 +333,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "core"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "core"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -389,14 +347,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "core"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "core"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -409,14 +361,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "non_sharded"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "non_sharded"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -429,14 +375,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "non_sharded"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "non_sharded"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -449,14 +389,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "mongos"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "mongos"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -470,14 +404,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "mongos"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "mongos"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -490,14 +418,8 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "move_chunk"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "move_chunk"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -510,14 +432,8 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "move_chunk"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "move_chunk"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -529,15 +445,9 @@ tasks:
     - func: "prepare environment"
       vars:
         storageEngine: "wiredTiger"
-        test: "initialSync"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "wiredTiger"
+        test: "initialsync"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "initialSync"
     - func: "make test log artifact"
     - func: "analyze"
 
@@ -549,15 +459,9 @@ tasks:
     - func: "prepare environment"
       vars:
         storageEngine: "mmapv1"
-        test: "initialSync"
-    - func: "infrastructure provisioning"
-    - func: "configure mongodb cluster"
-      vars:
-        storageEngine: "mmapv1"
+        test: "initialsync"
+    - func: "deploy cluster"
     - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "initialSync"
     - func: "make test log artifact"
     - func: "analyze"