SERVER-28000 sys-perf: Organize all dsi yaml config files under configurations/

(cherry picked from commit 39dcb6fa1d13fba5c5242d27a2a30a9175e89a23)

2 files changed, 35 insertions, 18 deletions

diff --git a/etc/longevity.yml b/etc/longevity.yml
index 576ac6851ab..1f95d977455 100644
--- a/etc/longevity.yml
+++ b/etc/longevity.yml
@@ -80,7 +80,7 @@ functions:
           platform: ${platform}
           setup: ${setup}
           storageEngine: ${storageEngine}
-          test: ${test}
+          test: ${test}.longevity
           production: true
           EOF
 
@@ -134,14 +134,16 @@ functions:
         # setup execution environment
         # configure environment
         script: |
+          set -e
           virtualenv ./venv
           source ./venv/bin/activate
           pip install -r ../src/dsi/dsi/requirements.txt
-          python ../src/dsi/dsi/bin/setup_work_env.py --production
+          python ../src/dsi/dsi/bin/bootstrap.py --production
     - command: shell.exec
       params:
         script: |
           set -v
+          set -e
           source work/dsienv.sh
           $DSI_PATH/bin/setup-dsi-env.sh
           cp terraform/* work/
@@ -151,6 +153,7 @@ functions:
         working_dir: work
         script: |
            set -v
+           set -e
            ./terraform get --update
 
   "bring up 3 shard cluster":
@@ -176,7 +179,6 @@ functions:
           set -o verbose
           source ./dsienv.sh
           source ./venv/bin/activate
-          cp mongodb_setup.${setup}.${storageEngine}.yml mongodb_setup.yml
           $DSI_PATH/bin/mongodb_setup.py && echo "${setup} MongoDB Cluster STARTED."
 
   "run test":
@@ -189,8 +191,6 @@ functions:
           echo "Run test for ${test}-${storageEngine} with setup ${setup}"
           source ./dsienv.sh
           source ./venv/bin/activate
-          # Copy over the proper test control file
-          cp $DSI_PATH/test_control/test_control.${test}.longevity.yml test_control.yml
           $DSI_PATH/bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
           echo "Complete test for ${test} with setup ${setup}!"
     - command: "json.send"
@@ -293,6 +293,9 @@ tasks:
   exec_timeout_secs: 216000
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "wiredTiger"
+        test: "ycsb"
     - func: "bring up 3 shard cluster"
     - func: "configure mongodb cluster"
     - func: "run test"
@@ -311,6 +314,9 @@ tasks:
   exec_timeout_secs: 216000
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "mmapv1"
+        test: "ycsb"
     - func: "bring up 3 shard cluster"
     - func: "configure mongodb cluster"
     - func: "run test"
@@ -351,7 +357,7 @@ buildvariants:
     - YCSB
   expansions:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
-    setup: shard
+    setup: shard-longevity
     cluster: longevity
     numShard: 3
     storageEngine: wiredTiger
@@ -370,7 +376,7 @@ buildvariants:
   modules: *modules
   expansions:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
-    setup: shard
+    setup: shard-longevity
     cluster: longevity
     numShard: 3
     storageEngine: mmapv1
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index 85ed6d21281..da1c8082008 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -141,16 +141,18 @@ functions:
         # setup execution environment
         # configure environment, has private information, no logging
         script: |
+          set -e
           virtualenv ./venv
           source ./venv/bin/activate
           pip install -r ../src/dsi/dsi/requirements.txt
-          python ../src/dsi/dsi/bin/setup_work_env.py --production
+          python ../src/dsi/dsi/bin/bootstrap.py --production
           ls
           pwd
     - command: shell.exec
       params:
         script: |
           set -v
+          set -e
           source work/dsienv.sh
           $DSI_PATH/bin/setup-dsi-env.sh
           cp terraform/* work/
@@ -160,6 +162,7 @@ functions:
         working_dir: work
         script: |
           set -v
+          set -e
           ./terraform get --update
     - command: shell.exec
       # set up /data/infrastructure_provisioning to keep track of resources,
@@ -253,7 +256,6 @@ functions:
           set -o verbose
           source ./dsienv.sh
           source ./venv/bin/activate
-          cp mongodb_setup.${setup}.${storageEngine}.yml mongodb_setup.yml
           $DSI_PATH/bin/mongodb_setup.py && echo "${cluster} MongoDB Cluster STARTED."
 
   "run test":
@@ -266,15 +268,6 @@ functions:
           source ./dsienv.sh
           source ./venv/bin/activate
           echo "Run test for ${test}-${storageEngine} with setup ${setup}"
-          # Copy over the proper test control file
-          # This is to work around the fact that multinode clusters need a different ycsb test control.
-          # This should eventually be fixed in a better fashion and removed.
-          if [ ${test} == 'ycsb' ] && ( [ ${cluster} == 'shard' ] || [ ${cluster} == 'replica' ] )
-          then
-            cp $DSI_PATH/test_control/test_control.ycsb.multi_node.yml test_control.yml
-          else
-            cp $DSI_PATH/test_control/test_control.${test}.yml test_control.yml
-          fi
           # Specify the test list to use. Change the testList parameter
           # if you want to run a non-default list.
           $DSI_PATH/bin/update_test_list.py ${testList}
@@ -415,6 +408,9 @@ tasks:
       variant: linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "wiredTiger"
+        test: "ycsb"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars:
@@ -435,6 +431,9 @@ tasks:
       variant: linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "mmapv1"
+        test: "ycsb"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars:
@@ -455,6 +454,9 @@ tasks:
       variant: linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "wiredTiger"
+        test: "benchRun"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars:
@@ -473,6 +475,9 @@ tasks:
       variant: linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "mmapv1"
+        test: "benchRun"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars:
@@ -555,6 +560,9 @@ tasks:
       variant: linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "wiredTiger"
+        test: "initialSync"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars:
@@ -573,6 +581,9 @@ tasks:
       variant : linux-standalone
   commands:
     - func: "prepare environment"
+      vars:
+        storageEngine: "mmapv1"
+        test: "initialSync"
     - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
       vars: