SERVER-23642 system_perf.yml refactoring

 - Collapse dsi/bin/run-*.sh scripts so
   that there's one each for ycsb, benchRun
   initialSync.
 - Push some "plumbing" bash code from .yml
   files to dsi files.
 - Remove
   dsi/bin/run-distributed_correctness_suite_2.sh
   It was 100% duplicate of suite_1.

(cherry picks commit 3bead6a7dee0c224242463d75acda511e8e8bb09)

3 files changed, 11 insertions, 123 deletions

diff --git a/etc/distributed_correctness.yml b/etc/distributed_correctness.yml
index f6ea7017f57..af24530c4dc 100644
--- a/etc/distributed_correctness.yml
+++ b/etc/distributed_correctness.yml
@@ -116,19 +116,8 @@ functions:
           set -e
           set -v
           cd ./clusters/${cluster}
-          cat ips.sh
-          ./update_run_config.sh
-          rm -rf ./reports
-          rm -f ../../reports.tgz
-          echo "Run test for ${test} with setup ${setup}"
-          # run test
+          echo "Run test for ${test}-${storageEngine} with setup ${setup}"
           ../../bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
-          rm -f ../perf.json
-          chmod 766 perf.json
-          cp ./perf.json ..
-          cd ..
-          pwd
-          cat perf.json
           echo "Done test for ${test} with setup ${setup}!"
     - command: "json.send"
       params:
@@ -283,7 +272,7 @@ tasks:
 
 - name: distributed_correctness_suite_1_MMAPv1
   depends_on:
-    - name: distributed_correctness_suite_2_wiredTiger
+    - name: distributed_correctness_suite_1_wiredTiger
       status : "*"
   commands:
     - func: "prepare environment"
@@ -301,44 +290,6 @@ tasks:
     - func: "upload log file"
       vars:
         test: "distributed_correctness_suite_1_MMAPv1"
-    - func: "analyze"
-
-- name: distributed_correctness_suite_2_wiredTiger
-  depends_on:
-    - name: distributed_correctness_suite_1_wiredTiger
-      status : "*"
-  commands:
-    - func: "prepare environment"
-    - func: "restore cluster"
-
-    - func: "run test"
-      vars:
-        storageEngine: "wiredTiger"
-        test: "distributed_correctness_suite_2"
-
-    - func: "make test log artifact"
-    - func: "upload log file"
-      vars:
-        test: "distributed_correctness_suite_2_wiredTiger"
-    - func: "analyze"
-
-- name: distributed_correctness_suite_2_MMAPv1
-  depends_on:
-    - name: distributed_correctness_suite_1_MMAPv1
-      status : "*"
-  commands:
-    - func: "prepare environment"
-    - func: "restore cluster"
-
-    - func: "run test"
-      vars:
-        storageEngine: "mmapv1"
-        test: "distributed_correctness_suite_2"
-
-    - func: "make test log artifact"
-    - func: "upload log file"
-      vars:
-        test: "distributed_correctness_suite_2_MMAPv1"
     - func: "destroy cluster"
     - func: "analyze"
 
@@ -363,9 +314,7 @@ buildvariants:
       distros:
       - rhel55
     - name: distributed_correctness_suite_1_wiredTiger
-    - name: distributed_correctness_suite_2_wiredTiger
     - name: distributed_correctness_suite_1_MMAPv1
-    - name: distributed_correctness_suite_2_MMAPv1
 
 - name: linux-3-node-replSet
   display_name: Linux 3-Node ReplSet
@@ -379,6 +328,4 @@ buildvariants:
       - "rhel70-perf-replset"
   tasks:
     - name: distributed_correctness_suite_1_wiredTiger
-    - name: distributed_correctness_suite_2_wiredTiger
     - name: distributed_correctness_suite_1_MMAPv1
-    - name: distributed_correctness_suite_2_MMAPv1
diff --git a/etc/longevity.yml b/etc/longevity.yml
index aa868bd88bb..7f84d4d1ac7 100644
--- a/etc/longevity.yml
+++ b/etc/longevity.yml
@@ -87,18 +87,8 @@ functions:
           set -e
           set -v
           cd ./clusters/${cluster}
-          # show cluster details
-          cat ips.sh
-          ./update_run_config.sh
-          cat run-${test}.json
-          # clean up old artifact from the last test
-          rm -f ../perf.json
-          rm -rf ./reports
-          rm -f ../../reports.tgz
-          echo "Run test for ${test} with setup ${setup}"
-          # run test
+          echo "Run test for ${test}-${storageEngine} with setup ${setup}"
           ../../bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
-          mv perf.json ..
           cd ..
           cat perf.json >> perf_all.json
           echo "Complete test for ${test} with setup ${setup}!"
@@ -245,6 +235,7 @@ tasks:
         use_csrs: "false"
     - func: "run test"
       vars:
+        storageEngine: "wiredTiger"
         test: "ycsb"
     - func: "make test log artifact"
     - func: "upload log file"
@@ -253,26 +244,6 @@ tasks:
     - func: "destroy cluster"
     - func: "analyze"
 
-- name: shard_cluster_CSRS_test
-  depends_on:
-    - name: compile
-  exec_timeout_secs: 28800
-  commands:
-    - func: "prepare environment"
-    - func: "bring up 3 shard cluster"
-    - func: "configure mongodb cluster"
-      vars:
-        use_csrs: "true"
-    - func: "run test"
-      vars:
-        test: "ycsb-csrs"
-    - func: "make test log artifact"
-    - func: "upload log file"
-      vars:
-        test: "ycsb"
-    - func: "destroy cluster"
-    - func: "analyze"
-
 - name: shard_cluster_MMAPv1_test
   depends_on:
     - name: compile
@@ -285,7 +256,9 @@ tasks:
         use_csrs: "false"
     - func: "run test"
       vars:
-        test: "ycsb-mmap"
+        test: "ycsb"
+        storageEngine: "mmapv1"
+        test: "ycsb"
     - func: "make test log artifact"
     - func: "upload log file"
       vars:
@@ -312,25 +285,6 @@ buildvariants:
       - rhel55
     - name: shard_cluster_test
 
-- name: linux-wt-shard-csrs
-  display_name: Linux WT Shard CSRS
-  batchtime: 20160 # 2 weeks
-  expansions:
-    compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
-    setup: shard
-    cluster: longevity
-    numShard: 3
-    storageEngine: wiredTiger
-
-  run_on:
-    - "linux-64-amzn-perf-longevity"
-  tasks:
-    - name: compile
-      distros:
-      - rhel55
-    - name: shard_cluster_CSRS_test
-
-
 - name: linux-mmapv1-shard
   display_name: Linux MMAPv1 Shard
   batchtime: 40320 # 4 week
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index 99cb2ccfbc1..7c0c978eb20 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -114,22 +114,9 @@ functions:
           set -e
           set -v
           cd ./clusters/${cluster}
-          cat ips.sh
-          cat run-${test}.json
-          ./update_run_config.sh
-          cat run-${test}.json
-          rm -rf ./reports
-          rm -f ../../reports.tgz
-          echo "Run test for ${test} with setup ${setup}"
-          # run test
+          echo "Run test for ${test}-${storageEngine} with setup ${setup}"
           ../../bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
-          rm -f ../perf.json
-          chmod 766 perf.json
-          cp ./perf.json ..
-          cd ..
-          pwd
-          cat perf.json
-          echo "Done test for ${test} with setup ${setup}!"
+          echo "Done test for ${test}-${storageEngine} with setup ${setup}!"
     - command: "json.send"
       params:
          name: "perf"
@@ -348,7 +335,7 @@ tasks:
     - func: "run test"
       vars:
         storageEngine: "mmapv1"
-        test: "ycsb-mmap"
+        test: "ycsb"
     - func: "make test log artifact"
     - func: "upload log file"
       vars:
@@ -382,7 +369,7 @@ tasks:
     - func: "run test"
       vars:
         storageEngine: "mmapv1"
-        test: "benchRun-mmap"
+        test: "benchRun"
     - func: "make test log artifact"
     - func: "upload log file"
       vars: