SERVER-26813: Enable longevity performance test for 3.2 branch

1 files changed, 113 insertions, 61 deletions

diff --git a/etc/longevity.yml b/etc/longevity.yml
index 4e642ea39ad..7ad3a1516ba 100644
--- a/etc/longevity.yml
+++ b/etc/longevity.yml
@@ -7,16 +7,14 @@ pre:
 post:
     - command: attach.results
       params:
-        file_location: src/report.json
-    - command: shell.cleanup
+        file_location: work/report.json
     - command: shell.exec
       # destroy the cluster
       params:
-        working_dir: dsi
+        working_dir: work
         script: |
           set -e
           set -o verbose
-          cd ./clusters/${cluster}
           # call terraform destroy twice to avoid AWS timeout
           yes yes | ./terraform destroy
           yes yes | ./terraform destroy
@@ -29,83 +27,109 @@ functions:
         script: |
           rm -rf ./*
           mkdir src
+          mkdir work
+          mkdir bin
+          mkdir keys
+          pwd
+          ls
+    - command: manifest.load
+    - command: git.get_project
+      params:
+        directory: src
+        revisions: # for each module include revision as <module_name> : ${<module_name>_rev}
+          dsi: ${dsi_rev}
+          YCSB: ${YCSB_rev}
+    - command: git.apply_patch
+      params:
+        directory: src
+    - command: shell.exec
+      params:
+        silent: true
+        script: |
+          # generate aws private key file
+          echo "${terraform_secret}" > secret
+          chmod 400 secret
+          echo "${ec2_pem}" > keys/aws.pem
+          chmod 400 keys/aws.pem
+    - command: shell.exec
+      params:
+        working_dir: work
+        # setup execution environment
+        # configure environment
+        script: |
+          virtualenv ./venv
+          source ./venv/bin/activate
+          pip install -r ../src/dsi/dsi/requirements.txt
+          python ../src/dsi/dsi/bin/setup_work_env.py --cluster-type ${cluster} --aws-key-name ${terraform_key} --ssh-keyfile-path ../keys/aws.pem --aws-secret-file ../secret --production
     - command: shell.exec
-      # checkout and setup DSI environment
       params:
         script: |
-          set -e
           set -v
-          git clone git@github.com:10gen/dsi.git
-          cd dsi
-          # setup execution environment
-          ./bin/setup-dsi-env.sh
+          source work/dsienv.sh
+          $DSI_PATH/bin/setup-dsi-env.sh
+          cp terraform/* work/
+          ls work
     - command: shell.exec
-      # configure environment, has private information, no logging
       params:
-        working_dir: dsi
-        silent: true
+        working_dir: work
         script: |
-          cd ./clusters/${cluster}
-          # stage aws credential for terraform
-          ../../bin/make_terraform_env.sh ${terraform_key} ${terraform_secret} https://s3.amazonaws.com/mciuploads/longevity-v3.2/${build_variant}/${revision}/mongodb-${build_id}.tar.gz
-          # generate aws private key file
-          echo "${ec2_pem}" > ../../keys/aws.pem
-          chmod 400 ../../keys/aws.pem
+           set -v
+           ./terraform get --update
 
   "bring up 3 shard cluster":
     - command: shell.exec
       # bring up the cluster
       params:
-        working_dir: dsi
+        working_dir: work
         silent: true
         script: |
           # to create a mongod EC2 cluster
           set -e
           set -o verbose
-          cd ./clusters/${cluster}
-          ../../bin/setup-cluster.sh ${cluster} false
-          echo "EC2 Cluster CREATED."
+          source ./dsienv.sh
+          $DSI_PATH/bin/setup-cluster.sh ${cluster} false
 
   "configure mongodb cluster":
     - command: shell.exec
       # bring up the mongod
       params:
-        working_dir: dsi
+        working_dir: work
         script: |
           set -e
           set -o verbose
+          source ./dsienv.sh
           source ./venv/bin/activate
-          cd ./clusters/${cluster}
           cp mongodb_setup.${setup}.${storageEngine}.yml mongodb_setup.yml
-          ../../bin/mongodb_setup.py --config
+          $DSI_PATH/bin/mongodb_setup.py --mongodb-binary-archive https://s3.amazonaws.com/mciuploads/longevity-v3.2/${build_variant}/${revision}/mongodb-${build_id}.tar.gz
 
   "run test":
     - command: shell.exec
       params:
-        working_dir: dsi
+        working_dir: work
         script: |
           set -e
           set -v
-          cd ./clusters/${cluster}
           echo "Run test for ${test}-${storageEngine} with setup ${setup}"
-          ../../bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
-          cd ..
-          cat perf.json >> perf_all.json
+          source ./dsienv.sh
+          source ./venv/bin/activate
+          # Copy over the proper test control file
+          cp $DSI_PATH/test_control/test_control.${test}.longevity.yml test_control.yml
+          $DSI_PATH/bin/run-${test}.sh ${storageEngine} ${setup} ${cluster}
           echo "Complete test for ${test} with setup ${setup}!"
     - command: "json.send"
       params:
           name: "perf"
-          file: "dsi/clusters/perf.json"
+          file: "work/perf.json"
 
   "destroy cluster":
     - command: shell.exec
       # destroy the cluster
       params:
-        working_dir: dsi
+        working_dir: work
         script: |
           set -e
           set -o verbose
-          cd ./clusters/${cluster}
+          source ./dsienv.sh
           # destroy the EC2 cluster
           yes yes | ./terraform destroy
           yes yes | ./terraform destroy
@@ -114,32 +138,31 @@ functions:
           echo "Cluster DESTROYED."
           echo "All perf results"
           cd ..
-          cat perf_all.json | egrep "name|ops_per_sec"
+          cat perf.json | egrep "name|ops_per_sec"
 
   "make test log artifact":
     - command: shell.exec
-      # destroy the cluster
       params:
-        working_dir: dsi
+        working_dir: work
         script: |
           set -e
           set -o verbose
-          cd ./clusters/${cluster}/reports
+          source ./dsienv.sh
           # move additional file here
+          cd reports
           cp ../infrastructure_provisioning.out.yml .
-          cp ../ips.sh .
-          cp ../ips.py .
-          cp ../../perf_all.json .
+          if [ -f "../terraform.log" ]; then cp ../terraform.log .; fi
+          cp ../perf.json .
           cd ..
-          ../../bin/retrieve-diag-data.sh
-          ../../bin/generate-timeseries-html.sh
-          rm -rf ../reports
-          mv ./reports ..
+          mkdir -p ./reports/graphs
+          touch ./reports/graphs/timeseries-p1.html
+          $DSI_PATH/bin/retrieve-diag-data.sh
+          $DSI_PATH/bin/generate-timeseries-html.sh || true
 
     - command: archive.targz_pack
       params:
         target: "reports.tgz"
-        source_dir: "dsi/clusters"
+        source_dir: "work"
         include:
           - "reports/**"
 
@@ -158,8 +181,8 @@ functions:
       params:
             aws_key: ${aws_key}
             aws_secret: ${aws_secret}
-            local_file: dsi/clusters/reports/graphs/timeseries-p1.html
-            remote_file: dsi/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${test}-${build_id}.html
+            local_file: work/reports/graphs/timeseries-p1.html
+            remote_file: longevity-v3.2/${build_variant}/${revision}/${task_id}/${version_id}/logs/timeseries-p1-${test}-${build_id}.html
             bucket: mciuploads
             permissions: public-read
             content_type: text/html
@@ -169,32 +192,35 @@ functions:
     - command: json.get_history
       params:
         task: ${task_name}
-        file: "src/history.json"
+        file: "work/history.json"
         name: "perf"
     - command: json.get_history
       params:
         tags: true
         task: ${task_name}
-        file: "src/tags.json"
+        file: "work/tags.json"
         name: "perf"
     - command: shell.exec
       # post_run_check.py and longevity_override.json for DSI tests are part of dsi repo
       type : test
       params:
-        working_dir: src
+        working_dir: work
         script: |
           set -o errexit
           set -o verbose
-          python -u ../dsi/analysis/post_run_check.py --ycsb-throughput-analysis .  --rev ${revision} -f history.json -t tags.json --refTag 3.0.6-Baseline --overrideFile ../dsi/analysis/v3.2/longevity_override.json --project_id mongo-longevity --task_name ${task_name} --variant ${build_variant}
-
-
+          TAG="3.0.12-Baseline"
+          PROJECT="mongo-longevity"
+          OVERRIDEFILE="../src/dsi/dsi/analysis/v3.2/longevity_override.json"
+          python -u ../src/dsi/dsi/analysis/post_run_check.py ${script_flags} --reports-analysis reports --perf-file reports/perf.json --rev ${revision} -f history.json -t tags.json --refTag $TAG --overrideFile $OVERRIDEFILE --project_id $PROJECT --task_name ${task_name} --variant ${build_variant}
+al
 tasks:
 - name: compile
   commands:
+    - command: manifest.load
     - command: git.get_project
       params:
         directory: src
-    - command: git.apply_patch
+    - command: shell.exec
       params:
         directory: src
     - command: shell.exec
@@ -241,33 +267,58 @@ tasks:
     - func: "upload log file"
       vars:
         test: "ycsb"
-    - func: "destroy cluster"
     - func: "analyze"
+      vars:
+        script_flags: --ycsb-throughput-analysis reports
+    - func: "destroy cluster"
 
 - name: shard_cluster_MMAPv1_test
   depends_on:
     - name: compile
-  exec_timeout_secs: 28800
+  exec_timeout_secs: 216000
   commands:
     - func: "prepare environment"
     - func: "bring up 3 shard cluster"
     - func: "configure mongodb cluster"
     - func: "run test"
       vars:
-        test: "ycsb"
         storageEngine: "mmapv1"
         test: "ycsb"
     - func: "make test log artifact"
     - func: "upload log file"
       vars:
         test: "ycsb"
-    - func: "destroy cluster"
     - func: "analyze"
+      vars:
+        script_flags: --ycsb-throughput-analysis reports
+    - func: "destroy cluster"
+
+
+#######################################
+#               Modules               #
+#######################################
+# if a module is added and to be added to the manifest
+# be sure to add the module to git.get_project revisions parameter
+modules:
+- name: dsi
+  repo: git@github.com:10gen/dsi.git
+  prefix: dsi
+  branch: master
+
+- name: YCSB
+  repo: git@github.com:mongodb-labs/YCSB.git
+  prefix: YCSB
+  branch: evergreen
+
+
 
 buildvariants:
 - name: linux-wt-shard
   display_name: Linux WT Shard
   batchtime: 40320 # 4 weeks
+  modules: &modules
+    - dsi
+    - YCSB
   expansions:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
     setup: shard
@@ -280,12 +331,13 @@ buildvariants:
   tasks:
     - name: compile
       distros:
-      - rhel55
+      - rhel70
     - name: shard_cluster_test
 
 - name: linux-mmapv1-shard
   display_name: Linux MMAPv1 Shard
   batchtime: 40320 # 4 week
+  modules: *modules
   expansions:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
     setup: shard
@@ -298,5 +350,5 @@ buildvariants:
   tasks:
     - name: compile
       distros:
-      - rhel55
+      - rhel70
     - name: shard_cluster_MMAPv1_test