SERVER-29527 Use infrastructure_provisioning.sh and infrastructure_teardown.sh

* Use infrastructure_provisioning.sh instead of setup_cluster.sh.

* The code to save and reuse terraform.tfstate in
  /data/infrastructure_provisioning/ goes into infrastructure_provisioning.sh.
  (From there, the good old setup_cluster.sh is called too.)

* Rename destroy_cluster.sh to infrastructure_teardown.sh

(cherry picked from commit 859d07ad4fcaafa4302149707afa17e58975e893)

2 files changed, 21 insertions, 111 deletions

diff --git a/etc/longevity.yml b/etc/longevity.yml
index 33d04107de2..f6da942e0c1 100644
--- a/etc/longevity.yml
+++ b/etc/longevity.yml
@@ -68,10 +68,6 @@ functions:
           rm -rf ./*
           mkdir src
           mkdir work
-          mkdir bin
-          mkdir keys
-          pwd
-          ls
     - command: manifest.load
     - command: git.get_project
       params:
@@ -144,7 +140,7 @@ functions:
           virtualenv ./venv
           source ./venv/bin/activate
           pip install -r ../src/dsi/dsi/requirements.txt
-          python ../src/dsi/dsi/bin/bootstrap.py --production
+          python ../src/dsi/dsi/bin/bootstrap.py
     - command: shell.exec
       params:
         script: |
@@ -152,28 +148,19 @@ functions:
           set -e
           source work/dsienv.sh
           $DSI_PATH/bin/setup-dsi-env.sh
-          cp terraform/* work/
-          ls work
-    - command: shell.exec
-      params:
-        working_dir: work
-        script: |
-           set -v
-           set -e
-           ./terraform get --update
+          ls -a work
 
-  "bring up 3 shard cluster":
+  "infrastructure provisioning":
     - command: shell.exec
-      # bring up the cluster
+      # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances.
       params:
         working_dir: work
-        silent: true
         script: |
-          # to create a mongod EC2 cluster
           set -e
-          set -o verbose
+          set -v
           source ./dsienv.sh
-          $DSI_PATH/bin/setup-cluster.sh ${cluster} false --skip-fio
+          export PRODUCTION=true
+          $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster}
 
   "configure mongodb cluster":
     - command: shell.exec
@@ -213,10 +200,13 @@ functions:
           set -e
           set -o verbose
           source ./dsienv.sh
-          # destroy the EC2 cluster
-          yes yes | ./terraform destroy --var-file=cluster.json
-          yes yes | ./terraform destroy --var-file=cluster.json
+          # Longevity runs so rarely, we simply teardown the cluster when done.
+          # Note that nowadays infrastructure_teardown.sh is actually copying the terraform.tfstate into /data/infrastructure_provisioning
+          # but as of this writing the rhel70-perf-longevity distro didn't actually use the teardown hook.
+          source ./dsienv.sh
+          $DSI_PATH/bin/infrastructure_teardown.sh
           echo "Cluster DESTROYED."
+          echo
           echo "All perf results"
           cd ..
           cat perf.json | egrep "name|ops_per_sec"
@@ -323,7 +313,7 @@ tasks:
       vars:
         storageEngine: "wiredTiger"
         test: "ycsb"
-    - func: "bring up 3 shard cluster"
+    - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
     - func: "run test"
       vars:
@@ -344,7 +334,7 @@ tasks:
       vars:
         storageEngine: "mmapv1"
         test: "ycsb"
-    - func: "bring up 3 shard cluster"
+    - func: "infrastructure provisioning"
     - func: "configure mongodb cluster"
     - func: "run test"
       vars:
diff --git a/etc/system_perf.yml b/etc/system_perf.yml
index 361adc24f48..dbd51db314d 100644
--- a/etc/system_perf.yml
+++ b/etc/system_perf.yml
@@ -63,9 +63,6 @@ functions:
           rm -rf ./*
           mkdir src
           mkdir work
-          mkdir bin
-          pwd
-          ls
     - command: manifest.load
     - command: git.get_project
       params:
@@ -149,96 +146,19 @@ functions:
           set -e
           source work/dsienv.sh
           $DSI_PATH/bin/setup-dsi-env.sh
-          cp terraform/* work/
-          ls work
-    - command: shell.exec
-      params:
-        working_dir: work
-        script: |
-          set -v
-          set -e
-          ./terraform get --update
-    - command: shell.exec
-      # set up /data/infrastructure_provisioning to keep track of resources,
-      # and allow Evergreen to release resources from there
-      params:
-        script: |
-          set -o verbose
-
-          # To force re-creation of existing instances, e.g. for a patch build, you can uncomment this:
-          #if [ -d "/data/infrastructure_provisioning" ]; then
-          #  /data/infrastructure_provisioning/terraform/infrastructure_teardown.sh \
-          #  && rm -rf /data/infrastructure_provisioning
-          #fi
-
-          # IF there's anything still in $HOME it's old or a mistake. Clean it up.
-          if [ -d "$HOME/infrastructure_provisioning" ]; then
-           $HOME/infrastructure_provisioning/terraform/infrastructure_teardown.sh \
-           && rm -rf $HOME/infrastructure_provisioning
-          fi
-
-          if [ ! -d "/data/infrastructure_provisioning" ]; then
-            echo "copying terraform to Evergreen host"
-            mkdir /data/infrastructure_provisioning
-            cp -r terraform /data/infrastructure_provisioning/.
-            cp -r modules /data/infrastructure_provisioning/.
-            echo "copying infrastructure_teardown.sh to Evergreen host"
-            cp src/dsi/dsi/bin/destroy_cluster.sh /data/infrastructure_provisioning/terraform/infrastructure_teardown.sh
-          fi
-          ls -l /data/infrastructure_provisioning
+          ls -a work
 
   "infrastructure provisioning":
     - command: shell.exec
-      # if /data/infrastructure_provisioning exists, get info about provisioned resources
-      # from there otherwise provision resources from the cloud
+      # call infrastructure-provisioning.sh. This will either create a cluster, or update tags on existing instances.
       params:
         working_dir: work
         script: |
           set -e
-          set -o verbose
-          if [ -e "/data/infrastructure_provisioning/terraform/terraform.tfstate" ]; then
-            echo "Restrieving info for existing resources"
-            cp /data/infrastructure_provisioning/terraform/terraform.tfstate .
-          else
-            echo "No existing resources found"
-          fi
-    - command: shell.exec
-      # call setup-cluster.sh, in most cases this only updates tags
-      params:
-        working_dir: work
-        script: |
-          # don't run this with "set -e" so we can set up properly for the teardown.sh
-          set -o verbose
+          set -v
           source ./dsienv.sh
-          if [ ! -e /data/infrastructure_provisioning/terraform/provisioned.${cluster} ]; then
-            EXISTING="false"
-          else
-            EXISTING="true"
-          fi
-
-          $DSI_PATH/bin/setup-cluster.sh ${cluster} $EXISTING
-          # handle the case when setup-cluster.sh fail
-          if [ $? -eq 0 ]; then
-            echo "Resource provisioned/updated."
-            # copy terraform information needed for teardown
-            cp {terraform.tfstate,cluster.tf,terraform.tfvars,security.tf,cluster.json} /data/infrastructure_provisioning/terraform/.
-            rsync -vr ../modules /data/infrastructure_provisioning/modules
-            cd /data/infrastructure_provisioning/terraform
-            ./terraform get
-            # use provisioned.${shard} to indicate the type of clusters held by EVG host
-            # remove previous information and keep the latest cluster type
-            rm provisioned.*
-            touch provisioned.${cluster}
-            echo "Provisioning state updated on Evergreen host."
-          else
-            echo "Failed to provision resources. Cleaning up partial state."
-            cp ../src/dsi/dsi/bin/destroy_cluster.sh .
-            ./destroy_cluster.sh
-            echo "Resource released."
-            rm -r /data/infrastructure_provisioning
-            echo "Cleaned up provisioning state on Evergreen host. Exiting test."
-            exit 1
-          fi
+          export PRODUCTION=true
+          $DSI_PATH/bin/infrastructure_provisioning.sh ${cluster}
 
   "configure mongodb cluster":
     - command: shell.exec
@@ -250,7 +170,7 @@ functions:
           set -o verbose
           source ./dsienv.sh
           source ./venv/bin/activate
-          $DSI_PATH/bin/mongodb_setup.py && echo "${cluster} MongoDB Cluster STARTED."
+          $DSI_PATH/bin/mongodb_setup.py && echo "${setup} MongoDB Cluster STARTED."
 
   "run test":
     - command: shell.exec