SERVER-27140 Linux package smoke testingr3.4.2

(cherry picked from commit 337df9553665d0a4a87ac66f6bd53159b87f67c0)

SERVER-27441 Reduce spurious failures in Linux package smoke testing
(cherry picked from commit 7f74694a21f4e9c0a75e69e715133cfb590129b7)

SERVER-27827 Improve package smoke testing retry logic
(cherry picked from commit 249eaae3e1b78ed5c4b6b7425145a23dc658fd75)

5 files changed, 505 insertions, 0 deletions

diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml
new file mode 100644
index 00000000000..e9b6989d221
--- /dev/null
+++ b/buildscripts/package_test/.kitchen.yml
@@ -0,0 +1,81 @@
+---
+driver:
+  name: ec2
+  region: us-east-1
+  subnet_id: <%= ENV['KITCHEN_SUBNET'] %>
+  security_group_ids:
+    - <%= ENV['KITCHEN_SECURITY_GROUP'] %>
+  aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %>
+  interface: dns
+  associate_public_ip: true
+
+verifier:
+  name: inspec
+  sudo: true
+
+provisioner:
+  name: chef_solo
+  require_chef_omnibus: 12.6.0
+
+platforms:
+  - name: amazon
+    driver:
+      image_id: ami-c481fad3
+    transport:
+      username: ec2-user
+  - name: debian71
+    driver:
+      image_id: ami-4b124a22
+    transport:
+      username: admin
+  - name: debian81
+    driver:
+      image_id: ami-896d85e2
+    transport:
+      username: admin
+  - name: rhel62
+    driver:
+      # Use rhel 6.3 because chef-solo fails on rhel 6.2
+      image_id: ami-a35a33ca
+    transport:
+      username: root
+  - name: rhel70
+    driver:
+      image_id: ami-60a1e808
+    transport:
+      username: root
+  - name: suse11
+    driver:
+      image_id: ami-7f2e6015
+    transport:
+      username: ec2-user
+  - name: suse12
+    driver:
+      image_id: ami-aeb532c6
+    transport:
+      username: ec2-user
+  - name: ubuntu1204
+    driver:
+      image_id: ami-3fec7956
+    transport:
+      username: ubuntu
+  - name: ubuntu1404
+    driver:
+      image_id: ami-1d8c9574
+    transport:
+      username: ubuntu
+  - name: ubuntu1604
+    driver:
+      image_id: ami-64140d0e
+    transport:
+      username: ubuntu
+
+transport:
+  ssh_key: ~/.ssh/kitchen.pem
+
+suites:
+  - name: service
+    run_list:
+      - recipe[package_test::install_mongodb]
+    attributes:
+      artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %>
diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb
new file mode 100644
index 00000000000..3bbfb147acc
--- /dev/null
+++ b/buildscripts/package_test/metadata.rb
@@ -0,0 +1,2 @@
+name 'package_test'
+version '0.1.0'
diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb
new file mode 100644
index 00000000000..cc26434e65c
--- /dev/null
+++ b/buildscripts/package_test/recipes/install_mongodb.rb
@@ -0,0 +1,117 @@
+artifacts_tarball = 'artifacts.tgz'
+user = node['current_user']
+homedir = node['etc']['passwd'][user]['dir']
+
+ruby_block 'allow sudo over tty' do
+  block do
+    file = Chef::Util::FileEdit.new('/etc/sudoers')
+    file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty')
+    file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw')
+    file.write_file
+  end
+end
+
+# This file limits processes to 1024. It therefore interfereres with `ulimit -u` when present.
+if platform_family? 'rhel'
+  file '/etc/security/limits.d/90-nproc.conf' do
+    action :delete
+  end
+end
+
+remote_file "#{homedir}/#{artifacts_tarball}" do
+  source node['artifacts_url']
+end
+
+execute 'extract artifacts' do
+  command "tar xzvf #{artifacts_tarball}"
+  cwd homedir
+end
+
+if platform_family? 'debian'
+  execute 'apt-get update' do
+    command 'apt-get update'
+  end
+
+  package 'openssl'
+
+  # dpkg returns 1 if dependencies are not satisfied, which they will not be
+  # for enterprise builds. We install dependencies in the next block.
+  execute 'install mongod' do
+    command 'dpkg -i `find . -name "*server*.deb"`'
+    cwd homedir
+    returns [0, 1]
+  end
+
+  # yum and zypper fetch dependencies automatically, but dpkg does not.
+  # Installing the dependencies explicitly is fragile, so we reply on apt-get
+  # to install dependencies after the fact.
+  execute 'install dependencies' do
+    command 'apt-get update && apt-get -y -f install'
+  end
+
+  execute 'install mongo shell' do
+    command 'dpkg -i `find . -name "*shell*.deb"`'
+    cwd homedir
+  end
+end
+
+if platform_family? 'rhel'
+  execute 'install mongod' do
+    command 'yum install -y `find . -name "*server*.rpm"`'
+    cwd homedir
+  end
+
+  execute 'install mongo shell' do
+    command 'yum install -y `find . -name "*shell*.rpm"`'
+    cwd homedir
+  end
+end
+
+if platform_family? 'suse'
+  bash 'wait for zypper lock to be released' do
+    code <<-EOD
+    retry_counter=0
+    # We also need to make sure another instance of zypper isn't running while
+    # we do our install, so just run zypper refresh until it doesn't fail.
+    # Waiting for 2 minutes is copied from an internal project where we do this.
+    until [ "$retry_counter" -ge "12" ]; do
+        zypper refresh && exit 0
+        retry_counter=$(($retry_counter + 1))
+        [ "$retry_counter" = "12" ] && break
+        sleep 10
+    done
+    exit 1
+  EOD
+  end
+
+  execute 'install mongod' do
+    command 'zypper -n install `find . -name "*server*.rpm"`'
+    cwd homedir
+  end
+
+  execute 'install mongo' do
+    command 'zypper -n install `find . -name "*shell*.rpm"`'
+    cwd homedir
+  end
+end
+
+inspec_wait = <<HEREDOC
+#!/bin/bash
+for i in {1..60}
+do
+  mongo --eval "db.smoke.insert({answer: 42})"
+  if [ $? -eq 0 ]
+  then
+    exit 0
+  else
+    echo "sleeping"
+    sleep 1
+  fi
+done
+exit 1
+HEREDOC
+
+file '/inspec_wait.sh' do
+  content inspec_wait
+  mode '0755'
+end
diff --git a/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb
new file mode 100644
index 00000000000..e974e4bd54c
--- /dev/null
+++ b/buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb
@@ -0,0 +1,181 @@
+############################################################
+# This section verifies start, stop, and restart.
+# - stop mongod so that we begin testing from a stopped state
+# - verify start, stop, and restart
+############################################################
+
+# service is not in path for commands with sudo on suse
+service = os[:name] == 'suse' ? '/sbin/service' : 'service'
+
+describe command("#{service} mongod stop") do
+  its('exit_status') { should eq 0 }
+end
+
+describe command("#{service} mongod start") do
+  its('exit_status') { should eq 0 }
+end
+
+describe service('mongod') do
+  it { should be_running }
+end
+
+describe command("#{service} mongod stop") do
+  its('exit_status') { should eq 0 }
+end
+
+describe service('mongod') do
+  it { should_not be_running }
+end
+
+describe command("#{service} mongod restart") do
+  its('exit_status') { should eq 0 }
+end
+
+describe service('mongod') do
+  it { should be_running }
+end
+
+# wait to make sure mongod is ready
+describe command("/inspec_wait.sh") do
+  its('exit_status') { should eq 0 }
+end
+
+############################################################
+# This section verifies files, directories, and users
+# - files and directories exist and have correct attributes
+# - mongod user exists and has correct attributes
+############################################################
+
+# convenience variables for init system and package type
+upstart = (os[:name] == 'ubuntu' && os[:release][0..1] == '14') ||
+          (os[:name] == 'amazon')
+sysvinit = if (os[:name] == 'debian' && os[:release][0] == '7') ||
+              (os[:name] == 'redhat' && os[:release][0] == '6') ||
+              (os[:name] == 'suse' && os[:release][0..1] == '11') ||
+              (os[:name] == 'ubuntu' && os[:release][0..1] == '12')
+             true
+           else
+             false
+           end
+systemd = !(upstart || sysvinit)
+rpm = if os[:name] == 'amazon' || os[:name] == 'redhat' || os[:name] == 'suse'
+        true
+      else
+        false
+      end
+deb = !rpm
+
+# these files should exist on all systems
+%w(
+  /etc/mongod.conf
+  /usr/bin/mongod
+  /var/log/mongodb/mongod.log
+).each do |filename|
+  describe file(filename) do
+    it { should be_file }
+  end
+end
+
+if sysvinit
+  describe file('/etc/init.d/mongod') do
+    it { should be_file }
+    it { should be_executable }
+  end
+end
+
+if systemd
+  describe file('/lib/systemd/system/mongod.service') do
+    it { should be_file }
+  end
+end
+
+if rpm
+  %w(
+    /var/lib/mongo
+    /var/run/mongodb
+  ).each do |filename|
+    describe file(filename) do
+      it { should be_directory }
+    end
+  end
+
+  describe user('mongod') do
+    it { should exist }
+    its('groups') { should include 'mongod' }
+    its('home') { should eq '/var/lib/mongo' }
+    its('shell') { should eq '/bin/false' }
+  end
+end
+
+if deb
+  describe file('/var/lib/mongodb') do
+    it { should be_directory }
+  end
+
+  describe user('mongodb') do
+    it { should exist }
+    its('groups') { should include 'mongodb' }
+    its('shell') { should eq '/bin/false' }
+  end
+end
+
+############################################################
+# This section verifies ulimits.
+############################################################
+
+ulimits = {
+  'Max file size'     => 'unlimited',
+  'Max cpu time'      => 'unlimited',
+  'Max address space' => 'unlimited',
+  'Max open files'    => '64000',
+  'Max resident set'  => 'unlimited',
+  'Max processes'     => '64000'
+}
+ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits'
+
+ulimits.each do |limit, value|
+  describe command("#{ulimits_cmd} | grep \"#{limit}\"") do
+    its('stdout') { should match(/#{limit}\s+#{value}/) }
+  end
+end
+
+############################################################
+# This section verifies reads and writes.
+# - insert a document into the database
+# - verify that findOne() returns a matching document
+############################################################
+
+describe command('mongo --eval "db.smoke.insert({answer: 42})"') do
+  its('exit_status') { should eq 0 }
+  its('stdout') { should match(/.+WriteResult\({ "nInserted" : 1 }\).+/m) }
+end
+
+# read a document from the db
+describe command('mongo --eval "db.smoke.findOne()"') do
+  its('exit_status') { should eq 0 }
+  its('stdout') { should match(/.+"answer" : 42.+/m) }
+end
+
+############################################################
+# This section verifies uninstall.
+############################################################
+
+if rpm
+  describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do
+    its('exit_status') { should eq 0 }
+  end
+elsif deb
+  describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do
+    its('exit_status') { should eq 0 }
+  end
+end
+
+# make sure we cleaned up
+%w(
+  /lib/systemd/system/mongod.service
+  /usr/bin/mongod
+).each do |filename|
+  describe file(filename) do
+    it { should_not exist }
+  end
+end
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 698beb96977..66f49098826 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -729,6 +729,62 @@ functions:
           kill_process "$process"
         done
 
+  "set up kitchen credentials":
+    command: shell.exec
+    params:
+      silent: true
+      script: |
+        set -o errexit
+
+        mkdir -p ~/.ssh ~/.aws
+        echo -n "${kitchen_private_key}" > ~/.ssh/kitchen.pem
+        chmod 0600 ~/.ssh/kitchen.pem
+
+        cat <<EOF > ~/.aws/config
+        [default]
+        region = us-east-1
+        EOF
+
+        cat <<EOF > ~/.aws/credentials
+        [default]
+        aws_access_key_id = ${kitchen_aws_key}
+        aws_secret_access_key = ${kitchen_aws_secret}
+        EOF
+
+  "run kitchen":
+    command: shell.exec
+    type: test
+    params:
+      shell: bash
+      working_dir: src/buildscripts/package_test
+      script: |
+        set -o errexit
+
+        export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz"
+        export KITCHEN_SECURITY_GROUP="${kitchen_security_group}"
+        export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}"
+        export KITCHEN_SUBNET="${kitchen_subnet}"
+
+        for i in {1..3}
+        do
+          if ! kitchen converge "${packager_distro}"; then
+            converged="false"
+            kitchen destroy "${packager_distro}" || true
+            sleep 30
+          else
+            break
+          fi
+        done
+
+        test "$converged" != "false"
+
+        if ! kitchen verify "${packager_distro}"; then
+          verified="false"
+        fi
+
+        kitchen destroy "${packager_distro}" || true
+        test "$verified" != "false"
+
 pre:
   - command: shell.track
   - func: "kill processes"
@@ -2642,6 +2698,14 @@ tasks:
       resmoke_args: --suites=views_rs --storageEngine=wiredTiger
       run_multiple_jobs: true
 
+- name: package
+  depends_on:
+  - name: compile
+  commands:
+    - func: "fetch artifacts"
+    - func: "set up kitchen credentials"
+    - func: "run kitchen"
+
 - name: push
   patchable: false
   depends_on:
@@ -3654,6 +3718,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: ubuntu1404
@@ -3782,6 +3849,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: ubuntu1604
@@ -3901,6 +3971,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: enterprise-ubuntu1604-arm64
@@ -4318,6 +4391,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: amazon
@@ -4440,6 +4516,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 ###########################################
@@ -5567,6 +5646,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: enterprise-rhel-62-64-bit-coverage
@@ -5840,6 +5922,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -5961,6 +6046,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: rhel70
@@ -6097,6 +6185,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: enterprise-rhel-71-ppc64le
@@ -6430,6 +6521,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -6494,6 +6588,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -6549,6 +6646,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -6625,6 +6725,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -6838,6 +6941,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -6892,6 +6998,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 - name: enterprise-suse11-s390x
@@ -7099,6 +7208,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -7250,6 +7362,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -7305,6 +7420,9 @@ buildvariants:
   - name: ssl
   - name: sslSpecial
   - name: unittests
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -7427,6 +7545,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push
 
 
@@ -7549,6 +7670,9 @@ buildvariants:
   - name: views_WT
   - name: views_rs
   - name: views_rs_WT
+  - name: package
+    distros:
+    - ubuntu1604-packer
   - name: push