SERVER-49222 ARM64 support on Amazon Linux 2

5 files changed, 157 insertions, 25 deletions

diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml
index 46ccb1659d9..1203ee05af7 100644
--- a/buildscripts/package_test/.kitchen.yml
+++ b/buildscripts/package_test/.kitchen.yml
@@ -21,92 +21,98 @@ verifier:
 
 provisioner:
   name: chef_solo
-  require_chef_omnibus: 12.6.0
+  require_chef_omnibus: 12
 
 platforms:
-  - name: amazon
+  - name: amazon-x86-64
     driver:
       image_id: ami-c481fad3
     transport:
       username: ec2-user
-  - name: amazon2
+  - name: amazon2-x86-64
     driver:
       image_id: ami-428aa838
     transport:
       username: ec2-user
-  - name: debian71
+  - name: amazon2-arm64
+    driver:
+      image_id: ami-0c582118883b46f4f
+      instance_type: c6g.medium
+    transport:
+      username: ec2-user
+  - name: debian71-x86-64
     driver:
       image_id: ami-4b124a22
     transport:
       username: admin
-  - name: debian81
+  - name: debian81-x86-64
     driver:
       image_id: ami-896d85e2
     transport:
       username: admin
-  - name: debian92
+  - name: debian92-x86-64
     driver:
       image_id: ami-71b7750b
     transport:
       username: admin
-  - name: debian10
+  - name: debian10-x86-64
     driver:
       image_id: ami-02ad06066b040a84b
     transport:
       username: admin
-  - name: rhel62
+  - name: rhel62-x86-64
     driver:
       # Use rhel 6.5 chef fails on previous versions.
       image_id: ami-5b697332
     transport:
       username: root
       instance_type: t2.small
-  - name: rhel70
+  - name: rhel70-x86-64
     driver:
       image_id: ami-60a1e808
     transport:
       username: root
-  - name: rhel80
+  - name: rhel80-x86-64
     driver:
       image_id: ami-0c322300a1dd5dc79
     transport:
       username: ec2-user
-  - name: suse11
+  - name: suse11-x86-64
     driver:
       image_id: ami-7f2e6015
     transport:
       username: ec2-user
-  - name: suse12
+  - name: suse12-x86-64
     driver:
       image_id: ami-043eebeabcc4e3d35
     transport:
       username: ec2-user
-  - name: suse15
+  - name: suse15-x86-64
     driver:
       image_id: ami-06ea7729e394412c8
     transport:
       username: ec2-user
-  - name: ubuntu1204
+  - name: ubuntu1204-x86-64
     driver:
       image_id: ami-3fec7956
     transport:
       username: ubuntu
-  - name: ubuntu1404
+  - name: ubuntu1404-x86-64
     driver:
       image_id: ami-1d8c9574
     transport:
       username: ubuntu
-  - name: ubuntu1604
+  - name: ubuntu1604-x86-64
     driver:
       image_id: ami-64140d0e
     transport:
       username: ubuntu
-  - name: ubuntu1804
+  - name: ubuntu1804-x86-64
     driver:
       image_id: ami-7ad76705
     transport:
       username: ubuntu
-  - name: ubuntu2004
+  - name: ubuntu2004-x86-64
     driver:
       image_id: ami-068663a3c619dd892
     transport:
diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb
index 7968a875f50..a6294da8dd5 100644
--- a/buildscripts/package_test/recipes/install_mongodb.rb
+++ b/buildscripts/package_test/recipes/install_mongodb.rb
@@ -24,6 +24,7 @@ end
 
 execute 'extract artifacts' do
   command "tar xzvf #{artifacts_tarball}"
+  live_stream true
   cwd homedir
 end
 
@@ -42,6 +43,7 @@ if platform_family? 'debian'
 
   execute 'apt-get update' do
     command 'apt-get update'
+    live_stream true
   end
 
   ENV['DEBIAN_FRONTEND'] = 'noninteractive'
@@ -51,6 +53,7 @@ if platform_family? 'debian'
   # for enterprise builds. We install dependencies in the next block.
   execute 'install mongod' do
     command 'dpkg -i `find . -name "*server*.deb"`'
+    live_stream true
     cwd homedir
     returns [0, 1]
   end
@@ -58,6 +61,7 @@ if platform_family? 'debian'
   # install the tools so we can test install_compass
   execute 'install mongo tools' do
     command 'dpkg -i `find . -name "*tools-extra*.deb"`'
+    live_stream true
     cwd homedir
     returns [0, 1]
   end
@@ -67,34 +71,43 @@ if platform_family? 'debian'
   # to install dependencies after the fact.
   execute 'install dependencies' do
     command 'apt-get update && apt-get -y -f install'
+    live_stream true
   end
 
   # the ubuntu 16.04 image does not have python installed by default
   # and it is required for the install_compass script
   execute 'install python' do
     command 'apt-get install -y python'
+    live_stream true
   end
 
   execute 'install mongo shell' do
     command 'dpkg -i `find . -name "*shell*.deb"`'
+    live_stream true
     cwd homedir
   end
 end
 
 if platform_family? 'rhel'
+  bash 'wait for yum updates if they are running' do
+    sleep 120
+  end
   execute 'install mongod' do
     command 'yum install -y `find . -name "*server*.rpm"`'
+    live_stream true
     cwd homedir
   end
 
   # install the tools so we can test install_compass
   execute 'install mongo tools' do
     command 'yum install -y `find . -name "*tools-extra*.rpm"`'
+    live_stream true
     cwd homedir
   end
 
   execute 'install mongo shell' do
     command 'yum install -y `find . -name "*shell*.rpm"`'
+    live_stream true
     cwd homedir
   end
 end
@@ -114,21 +127,24 @@ if platform_family? 'suse'
     done
     exit 1
   EOD
+  flags "-x"
   end
 
   execute 'install mongod' do
     command 'zypper --no-gpg-checks -n install `find . -name "*server*.rpm"`'
+    live_stream true
     cwd homedir
   end
 
   execute 'install mongo' do
     command 'zypper --no-gpg-checks -n install `find . -name "*shell*.rpm"`'
+    live_stream true
     cwd homedir
   end
 end
 
 inspec_wait = <<HEREDOC
-#!/bin/bash
+#!/bin/bash -x
 ulimit -v unlimited
 for i in {1..60}
 do
diff --git a/buildscripts/packager_enterprise.py b/buildscripts/packager_enterprise.py
index 6c32ea4251c..4bbbfb4b738 100755
--- a/buildscripts/packager_enterprise.py
+++ b/buildscripts/packager_enterprise.py
@@ -137,10 +137,11 @@ class EnterpriseDistro(packager.Distro):
         if arch == "arm64":
             if self.dname == 'ubuntu':
                 return ["ubuntu1604", "ubuntu1804", "ubuntu2004"]
-            return []
         if arch == "aarch64":
             if self.dname == 'redhat':
                 return ["rhel82"]
+            if self.dname == 'amazon2':
+                return ["amazon2"]
             return []
 
         if re.search("(redhat|fedora|centos)", self.dname):
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 4fe23061b60..a6c7a4cddff 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -333,6 +333,7 @@ variables:
 # If a variant is listed here and has a push task, mongocryptd is pushed
 - mongocryptd_variants: &mongocryptd_variants
   - enterprise-amazon2
+  - enterprise-amazon2-arm64
   - enterprise-debian92-64
   - enterprise-debian10-64
   - enterprise-linux-64-amazon-ami
@@ -391,6 +392,7 @@ variables:
   - enterprise-rhel-62-64-bit
   - enterprise-rhel-70-64-bit
   - enterprise-rhel-82-arm64
+  - enterprise-amazon2-arm64
   - enterprise-ubuntu1604-64
   - enterprise-ubuntu1804-64
   - enterprise-windows
@@ -400,7 +402,9 @@ variables:
   - amazon
   - enterprise-linux-64-amazon-ami
   - amazon2
+  - amazon2-arm64
   - enterprise-amazon2
+  - enterprise-amazon2-arm64
   - debian10
   - enterprise-debian10-64
   - debian92
@@ -2547,17 +2551,21 @@ functions:
         export KITCHEN_SUBNET="${kitchen_subnet}"
         export KITCHEN_VPC="${kitchen_vpc}"
 
+        if [[ "${packager_arch}" == "aarch64" || "${packager_arch}" == "arm64" ]]; then
+          kitchen_packager_distro="${packager_distro}-arm64"
+        else
+          kitchen_packager_distro="${packager_distro}-x86-64"
+        fi
+
         ${activate_virtualenv}
         # set expiration tag 2 hours in the future, since no test should take this long
         export KITCHEN_EXPIRE="$($python -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')"
 
         for i in {1..3}
         do
-          # kitchen commands use regex so 'kitchen verify amazon' matches both amazon and amazon2
-          # that's why we pass $ at the end of "${packager_distro}"
-          if ! kitchen verify "${packager_distro}"\$; then
+          if ! kitchen verify $kitchen_packager_distro; then
             verified="false"
-            kitchen destroy "${packager_distro}"\$ || true
+            kitchen destroy $kitchen_packager_distro || true
             sleep 30
           else
             verified="true"
@@ -2565,7 +2573,7 @@ functions:
           fi
         done
 
-        kitchen destroy "${packager_distro}"\$ || true
+        kitchen destroy $kitchen_packager_distro || true
         test "$verified" = "true"
 
   "copy ec2 monitor files": &copy_ec2_monitor_files
@@ -9703,6 +9711,106 @@ buildvariants:
     distros:
     - amazon2-small
 
+- name: enterprise-amazon2-arm64
+  display_name: "Enterprise Amazon Linux 2 arm64"
+  modules:
+  - enterprise
+  run_on:
+  - amazon2-arm64-small
+  expansions:
+    additional_package_targets: archive-mongocryptd archive-mongocryptd-debug archive-mh archive-mh-debug
+    push_path: linux
+    push_bucket: downloads.10gen.com
+    push_name: linux
+    push_arch: aarch64-enterprise-amazon2
+    compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    has_packages: true
+    packager_script: packager_enterprise.py
+    packager_arch: aarch64
+    packager_distro: amazon2
+    repo_edition: enterprise
+    scons_cache_scope: shared
+  tasks:
+  - name: compile_all_run_unittests_TG
+    distros:
+    - amazon2-arm64-large
+  - name: .aggfuzzer !.multiversion
+  - name: audit
+  - name: auth_audit_gen
+  - name: auth_gen
+  - name: causally_consistent_jscore_txns_passthrough
+  - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
+  - name: external_auth
+  - name: external_auth_aws
+  - name: .jscore .common !.compat !.decimal !.sharding
+  - name: jsCore_txns_large_txns_format
+  - name: .jstestfuzz .common
+  - name: .logical_session_cache .one_sec
+  - name: replica_sets_auth_gen
+  - name: replica_sets_jscore_passthrough
+  - name: .replica_sets .multi_oplog
+  - name: sasl
+  - name: search
+  - name: search_auth
+  - name: search_ssl
+  - name: sharding_auth_audit_gen
+  - name: sharding_auth_gen
+  - name: snmp
+  - name: .stitch
+  - name: test_packages
+    distros:
+    - ubuntu1604-packer
+  - name: .publish
+    distros:
+    - rhel80-small
+
+- name: amazon2-arm64
+  display_name: Amazon Linux 2 arm64
+  run_on:
+  - amazon2-arm64-small
+  expansions:
+    push_path: linux
+    push_bucket: downloads.mongodb.org
+    push_name: linux
+    push_arch: aarch64-amazon2
+    compile_flags: --ssl MONGO_DISTMOD=amazon2 -j$(grep -c ^processor /proc/cpuinfo) CCFLAGS="-march=armv8-a+crc -mtune=generic" --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    has_packages: true
+    packager_script: packager.py
+    packager_arch: aarch64
+    packager_distro: amazon2
+    repo_edition: org
+    scons_cache_scope: shared
+    large_distro_name: amazon2-arm64-large
+  tasks:
+  - name: compile_all_run_unittests_TG
+    distros:
+    - amazon2-arm64-large
+  - name: aggregation
+  - name: .auth !.audit !.multiversion
+  - name: causally_consistent_jscore_txns_passthrough
+  - name: .misc_js
+  - name: .concurrency .common
+    distros:
+      - amazon2-arm64-large
+  - name: concurrency_replication_causal_consistency_gen
+  - name: disk_wiredtiger
+  - name: free_monitoring
+  - name: .jscore .common
+  - name: .jstestfuzz .common
+  - name: .logical_session_cache .one_sec
+  - name: replica_sets
+  - name: .replica_sets .common
+  - name: .sharding .txns
+  - name: .sharding .common !.op_query !.csrs !.multiversion
+  - name: .ssl
+  - name: .stitch
+  - name: test_packages
+    distros:
+    - ubuntu1604-packer
+  - name: .publish
+    distros:
+    - rhel80-small
+
 - name: tig-daily-cron
   modules:
   - enterprise
diff --git a/etc/generate_subtasks_config.yml b/etc/generate_subtasks_config.yml
index 326067b0ef1..8a34a517089 100644
--- a/etc/generate_subtasks_config.yml
+++ b/etc/generate_subtasks_config.yml
@@ -4,6 +4,7 @@ build_variant_large_distro_exceptions:
   - debian10
   - debian92
   - enterprise-amazon2
+  - enterprise-amazon2-arm64
   - enterprise-debian10-64
   - enterprise-debian92-64
   - enterprise-linux-64-amazon-ami