From 92b396db0088e4385e827a0b2306ce7c9ea8b941 Mon Sep 17 00:00:00 2001 From: Ryan Egesdahl Date: Fri, 3 Feb 2023 20:38:22 +0000 Subject: SERVER-70427 Remove Chef+Kitchen package test infrastructure (cherry picked from commit 09ca39216efd0c28c71144d25f55b72a9bba24a4) (cherry picked from commit 544d9e6c34b76575662d8ab5eec5d5e4247e93f2) (cherry picked from commit 3a50f68f565d4fda4120ab272153d1cf7394d4a3) (cherry picked from commit 9ed6f0873d305f4ab19d8b45a9217784a1145ad2) --- buildscripts/package_test/.kitchen.yml | 123 ---------- .../package_test/files/sources.list.debian8 | 4 - buildscripts/package_test/metadata.rb | 2 - .../package_test/recipes/install_mongodb.rb | 204 ----------------- .../test/recipes/service/install_mongodb_spec.rb | 249 --------------------- etc/evergreen.yml | 47 ---- 6 files changed, 629 deletions(-) delete mode 100644 buildscripts/package_test/.kitchen.yml delete mode 100644 buildscripts/package_test/files/sources.list.debian8 delete mode 100644 buildscripts/package_test/metadata.rb delete mode 100644 buildscripts/package_test/recipes/install_mongodb.rb delete mode 100644 buildscripts/package_test/test/recipes/service/install_mongodb_spec.rb diff --git a/buildscripts/package_test/.kitchen.yml b/buildscripts/package_test/.kitchen.yml deleted file mode 100644 index d31c86a2529..00000000000 --- a/buildscripts/package_test/.kitchen.yml +++ /dev/null @@ -1,123 +0,0 @@ ---- -driver: - name: ec2 - region: us-east-1 - vpc_mode: true - vpc_id: <%= ENV['KITCHEN_VPC'] %> - subnet_id: <%= ENV['KITCHEN_SUBNET'] %> - security_group_ids: - - <%= ENV['KITCHEN_SECURITY_GROUP'] %> - aws_ssh_key_id: <%= ENV['KITCHEN_SSH_KEY_ID'] %> - interface: private - associate_public_ip: true - tags: - name: "server package test" - owner: "build" - expire-on: "<%= ENV['KITCHEN_EXPIRE'] %>" - -verifier: - name: inspec - sudo: true - -provisioner: - name: chef_solo - require_chef_omnibus: 12 - -platforms: - - name: amazon-x86-64 - driver: - image_id: ami-0080e4c5bc078760e - transport: - username: ec2-user - - name: amazon2-x86-64 - driver: - image_id: ami-428aa838 - transport: - username: ec2-user - - name: amazon2-arm64 - driver: - image_id: ami-0c582118883b46f4f - instance_type: c6g.medium - transport: - username: ec2-user - - name: debian71-x86-64 - driver: - image_id: ami-4b124a22 - transport: - username: admin - - name: debian81-x86-64 - driver: - image_id: ami-896d85e2 - transport: - username: admin - - name: debian92-x86-64 - driver: - image_id: ami-71b7750b - transport: - username: admin - - name: debian10-x86-64 - driver: - image_id: ami-0dedf6a6502877301 - transport: - username: admin - - name: rhel62-x86-64 - driver: - image_id: ami-0a47672f6c7827dd2 - transport: - username: ec2-user - instance_type: t2.small - - name: rhel70-x86-64 - driver: - image_id: ami-2051294a - transport: - username: root - - name: rhel80-x86-64 - driver: - image_id: ami-0c322300a1dd5dc79 - transport: - username: ec2-user - - name: suse11-x86-64 - driver: - image_id: ami-7f2e6015 - transport: - username: ec2-user - - name: suse12-x86-64 - driver: - image_id: ami-043eebeabcc4e3d35 - transport: - username: ec2-user - - name: suse15-x86-64 - driver: - image_id: ami-06ea7729e394412c8 - transport: - username: ec2-user - - name: ubuntu1204-x86-64 - driver: - image_id: ami-3fec7956 - transport: - username: ubuntu - - name: ubuntu1404-x86-64 - driver: - image_id: ami-1d8c9574 - transport: - username: ubuntu - - name: ubuntu1604-x86-64 - driver: - image_id: ami-64140d0e - transport: - username: ubuntu - - name: ubuntu1804-x86-64 - driver: - image_id: ami-7ad76705 - transport: - username: ubuntu - -transport: - ssh_key: ~/.ssh/kitchen.pem - -suites: - - name: service - run_list: - - recipe[package_test::install_mongodb] - attributes: - artifacts_url: <%= ENV['KITCHEN_ARTIFACTS_URL'] %> diff --git a/buildscripts/package_test/files/sources.list.debian8 b/buildscripts/package_test/files/sources.list.debian8 deleted file mode 100644 index 0ad1174270a..00000000000 --- a/buildscripts/package_test/files/sources.list.debian8 +++ /dev/null @@ -1,4 +0,0 @@ -deb http://archive.debian.org/debian jessie main -deb-src http://archive.debian.org/debian jessie main -deb http://security.debian.org/ jessie/updates main -deb-src http://security.debian.org/ jessie/updates main diff --git a/buildscripts/package_test/metadata.rb b/buildscripts/package_test/metadata.rb deleted file mode 100644 index 3bbfb147acc..00000000000 --- a/buildscripts/package_test/metadata.rb +++ /dev/null @@ -1,2 +0,0 @@ -name 'package_test' -version '0.1.0' diff --git a/buildscripts/package_test/recipes/install_mongodb.rb b/buildscripts/package_test/recipes/install_mongodb.rb deleted file mode 100644 index 56385dbc2b0..00000000000 --- a/buildscripts/package_test/recipes/install_mongodb.rb +++ /dev/null @@ -1,204 +0,0 @@ -# This Chef task installs MongoDB in a new EC2 instance spun up by Kitchen in -# preparation for running some basic server functionality tests. - -artifacts_tarball = 'artifacts.tgz' -user = node['current_user'] -homedir = node['etc']['passwd'][user]['dir'] - -ruby_block 'allow sudo over tty' do - block do - file = Chef::Util::FileEdit.new('/etc/sudoers') - file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults requiretty') - file.search_file_replace_line(/Defaults\s+requiretty/, '#Defaults !visiblepw') - file.write_file - end -end - -# This file limits processes to 1024. It therefore interfereres with `ulimit -u` when present. -if platform_family? 'rhel' - file '/etc/security/limits.d/90-nproc.conf' do - action :delete - end -end - -if %w(7).include?(node['platform_version'][0]) and node['platform'] == 'redhat' - - # RHEL 7 client - if node['platform'] == 'redhat' and node['platform_version'][0] == "7" - rhui_client_url = "http://boxes.10gen.com/build/rh-amazon-rhui-client-3.0.45-1.el7.noarch.rpm" - end - - execute 'remove old amazon rhui client' do - command 'rpm -ev rh-amazon-rhui-client' - live_stream true - end - execute 'install updated amazon rhui client' do - command "rpm -ivh --nodeps #{rhui_client_url}" - cwd homedir - live_stream true - end - execute 'reinstall updated amazon rhui client for dependencies' do - command 'yum reinstall -y rh-amazon-rhui-client*' - live_stream true - end -end - -remote_file "#{homedir}/#{artifacts_tarball}" do - source node['artifacts_url'] -end - -execute 'extract artifacts' do - command "tar xzvf #{artifacts_tarball}" - live_stream true - cwd homedir -end - -if platform_family? 'debian' - - # SERVER-40491 Debian 8 sources.list need to point to archive url - if node['platform'] == 'debian' and node['platform_version'] == '8.1' - cookbook_file '/etc/apt/sources.list' do - source 'sources.list.debian8' - owner 'root' - group 'root' - mode '0644' - action :create - end - end - - execute 'apt update' do - command 'apt update' - live_stream true - end - - ENV['DEBIAN_FRONTEND'] = 'noninteractive' - package 'openssl' - - if node['platform_version'] == '16.04' - execute 'update ca-certificates' do - command 'apt-get install -y ca-certificates' - live_stream true - end - end - - # the ubuntu 16.04 image does not have some dependencies installed by default - # and it is required for the install_compass script - execute 'install dependencies' do - command 'apt-get install -y python libsasl2-modules-gssapi-mit' - live_stream true - end - - # dpkg returns 1 if dependencies are not satisfied, which they will not be - # for enterprise builds. We install dependencies in the next block. - execute 'install mongod' do - command 'dpkg -i `find . -name "*server*.deb"`' - live_stream true - cwd homedir - returns [0, 1] - end - - # install the tools so we can test install_compass - execute 'install mongo tools' do - command 'dpkg -i `find . -name "*tools*.deb"`' - live_stream true - cwd homedir - returns [0, 1] - end - - # yum and zypper fetch dependencies automatically, but dpkg does not. - # Installing the dependencies explicitly is fragile, so we reply on apt-get - # to install dependencies after the fact. - execute 'update and fix broken dependencies' do - command 'apt update && apt -y -f install' - live_stream true - end - - execute 'install mongo shell' do - command 'dpkg -i `find . -name "*shell*.deb"`' - live_stream true - cwd homedir - end -end - -if platform_family? 'rhel' - bash 'wait for yum updates if they are running' do - sleep 120 - end - execute 'install mongod' do - command 'yum install -y `find . -name "*server*.rpm"`' - live_stream true - cwd homedir - end - - # install the tools so we can test install_compass - execute 'install mongo tools' do - command 'yum install -y `find . -name "*tools*.rpm"`' - live_stream true - cwd homedir - end - - execute 'install mongo shell' do - command 'yum install -y `find . -name "*shell*.rpm"`' - live_stream true - cwd homedir - end -end - -if platform_family? 'suse' - bash 'wait for zypper lock to be released' do - code <<-EOD - retry_counter=0 - # We also need to make sure another instance of zypper isn't running while - # we do our install, so just run zypper refresh until it doesn't fail. - # Waiting for 2 minutes is copied from an internal project where we do this. - until [ "$retry_counter" -ge "12" ]; do - zypper refresh && exit 0 - retry_counter=$(($retry_counter + 1)) - [ "$retry_counter" = "12" ] && break - sleep 10 - done - exit 1 - EOD - flags "-x" - end - - execute 'install mongod' do - command 'zypper --no-gpg-checks -n install `find . -name "*server*.rpm"`' - live_stream true - cwd homedir - end - - execute 'install mongo tools' do - command 'zypper --no-gpg-checks -n install `find . -name "*tools*.rpm"`' - live_stream true - cwd homedir - end - - execute 'install mongo' do - command 'zypper --no-gpg-checks -n install `find . -name "*shell*.rpm"`' - live_stream true - cwd homedir - end -end - -inspec_wait = <= 7) or - (os[:name] == 'ubuntu' and os[:release].split('.')[0].to_i >= 16) or - (os[:name] == 'debian' and os[:release].split('.')[0].to_i >= 9) or - (os[:name] == 'amazon' and os[:release].split('.')[0].to_i == 2)) - describe command("install_compass") do - its('exit_status') { should eq 0 } - its('stderr') { should eq '' } - end - elsif os[:name] == 'suse' - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /You are using an unsupported platform/ } - end - else - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /You are using an unsupported Linux distribution/ } - end - end -else - describe command("install_compass") do - its('exit_status') { should eq 1 } - its('stderr') { should match /Sorry, MongoDB Compass is only supported on 64-bit Intel platforms/ } - end -end - -# wait to make sure mongod is ready -describe command("/inspec_wait.sh") do - its('exit_status') { should eq 0 } -end - -############################################################ -# This section verifies files, directories, and users -# - files and directories exist and have correct attributes -# - mongod user exists and has correct attributes -############################################################ - -# convenience variables for init system and package type -upstart = (os[:name] == 'ubuntu' && os[:release][0..1] == '14') || - (os[:name] == 'amazon') -sysvinit = if (os[:name] == 'debian' && os[:release][0] == '7') || - (os[:name] == 'redhat' && os[:release][0] == '6') || - (os[:name] == 'suse' && os[:release][0..1] == '11') || - (os[:name] == 'ubuntu' && os[:release][0..1] == '12') - true - else - false - end -systemd = !(upstart || sysvinit) -rpm = if os[:name] == 'amazon' || os[:name] == 'redhat' || os[:name] == 'suse' - true - else - false - end -deb = !rpm - -# these files should exist on all systems -%w( - /etc/mongod.conf - /usr/bin/mongod - /var/log/mongodb/mongod.log -).each do |filename| - describe file(filename) do - it { should be_file } - end -end - -if sysvinit - describe file('/etc/init.d/mongod') do - it { should be_file } - it { should be_executable } - end -end - -if systemd - unit_file_prefix = '' - if os[:name] == 'suse' - # Putting systemd unit files in /usr, which may be a separate partition - # and therefore not available during isolated startups, is bad practice. - # But it's what SUSE has chosen to do, so we have to deal with it. - unit_file_prefix = '/usr' - end - describe file("#{unit_file_prefix}/lib/systemd/system/mongod.service") do - it { should be_file } - end -end - -if rpm - %w( - /var/lib/mongo - /var/run/mongodb - ).each do |filename| - describe file(filename) do - it { should be_directory } - end - end - - describe user('mongod') do - it { should exist } - its('groups') { should include 'mongod' } - its('home') { should eq '/var/lib/mongo' } - its('shell') { should eq '/bin/false' } - end -end - -if deb - describe file('/var/lib/mongodb') do - it { should be_directory } - end - describe user('mongodb') do - it { should exist } - its('groups') { should include 'mongodb' } - # All versions of Debian 10 will use /usr/sbin/nologin for service - # account shells - its('shell') { - if ((os[:name] == 'debian' and os[:release].split('.')[0] == '10') or - (os[:name] == 'ubuntu' and os[:release] == '18.04')) - should eq '/usr/sbin/nologin' - else - should eq '/bin/false' - end - } - end -end - -############################################################ -# This section verifies ulimits. -############################################################ - -ulimits = { - 'Max file size' => 'unlimited', - 'Max cpu time' => 'unlimited', - 'Max address space' => 'unlimited', - 'Max open files' => '64000', - 'Max resident set' => 'unlimited', - 'Max processes' => '64000' -} -ulimits_cmd = 'cat /proc/$(pgrep mongod)/limits' - -ulimits.each do |limit, value| - describe command("#{ulimits_cmd} | grep \"#{limit}\"") do - its('stdout') { should match(/#{limit}\s+#{value}/) } - end -end - -############################################################ -# This section verifies reads and writes. -# - insert a document into the database -# - verify that findOne() returns a matching document -############################################################ - -describe command('sh -c "ulimit -v unlimited && mongo --eval \"db.smoke.insert({answer: 42})\""') do - its('exit_status') { should eq 0 } - its('stdout') { should match(/.+WriteResult\({ "nInserted" : 1 }\).+/m) } -end - -# read a document from the db -describe command('sh -c "ulimit -v unlimited && mongo --eval \"db.smoke.findOne()\""') do - its('exit_status') { should eq 0 } - its('stdout') { should match(/.+"answer" : 42.+/m) } -end - -############################################################ -# This section verifies uninstall. -############################################################ - -if rpm - describe command('rpm -e $(rpm -qa | grep "mongodb.*server" | awk \'{print $1}\')') do - its('exit_status') { should eq 0 } - end -elsif deb - describe command('dpkg -r $(dpkg -l | grep "mongodb.*server" | awk \'{print $2}\')') do - its('exit_status') { should eq 0 } - end -end - -# make sure we cleaned up -%w( - /lib/systemd/system/mongod.service - /usr/bin/mongod -).each do |filename| - describe file(filename) do - it { should_not exist } - end -end diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9fb566c8a77..a9e51f80d20 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -2017,46 +2017,6 @@ functions: exit 0 - "run kitchen": - command: shell.exec - type: test - params: - shell: bash - working_dir: src/buildscripts/package_test - script: | - set -o errexit - - export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}-packages.tgz" - export KITCHEN_SECURITY_GROUP="${kitchen_security_group}" - export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}" - export KITCHEN_SUBNET="${kitchen_subnet}" - export KITCHEN_VPC="${kitchen_vpc}" - - if [[ "${packager_arch}" == "aarch64" || "${packager_arch}" == "arm64" ]]; then - kitchen_packager_distro="${packager_distro}-arm64" - else - kitchen_packager_distro="${packager_distro}-x86-64" - fi - - ${activate_virtualenv} - # set expiration tag 2 hours in the future, since no test should take this long - export KITCHEN_EXPIRE="$($python -c 'import datetime; print((datetime.datetime.utcnow() + datetime.timedelta(hours=2)).strftime("%Y-%m-%d %H:%M:%S"))')" - - for i in {1..3} - do - if ! kitchen verify $kitchen_packager_distro; then - verified="false" - kitchen destroy $kitchen_packager_distro || true - sleep 30 - else - verified="true" - break - fi - done - - kitchen destroy $kitchen_packager_distro || true - test "$verified" = "true" - "copy ec2 monitor files": ©_ec2_monitor_files command: shell.exec params: @@ -6594,13 +6554,6 @@ tasks: - name: compile commands: - func: "fetch artifacts" - - func: "set up remote credentials" - vars: - private_key_file: ~/.ssh/kitchen.pem - private_key_remote: ${kitchen_private_key} - aws_key_remote: ${kitchen_aws_key} - aws_secret_remote: ${kitchen_aws_secret} - - func: "run kitchen" - name: publish_packages tags: ["publish"] -- cgit v1.2.1