Rework CI setup (#2367)

This commit:

* Removes Travis CI from the build (no more .travis.yml)
* Moves Jenkinsfile to `build-aux/Jenkinsfile.full` and updates the version
  of Erlang used in all steps to `20.3.8.24-1` (Erlang Solutions
  packages).
* Introduces a new `build-aux/Jenkinsfile.pr` that just builds
  CouchDB against the 3 most important versions of Erlang,
  intended for use when building PRs, using a special
  `debian-buster-erlang-all` image that has `kerl` builds of those 3
  versions of Erlang:
    * the oldest supported
    * the version we release our binary convenience builds with
    * the latest supported
* Builds against SpiderMonkey 60 on the platforms where it is available
  (currently, only Debian buster)
* Updated README file with new, dynamic Jenkins embeddable build status
  badge

3 files changed, 927 insertions, 0 deletions

diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
new file mode 100644
index 000000000..7087bba63
--- /dev/null
+++ b/build-aux/Jenkinsfile.full
@@ -0,0 +1,633 @@
+#!groovy
+//
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+build_and_test = '''
+mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}
+cd ${platform}
+rm -rf build
+mkdir build
+cd build
+tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
+cd apache-couchdb-*
+./configure --with-curl --spidermonkey-version ${sm_ver}
+make check || (build-aux/logfile-uploader.py && false)
+'''
+
+make_packages = '''
+cd ${platform}
+git clone https://github.com/apache/couchdb-pkg
+rm -rf couchdb
+mkdir couchdb
+cp ${WORKSPACE}/apache-couchdb-*.tar.gz couchdb
+tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -C couchdb
+cd couchdb-pkg
+make ${platform} PLATFORM=${platform}
+'''
+
+cleanup_and_save = '''
+rm -rf ${WORKSPACE}/pkgs/${platform}
+mkdir -p ${WORKSPACE}/pkgs/${platform}
+mv ${WORKSPACE}/${platform}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
+mv ${WORKSPACE}/${platform}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
+'''
+
+update_qemu = '''
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+'''
+
+pipeline {
+
+  // no top-level agent; agents must be declared for each stage
+  agent none
+
+  environment {
+    COUCHAUTH = credentials('couchdb_vm2_couchdb')
+    recipient = 'notifications@couchdb.apache.org'
+    COUCHDB_IO_LOG_DIR = '/tmp/couchjslogs'
+    // Following fix an issue with git <= 2.6.5 where no committer
+    // name or email are present for reflog, required for git clone
+    GIT_COMMITTER_NAME = 'Jenkins User'
+    GIT_COMMITTER_EMAIL = 'couchdb@apache.org'
+    // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+    // We need the jenkins user mapped inside of the image
+    // npm config cache below deals with /home/jenkins not mapping correctly
+    // inside the image
+    DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+  }
+
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
+    // This fails the build immediately if any parallel step fails
+    parallelsAlwaysFailFast()
+    preserveStashes(buildCount: 10)
+    timeout(time: 3, unit: 'HOURS')
+    timestamps()
+  }
+
+  stages {
+    stage('Build Release Tarball') {
+      agent {
+        docker {
+          image 'couchdbdev/debian-stretch-erlang-20.3.8.24-1:latest'
+          args "${DOCKER_ARGS}"
+          alwaysPull true
+        }
+      }
+      options {
+        timeout(time: 15, unit: "MINUTES")
+      }
+      steps {
+        sh '''
+          set
+          rm -rf apache-couchdb-*
+          ./configure --with-curl
+          make dist
+          chmod -R a+w * .
+        '''
+      }
+      post {
+        success {
+          stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
+          archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true
+        }
+        cleanup {
+          // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894
+          sh 'rm -rf ${WORKSPACE}/*'
+        }
+      }
+    } // stage Build Release Tarball
+
+    // TODO Rework once Improved Docker Pipeline Engine is released
+    // https://issues.jenkins-ci.org/browse/JENKINS-47962
+    // https://issues.jenkins-ci.org/browse/JENKINS-48050
+
+    stage('Test and Package') {
+
+      options {
+        skipDefaultCheckout()
+        timeout(time: 90, unit: "MINUTES")
+      }
+
+      parallel {
+
+        stage('FreeBSD') {
+          agent {
+            label 'couchdb && freebsd'
+          }
+          steps {
+            // deleteDir is OK here because we're not inside of a Docker container!
+            deleteDir()
+            unstash 'tarball'
+            withEnv(['HOME='+pwd()]) {
+              sh '''
+                mkdir -p $COUCHDB_IO_LOG_DIR
+
+                # Build CouchDB from tarball & test
+                mkdir build
+                cd build
+                tar -xf $WORKSPACE/apache-couchdb-*.tar.gz
+                cd apache-couchdb-*
+                ./configure --with-curl
+                gmake check || (build-aux/logfile-uploader.py && false)
+
+                # No package build for FreeBSD at this time
+              '''
+            } // withEnv
+          } // steps
+          post {
+            always {
+              junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+            }
+            cleanup {
+              sh 'rm -rf $COUCHDB_IO_LOG_DIR'
+            }
+          } // post
+        } // stage FreeBSD
+
+        stage('CentOS 6') {
+          agent {
+            docker {
+              image 'couchdbdev/centos-6-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
+              alwaysPull true
+            }
+          }
+          environment {
+            platform = 'centos6'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('CentOS 7') {
+          agent {
+            docker {
+              image 'couchdbdev/centos-7-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
+              alwaysPull true
+            }
+          }
+          environment {
+            platform = 'centos7'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                unstash 'tarball'
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('CentOS 8') {
+          agent {
+            docker {
+              image 'couchdbdev/centos-8-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
+              alwaysPull true
+            }
+          }
+          environment {
+            platform = 'centos8'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                unstash 'tarball'
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('Ubuntu Xenial') {
+          agent {
+            docker {
+              image 'couchdbdev/ubuntu-xenial-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
+              alwaysPull true
+            }
+          }
+          environment {
+            platform = 'xenial'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('Ubuntu Bionic') {
+          agent {
+            docker {
+              image 'couchdbdev/ubuntu-bionic-erlang-20.3.8.24-1:latest'
+              alwaysPull true
+              args "${DOCKER_ARGS}"
+            }
+          }
+          environment {
+            platform = 'bionic'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('Debian Stretch') {
+          agent {
+            docker {
+              image 'couchdbdev/debian-stretch-erlang-20.3.8.24-1:latest'
+              alwaysPull true
+              args "${DOCKER_ARGS}"
+            }
+          }
+          environment {
+            platform = 'stretch'
+            sm_ver = '1.8.5'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+        stage('Debian Buster amd64') {
+          agent {
+            docker {
+              image 'couchdbdev/debian-buster-erlang-20.3.8.24-1:latest'
+              alwaysPull true
+              args "${DOCKER_ARGS}"
+            }
+          }
+          environment {
+            platform = 'buster'
+            sm_ver = '60'
+          }
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+              post {
+                always {
+                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+                }
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+
+	/*
+	 *
+	 * This is just taking too long to run. Right now, the khash tests are timing out
+	 * on the IBM servers:
+	 *
+         * [2019-12-31T20:58:48.704Z]   khash randomized test
+         * [2019-12-31T20:59:04.869Z]     khash_test:103: randomized_test_ (State matches dict implementation)...*timed out*
+	 * 
+	 * So, this is DISABLED until we get an actual arm builder machine.
+	 *
+	 * ppc64le is actually slower to emulate than arm, so we're not even going to try that.
+	 */
+
+/*
+        stage('Debian Buster arm64v8') {
+          // once we have an arm64v8 node again, can restore this to original form that is less ugly
+          // the process is convoluted to ensure we have the latest qemu static binaries on the node first
+          // before trying to run a foreign docker container type. Alternately ensuring the `update_qemu`
+          // container is run on every Jenkins agent *after every restart of the Docker daemon* would work.
+          agent {
+            any {
+            }
+          }
+          options {
+            timeout(time: 120, unit: "MINUTES")
+          }
+          environment {
+            platform = 'aarch64-debian-stretch'
+            sm_ver = '60'
+          }
+          stages {
+            stage('Install latest qemu binaries') {
+              steps {
+                sh( script: update_qemu )
+              }
+            }
+            stage('Pull latest docker image') {
+              steps {
+                sh "docker pull couchdbdev/arm64v8-debian-buster-erlang-20.3.8.24-1:latest"
+              }
+            }
+            stage('Build from tarball & test & packages') {
+              steps {
+                withDockerContainer(image: "couchdbdev/arm64v8-debian-buster-erlang-20.3.8.24-1:latest", args: "${DOCKER_ARGS}") {
+                  unstash 'tarball'
+                  withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) {
+                    sh( script: build_and_test )
+                    sh( script: make_packages )
+                    sh( script: cleanup_and_save )
+                  }
+                }
+              }
+              post {
+                always {
+*/
+//                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+/*
+                }
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+*/
+
+      } // parallel
+    } // stage "Test and Package"
+
+    stage('Publish') {
+
+      when {
+        expression { return env.BRANCH_NAME ==~ /master|2.*.x|3.*.x|4.*.x|jenkins-.*/ }
+      }
+
+      agent {
+        docker {
+          image 'couchdbdev/debian-buster-erlang-20.3.8.24-1:latest'
+          alwaysPull true
+          args "${DOCKER_ARGS}"
+        }
+      }
+      options {
+        skipDefaultCheckout()
+        timeout(time: 90, unit: "MINUTES")
+      }
+
+      steps {
+        withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-key', keyFileVariable: 'KEY')]) {
+          sh 'rm -rf ${WORKSPACE}/*'
+          unstash 'tarball'
+          unarchive mapping: ['pkgs/' : '.']
+  
+          echo 'Retrieving & cleaning current couchdb-vm2 tree...'
+          sh '''
+            rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@couchdb-vm2.apache.org:/var/www/html/$BRANCH_NAME . || mkdir -p $BRANCH_NAME
+            rm -rf $BRANCH_NAME/debian/* $BRANCH_NAME/el6/* $BRANCH_NAME/el7/*
+            mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el6 $BRANCH_NAME/el7 $BRANCH_NAME/source
+            rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@couchdb-vm2.apache.org:/var/www/html/js .
+          '''
+
+          echo 'Building Debian repo...'
+          sh '''
+            git clone https://github.com/apache/couchdb-pkg
+            cp js/debian-jessie/*.deb pkgs/jessie
+            reprepro -b couchdb-pkg/repo includedeb jessie pkgs/jessie/*.deb
+            cp js/debian-stretch/*.deb pkgs/stretch
+            reprepro -b couchdb-pkg/repo includedeb stretch pkgs/stretch/*.deb
+            cp js/ubuntu-xenial/*.deb pkgs/xenial
+            reprepro -b couchdb-pkg/repo includedeb xenial pkgs/xenial/*.deb
+            cp js/ubuntu-bionic/*.deb pkgs/bionic
+            reprepro -b couchdb-pkg/repo includedeb bionic pkgs/bionic/*.deb
+          '''
+
+          echo 'Building CentOS repos...'
+          sh '''
+            cp js/centos-6/*rpm pkgs/centos6
+            cp js/centos-7/*rpm pkgs/centos7
+            cd pkgs/centos6 && createrepo --database .
+            cd ../centos7 && createrepo --database .
+          '''
+
+          echo 'Building tree to upload...'
+          sh '''
+            mv couchdb-pkg/repo/pool $BRANCH_NAME/debian
+            mv couchdb-pkg/repo/dists $BRANCH_NAME/debian
+            mv pkgs/centos6/* $BRANCH_NAME/el6
+            mv pkgs/centos7/* $BRANCH_NAME/el7
+            mv apache-couchdb-*.tar.gz $BRANCH_NAME/source
+            cd $BRANCH_NAME/source
+            ls -1tr | head -n -10 | xargs -d '\n' rm -f --
+            cd ../..
+          '''
+
+          echo 'rsyncing tree to couchdb-vm2...'
+          sh '''
+            rsync -avz --delete -e "ssh -o StrictHostKeyChecking=no -i $KEY" $BRANCH_NAME jenkins@couchdb-vm2.apache.org:/var/www/html
+            rm -rf $BRANCH_NAME couchdb-pkg *.tar.gz
+          '''
+        } // withCredentials
+      } // steps
+    } // stage
+  } // stages
+
+  post {
+    success {
+      mail to: "${env.recipient}",
+        replyTo: "${env.recipient}",
+        subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}",
+        body: "Yay, we passed. ${env.RUN_DISPLAY_URL}"
+    }
+    unstable {
+      mail to: "${env.recipient}",
+        replyTo: "${env.recipient}",
+        subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}",
+        body: "Eep! Build is unstable... ${env.RUN_DISPLAY_URL}"
+    }
+    failure {
+      mail to: "${env.recipient}",
+        replyTo: "${env.recipient}",
+        subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}",
+        body: "Boo, we failed. ${env.RUN_DISPLAY_URL}"
+    }
+  }
+
+} // pipeline
diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr
new file mode 100644
index 000000000..c7cb68f71
--- /dev/null
+++ b/build-aux/Jenkinsfile.pr
@@ -0,0 +1,145 @@
+#!groovy
+//
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+build_and_test = '''
+mkdir -p ${COUCHDB_IO_LOG_DIR} ${ERLANG_VERSION}
+cd ${ERLANG_VERSION}
+rm -rf build
+mkdir build
+cd build
+tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
+cd apache-couchdb-*
+. /usr/local/kerl/${KERL_VER}/activate
+./configure --with-curl --spidermonkey-version 60
+make check || (build-aux/logfile-uploader.py && false)
+'''
+
+pipeline {
+
+  // no top-level agent; agents must be declared for each stage
+  agent none
+
+  environment {
+    COUCHAUTH = credentials('couchdb_vm2_couchdb')
+    recipient = 'notifications@couchdb.apache.org'
+    COUCHDB_IO_LOG_DIR = '/tmp/couchjslogs'
+    // Following fix an issue with git <= 2.6.5 where no committer
+    // name or email are present for reflog, required for git clone
+    GIT_COMMITTER_NAME = 'Jenkins User'
+    GIT_COMMITTER_EMAIL = 'couchdb@apache.org'
+    // Parameters for the matrix build
+    DOCKER_IMAGE = 'couchdbdev/debian-buster-erlang-all:latest'
+    // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+    // We need the jenkins user mapped inside of the image
+    // npm config cache below deals with /home/jenkins not mapping correctly
+    // inside the image
+    DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+    // Also be sure to change these values in the matrix below...
+    // see https://issues.jenkins-ci.org/browse/JENKINS-40986
+    LOW_ERLANG_VER = '20.3.8.11'
+    MID_ERLANG_VER = '20.3.8.24'
+    HIGH_ERLANG_VER = '22.2'
+  }
+
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
+    // This fails the build immediately if any parallel step fails
+    parallelsAlwaysFailFast()
+    preserveStashes(buildCount: 10)
+    timeout(time: 3, unit: 'HOURS')
+    timestamps()
+  }
+
+  stages {
+    stage('Build Release Tarball') {
+      agent {
+        docker {
+          image "${DOCKER_IMAGE}"
+          args "${DOCKER_ARGS}"
+          alwaysPull true
+        }
+      }
+      options {
+        timeout(time: 15, unit: "MINUTES")
+      }
+      steps {
+        sh '''
+          set
+          rm -rf apache-couchdb-*
+          . /usr/local/kerl/${LOW_ERLANG_VER}/activate
+          ./configure --with-curl
+          make dist
+          chmod -R a+w * .
+        '''
+      }
+      post {
+        success {
+          stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
+        }
+        cleanup {
+          // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894
+          sh 'rm -rf ${WORKSPACE}/*'
+        }
+      }
+    } // stage Build Release Tarball
+
+    // TODO Rework once Improved Docker Pipeline Engine is released
+    // https://issues.jenkins-ci.org/browse/JENKINS-47962
+    // https://issues.jenkins-ci.org/browse/JENKINS-48050
+
+    stage('Make Check') {
+
+      matrix {
+        axes {
+          axis {
+            name 'ERLANG_VERSION'
+            values "20.3.8.11", "20.3.8.24", "22.2"
+          }
+        }
+
+        stages {
+          stage('Build and Test') {
+            agent {
+              docker {
+                image "${DOCKER_IMAGE}"
+                args "${DOCKER_ARGS}"
+                alwaysPull true
+              }
+            }
+            environment {
+              KERL_VER = "${ERLANG_VERSION}"
+            }
+            options {
+              skipDefaultCheckout()
+              timeout(time: 90, unit: "MINUTES")
+            }
+            steps {
+              unstash 'tarball'
+              sh( script: build_and_test )
+            }
+            post {
+              always {
+                junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+              }
+              cleanup {
+                sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}'
+              }
+            }
+          } // stage
+        } // stages
+      } // matrix
+    } // stage "Make Check"
+  } // stages
+} // pipeline
diff --git a/build-aux/README.Jenkins b/build-aux/README.Jenkins
new file mode 100644
index 000000000..014fbaad8
--- /dev/null
+++ b/build-aux/README.Jenkins
@@ -0,0 +1,149 @@
+# Building with a dedicated Jenkins CI instance
+
+## History
+
+Around April 2019, once Travis CI performance became so abysmal that the
+development team couldn't tolerate it any longer. We decided to move off
+of it entirely and onto our own dedicated Jenkins instance. Performance
+was bad because:
+
+* The Travis CI VMs are severely underpowered, leading to many
+  unexpected test failures
+* The ASF wait queue for Travis was exceedingly long, with jobs
+  sometimes taking 30-45 minutes just to _launch_
+* Our test suites (all 3 of Erlang, JavaScript and Elixir) had
+  sporadic failures, requiring many re-launches of builds before
+  we could move forward
+
+We also had other reasons to want to move to a dedicated server:
+
+* Credentials on the main Jenkins instance could be reused by *anyone*
+  in the ASF
+* Credentials on the Travis CI setup should not contain credentials
+  for things such as building binaries, to ensure the best possible
+  security
+
+Other options we explored:
+
+* ASF BuildBot. This required a lot more work and couldn't guarantee
+  availability of sufficient workers.
+* Circle CI, GitHub CI, etc.: Lack of availability of alternate binary
+  platforms we've been asked to support (arm64v8, ppc64le) or alternate
+  OSes (OSX, FreeBSD, Windows), though there is some support for either
+  or both.
+
+## Cloudbees Core
+
+End of November 2019, ASF and CloudBees reached an agreement to allow
+the Foundation to use CloudBees Core to have a farm of managed Jenkins
+masters. This allows the ASF to give larger projects their own dedicated
+Jenkins master, which can be custom configured for the project. They can
+readily manage this farm of Jenkins masters centrally, including push
+updates to all masters and their plugins. Naturally, this also reduces
+contention, as well as providing increased security for project-level
+credentials.  CouchDB is the first project to use this setup, via
+https://ci-couchdb.apache.org/ (aka https://jenkins-cm1.apache.org/)
+
+Only members of the ASF LDAP group `couchdb-pmc` have write access to
+the Jenkins CI server, and all jobs are set to not trust changes to the
+Jenkinsfile from forked repos (see below).
+
+Further, IBM is sponsoring CouchDB with cloud-based worker nodes, for
+the project's exclusive use. Combined with the FreeBSD and OSX nodes the
+project already had, this will provide the necessary compute resources
+to meet the needs of the project for some time to come.
+
+# Jenkins configuration
+
+All jobs on the new Jenkins master are contained in a CouchDB folder.
+Credentials for the project are placed within this folder; this is a
+unique capability of CloudBees Core at this time.
+
+## Pull Request job configuration
+
+Blue Ocean link: https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FPullRequests/activity/
+
+To implement build-a-PR functionality in the same way that Travis
+performs builds, Jenkins offers the Multibranch Pipeline job. Here's how
+it's configured for CouchDB through the GUI:
+
+* Job name: PullRequests (jobs shouldn't have spaces in them, because
+  the job name is used for workspace path naming.)
+* Display Name: "Pull Requests"
+* Description: "This job builds all GitHub pull requests against
+  apache/couchdb."
+* Branch sources: Github
+  * Credentials: a GitHub API key from wohali. These credentials are
+    stored in the top-level Jenkins CouchDB folder on the server.
+    The API token credentials are `user:email` and `repo:status`.
+  * URL https://github.com/apache/couchdb
+  * Behaviors
+    * Discover branches: Exclude branches that are also filed as PRs
+    * Discover PRs from origin: Merging the PR with the current target
+      branch revision
+    * Discover PRs from works: Merging the PR with the current target
+      branch revision, trust Nobody [2]
+    * Advanced clone behaviours:
+      * Fetch tags
+      * Clear "Shallow clone" [1]
+    * Clean before checkout [1]
+    * Prune stale remote-tracking branches
+  * Property strategy: All branches get the same properties
+* Build Configuration
+  * Mode: by Jenkinsfile
+  * Script path: `build-aux/Jenkinsfile.pr`
+* Scan Repository Triggers
+  * Periodically if not other wise run
+  * Interval: 1 day (do not set this too high, GitHub has an API token
+  throttle that can cause issues!)
+* Orphaned Item Strategy
+  * Discard old items
+  * Days to keep old items: <blank>
+  * Max # of old items to keep: 10
+* Everything else set as defaults.
+
+[1]: https://issues.jenkins-ci.org/browse/JENKINS-44598 explains why we have the build set to Clean Before Checkout every time, and why clones are not shallow.
+
+[2]: https://issues.apache.org/jira/browse/INFRA-17449 explains why "Discover pull requests from forks/Trust" has been set to "Nobody."
+
+## `master` and release branch configuration
+
+Our old Jenkins job (formerly `/Jenkinsfile`) is now
+`build-aux/Jenkinsfile.full`. This builds CouchDB on `master`, all of
+our release branches (`2.*`, `3.*`, etc.) as well as `jenkins-*` for
+testing.
+
+Settings are as follows:
+
+* Job name: FullPlatformMatrix (jobs shouldn't have spaces in them, because
+  the job name is used for workspace path naming.)
+* Display Name: "Full Platform Builds"
+* Description: "This job builds on our master and release branches,
+  and builds packages on all."
+* Branch sources: Github
+  * Credentials: a GitHub API key from wohali. These credentials are
+    stored in the top-level Jenkins CouchDB folder on the server.
+    The API token credentials are `user:email` and `repo:status`.
+  * URL https://github.com/apache/couchdb
+  * Behaviors
+    * Discover branches: All branches
+    * Filter by name (with wildcards): Include: master 2.*.x 3.*.x 4.*.x jenkins-*
+    * Advanced clone behaviours:
+      * Fetch tags
+      * Clear "Shallow clone" [1]
+    * Clean before checkout [1]
+    * Prune stale remote-tracking branches
+  * Property strategy: All branches get the same properties
+* Build Configuration
+  * Mode: by Jenkinsfile
+  * Script path: `build-aux/Jenkinsfile.pull`
+* Scan Repository Triggers
+  * none
+* Orphaned Item Strategy
+  * Discard old items
+  * Days to keep old items: <blank>
+  * Max # of old items to keep: 10
+* Everything else set as defaults.
+
+
+