Rework CI setup (#2367)

This commit:

* Removes Travis CI from the build (no more .travis.yml)
* Moves Jenkinsfile to `build-aux/Jenkinsfile.full` and updates the version
  of Erlang used in all steps to `20.3.8.24-1` (Erlang Solutions
  packages).
* Introduces a new `build-aux/Jenkinsfile.pr` that just builds
  CouchDB against the 3 most important versions of Erlang,
  intended for use when building PRs, using a special
  `debian-buster-erlang-all` image that has `kerl` builds of those 3
  versions of Erlang:
    * the oldest supported
    * the version we release our binary convenience builds with
    * the latest supported
* Builds against SpiderMonkey 60 on the platforms where it is available
  (currently, only Debian buster)
* Updated README file with new, dynamic Jenkins embeddable build status
  badge

5 files changed, 431 insertions, 188 deletions

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 2423fcaef..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-language: elixir
-elixir: 1.7.4
-os: linux
-dist: trusty
-
-otp_release:
-   - 22.0.5
-   - 21.2.3
-   - 20.3.8.5
-   - 19.3
-
-addons:
-  apt:
-    sources:
-      - deadsnakes
-    packages:
-    - build-essential
-    - curl
-    - libcurl4-openssl-dev
-    - libicu-dev
-    - libmozjs185-dev
-    - pkg-config
-    - python3.6
-    - python3.6-dev
-    - python3.6-venv
-    - python3-requests
-    - python3-sphinx
-#    - sphinx-rtd-theme
-    - help2man
-    - shunit2
-
-git:
-  depth: 10
-
-# logfile uploader uses requests
-cache:
-  - pip
-
-# logfile uploader credentials
-env:
-  global:
-    - secure: "UdA/gKIlyuXaW+hUgRx40t1TYjLCGxMqHvM5Uw7UbUH2dqEkgJiLfhZGchS1JVzl8M01VKZUUzS7v2nvRLiHZN1kvaw5kfq31VRoafUah8jfmvqNWZVdLovHl3aw5UX/HRt0RkbWbhdbdknTfh6+YinSZ+Nb54jCErMg9nabXtM="
-    - COUCHDB_IO_LOG_DIR=/tmp/couchjslogs
-
-# Enable this block if you want to build docs & fauxton too
-#node_js:
-#  - 6
-#before_script:
-#  - ./configure -c
-
-# Then comment this section out
-before_script:
-  - kerl list installations
-  - rm -rf /tmp/couchjslogs
-  - mkdir -p /tmp/couchjslogs
-  - ./configure -c --disable-docs --disable-fauxton
-  - python3.6 -m venv /tmp/.venv
-  - source /tmp/.venv/bin/activate
-  - pip install requests
-
-script:
-   - make check
-
-after_failure:
-  - build-aux/show-test-results.py
-  - build-aux/logfile-uploader.py
-
-# start a push build on master and release branches + PRs build on every branch
-# Avoid double build on PRs (See https://github.com/travis-ci/travis-ci/issues/1147)
-branches:
-  only:
-    - master
-    - /^\d+\.x\.x$/
-    - /^\d+\.\d+\.x$/
-
-# Re-enable once test suite is reliable
-#notifications:
-#  email: false
-#  irc:
-#    channels:
-#      "irc.freenode.org#couchdb-dev"
-#  on_success: change
-#  on_failure: always
-#  use_notice: true
-#  skip_join: true
-#  template:
-#    - %{repository_slug}/%{branch}: %{message} %{build_url}"
diff --git a/README.rst b/README.rst
index a03d6ec5a..47ce32e19 100644
--- a/README.rst
+++ b/README.rst
@@ -1,17 +1,12 @@
 Apache CouchDB README
 =====================
 
-+--------------------------------+------------+
-| Travis CI master build status  | |travisci| |
-+--------------------------------+------------+
-| Jenkins CI master build status | |jenkins|  |
-+--------------------------------+------------+
++-----+
+| |1| |
++-----+
 
-.. |travisci| image:: https://travis-ci.org/apache/couchdb.svg?branch=master
-    :target: https://travis-ci.org/apache/couchdb
-
-.. |jenkins| image:: https://builds.apache.org/job/CouchDB/job/master/badge/icon
-    :target: https://builds.apache.org/blue/organizations/jenkins/CouchDB/activity
+.. |1| image:: https://ci-couchdb.apache.org/job/jenkins-cm1/job/FullPlatformMatrix/job/master/badge/icon?subject=master%20build
+    :target: https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FFullPlatformMatrix/activity?branch=master
 
 Installation
 ------------
diff --git a/Jenkinsfile b/build-aux/Jenkinsfile.full
index b98b47a10..7087bba63 100644
--- a/Jenkinsfile
+++ b/build-aux/Jenkinsfile.full
@@ -13,20 +13,20 @@
 // License for the specific language governing permissions and limitations under
 // the License.
 
-// DRYing out the Jenkinsfile...
-
 build_and_test = '''
-mkdir -p ${COUCHDB_IO_LOG_DIR}
+mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}
+cd ${platform}
 rm -rf build
 mkdir build
 cd build
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
 cd apache-couchdb-*
-./configure --with-curl
+./configure --with-curl --spidermonkey-version ${sm_ver}
 make check || (build-aux/logfile-uploader.py && false)
 '''
 
 make_packages = '''
+cd ${platform}
 git clone https://github.com/apache/couchdb-pkg
 rm -rf couchdb
 mkdir couchdb
@@ -39,16 +39,18 @@ make ${platform} PLATFORM=${platform}
 cleanup_and_save = '''
 rm -rf ${WORKSPACE}/pkgs/${platform}
 mkdir -p ${WORKSPACE}/pkgs/${platform}
-mv ${WORKSPACE}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
-mv ${WORKSPACE}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
+mv ${WORKSPACE}/${platform}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
+mv ${WORKSPACE}/${platform}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
+'''
+
+update_qemu = '''
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 '''
 
 pipeline {
 
   // no top-level agent; agents must be declared for each stage
-  agent {
-    label 'ubuntu'
-  }
+  agent none
 
   environment {
     COUCHAUTH = credentials('couchdb_vm2_couchdb')
@@ -58,6 +60,11 @@ pipeline {
     // name or email are present for reflog, required for git clone
     GIT_COMMITTER_NAME = 'Jenkins User'
     GIT_COMMITTER_EMAIL = 'couchdb@apache.org'
+    // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+    // We need the jenkins user mapped inside of the image
+    // npm config cache below deals with /home/jenkins not mapping correctly
+    // inside the image
+    DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
   }
 
   options {
@@ -72,15 +79,10 @@ pipeline {
   stages {
     stage('Build Release Tarball') {
       agent {
-        // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
-        // We need the jenkins user mapped inside of the image
-        // npm config cache below is required because /home/jenkins doesn't
-        // ACTUALLY exist in the image
         docker {
-          image 'couchdbdev/debian-stretch-erlang-19.3.6:latest'
+          image 'couchdbdev/debian-stretch-erlang-20.3.8.24-1:latest'
+          args "${DOCKER_ARGS}"
           alwaysPull true
-          args '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
-          label 'ubuntu'
         }
       }
       options {
@@ -113,16 +115,17 @@ pipeline {
 
     stage('Test and Package') {
 
+      options {
+        skipDefaultCheckout()
+        timeout(time: 90, unit: "MINUTES")
+      }
+
       parallel {
 
         stage('FreeBSD') {
           agent {
             label 'couchdb && freebsd'
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           steps {
             // deleteDir is OK here because we're not inside of a Docker container!
             deleteDir()
@@ -156,19 +159,14 @@ pipeline {
         stage('CentOS 6') {
           agent {
             docker {
-              image 'couchdbdev/centos-6-erlang-19.3.6:latest'
+              image 'couchdbdev/centos-6-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
               alwaysPull true
-              label 'ubuntu'
-              // this keeps builds landing on the same host from clashing with each other
-              customWorkspace pwd() + '/centos6'
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
             platform = 'centos6'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -204,18 +202,14 @@ pipeline {
         stage('CentOS 7') {
           agent {
             docker {
-              image 'couchdbdev/centos-7-erlang-19.3.6:latest'
+              image 'couchdbdev/centos-7-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
               alwaysPull true
-              label 'ubuntu'
-              customWorkspace pwd() + '/centos7'
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
             platform = 'centos7'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -249,21 +243,17 @@ pipeline {
           } // post
         } // stage
 
-        stage('Ubuntu Xenial') {
+        stage('CentOS 8') {
           agent {
             docker {
-              image 'couchdbdev/ubuntu-xenial-erlang-19.3.6:latest'
+              image 'couchdbdev/centos-8-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
               alwaysPull true
-              label 'ubuntu'
-              customWorkspace pwd() + '/xenial'
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
-            platform = 'xenial'
+            platform = 'centos8'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -279,6 +269,7 @@ pipeline {
             }
             stage('Build CouchDB packages') {
               steps {
+                unstash 'tarball'
                 sh( script: make_packages )
                 sh( script: cleanup_and_save )
               }
@@ -296,21 +287,17 @@ pipeline {
           } // post
         } // stage
 
-        stage('Ubuntu Bionic') {
+        stage('Ubuntu Xenial') {
           agent {
             docker {
-              image 'couchdbdev/ubuntu-bionic-erlang-19.3.6:latest'
+              image 'couchdbdev/ubuntu-xenial-erlang-20.3.8.24-1:latest'
+              args "${DOCKER_ARGS}"
               alwaysPull true
-              label 'ubuntu'
-              customWorkspace pwd() + '/bionic'
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
-            platform = 'bionic'
+            platform = 'xenial'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -343,21 +330,17 @@ pipeline {
           } // post
         } // stage
 
-        stage('Debian Jessie') {
+        stage('Ubuntu Bionic') {
           agent {
             docker {
-              image 'couchdbdev/debian-jessie-erlang-19.3.6:latest'
+              image 'couchdbdev/ubuntu-bionic-erlang-20.3.8.24-1:latest'
               alwaysPull true
-              label 'ubuntu'
-              customWorkspace pwd() + '/jessie'
+              args "${DOCKER_ARGS}"
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
-            platform = 'jessie'
+            platform = 'bionic'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -390,21 +373,17 @@ pipeline {
           } // post
         } // stage
 
-        stage('Debian Stretch x86_64') {
+        stage('Debian Stretch') {
           agent {
             docker {
-              image 'couchdbdev/debian-stretch-erlang-19.3.6:latest'
+              image 'couchdbdev/debian-stretch-erlang-20.3.8.24-1:latest'
               alwaysPull true
-              label 'ubuntu'
-              customWorkspace pwd() + '/stretch'
+              args "${DOCKER_ARGS}"
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
             platform = 'stretch'
+            sm_ver = '1.8.5'
           }
           stages {
             stage('Build from tarball & test') {
@@ -437,29 +416,23 @@ pipeline {
           } // post
         } // stage
 
-        stage('Debian Stretch aarch64') {
+        stage('Debian Buster amd64') {
           agent {
             docker {
-              image 'couchdbdev/aarch64-debian-stretch-erlang-20.3.8.20:latest'
+              image 'couchdbdev/debian-buster-erlang-20.3.8.24-1:latest'
               alwaysPull true
-              label 'arm'
-              customWorkspace pwd() + '/arm'
+              args "${DOCKER_ARGS}"
             }
           }
-          options {
-            skipDefaultCheckout()
-            timeout(time: 90, unit: "MINUTES")
-          }
           environment {
-            platform = 'aarch64-debian-stretch'
+            platform = 'buster'
+            sm_ver = '60'
           }
           stages {
             stage('Build from tarball & test') {
               steps {
                 unstash 'tarball'
-                withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) {
-                  sh( script: build_and_test )
-                }
+                sh( script: build_and_test )
               }
               post {
                 always {
@@ -486,21 +459,92 @@ pipeline {
           } // post
         } // stage
 
+	/*
+	 *
+	 * This is just taking too long to run. Right now, the khash tests are timing out
+	 * on the IBM servers:
+	 *
+         * [2019-12-31T20:58:48.704Z]   khash randomized test
+         * [2019-12-31T20:59:04.869Z]     khash_test:103: randomized_test_ (State matches dict implementation)...*timed out*
+	 * 
+	 * So, this is DISABLED until we get an actual arm builder machine.
+	 *
+	 * ppc64le is actually slower to emulate than arm, so we're not even going to try that.
+	 */
+
+/*
+        stage('Debian Buster arm64v8') {
+          // once we have an arm64v8 node again, can restore this to original form that is less ugly
+          // the process is convoluted to ensure we have the latest qemu static binaries on the node first
+          // before trying to run a foreign docker container type. Alternately ensuring the `update_qemu`
+          // container is run on every Jenkins agent *after every restart of the Docker daemon* would work.
+          agent {
+            any {
+            }
+          }
+          options {
+            timeout(time: 120, unit: "MINUTES")
+          }
+          environment {
+            platform = 'aarch64-debian-stretch'
+            sm_ver = '60'
+          }
+          stages {
+            stage('Install latest qemu binaries') {
+              steps {
+                sh( script: update_qemu )
+              }
+            }
+            stage('Pull latest docker image') {
+              steps {
+                sh "docker pull couchdbdev/arm64v8-debian-buster-erlang-20.3.8.24-1:latest"
+              }
+            }
+            stage('Build from tarball & test & packages') {
+              steps {
+                withDockerContainer(image: "couchdbdev/arm64v8-debian-buster-erlang-20.3.8.24-1:latest", args: "${DOCKER_ARGS}") {
+                  unstash 'tarball'
+                  withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) {
+                    sh( script: build_and_test )
+                    sh( script: make_packages )
+                    sh( script: cleanup_and_save )
+                  }
+                }
+              }
+              post {
+                always {
+*/
+//                  junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+/*
+                }
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
+          post {
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
+            }
+          } // post
+        } // stage
+*/
+
       } // parallel
-    } // stage "Make Check"
+    } // stage "Test and Package"
 
     stage('Publish') {
 
       when {
-        expression { return env.BRANCH_NAME ==~ /master|2.0.x|2.1.x|jenkins-.*/ }
+        expression { return env.BRANCH_NAME ==~ /master|2.*.x|3.*.x|4.*.x|jenkins-.*/ }
       }
 
       agent {
         docker {
-          image 'couchdbdev/debian-stretch-erlang-19.3.6:latest'
+          image 'couchdbdev/debian-buster-erlang-20.3.8.24-1:latest'
           alwaysPull true
-          args '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
-          label 'ubuntu'
+          args "${DOCKER_ARGS}"
         }
       }
       options {
@@ -509,7 +553,7 @@ pipeline {
       }
 
       steps {
-        withCredentials([file(credentialsId: 'jenkins-key', variable: 'KEY')]) {
+        withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-key', keyFileVariable: 'KEY')]) {
           sh 'rm -rf ${WORKSPACE}/*'
           unstash 'tarball'
           unarchive mapping: ['pkgs/' : '.']
@@ -584,9 +628,6 @@ pipeline {
         subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}",
         body: "Boo, we failed. ${env.RUN_DISPLAY_URL}"
     }
-    cleanup {
-      sh 'rm -rf ${COUCHDB_IO_LOG_DIR}'
-    }
   }
 
 } // pipeline
diff --git a/build-aux/Jenkinsfile.pr b/build-aux/Jenkinsfile.pr
new file mode 100644
index 000000000..c7cb68f71
--- /dev/null
+++ b/build-aux/Jenkinsfile.pr
@@ -0,0 +1,145 @@
+#!groovy
+//
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+build_and_test = '''
+mkdir -p ${COUCHDB_IO_LOG_DIR} ${ERLANG_VERSION}
+cd ${ERLANG_VERSION}
+rm -rf build
+mkdir build
+cd build
+tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
+cd apache-couchdb-*
+. /usr/local/kerl/${KERL_VER}/activate
+./configure --with-curl --spidermonkey-version 60
+make check || (build-aux/logfile-uploader.py && false)
+'''
+
+pipeline {
+
+  // no top-level agent; agents must be declared for each stage
+  agent none
+
+  environment {
+    COUCHAUTH = credentials('couchdb_vm2_couchdb')
+    recipient = 'notifications@couchdb.apache.org'
+    COUCHDB_IO_LOG_DIR = '/tmp/couchjslogs'
+    // Following fix an issue with git <= 2.6.5 where no committer
+    // name or email are present for reflog, required for git clone
+    GIT_COMMITTER_NAME = 'Jenkins User'
+    GIT_COMMITTER_EMAIL = 'couchdb@apache.org'
+    // Parameters for the matrix build
+    DOCKER_IMAGE = 'couchdbdev/debian-buster-erlang-all:latest'
+    // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+    // We need the jenkins user mapped inside of the image
+    // npm config cache below deals with /home/jenkins not mapping correctly
+    // inside the image
+    DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+    // Also be sure to change these values in the matrix below...
+    // see https://issues.jenkins-ci.org/browse/JENKINS-40986
+    LOW_ERLANG_VER = '20.3.8.11'
+    MID_ERLANG_VER = '20.3.8.24'
+    HIGH_ERLANG_VER = '22.2'
+  }
+
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10'))
+    // This fails the build immediately if any parallel step fails
+    parallelsAlwaysFailFast()
+    preserveStashes(buildCount: 10)
+    timeout(time: 3, unit: 'HOURS')
+    timestamps()
+  }
+
+  stages {
+    stage('Build Release Tarball') {
+      agent {
+        docker {
+          image "${DOCKER_IMAGE}"
+          args "${DOCKER_ARGS}"
+          alwaysPull true
+        }
+      }
+      options {
+        timeout(time: 15, unit: "MINUTES")
+      }
+      steps {
+        sh '''
+          set
+          rm -rf apache-couchdb-*
+          . /usr/local/kerl/${LOW_ERLANG_VER}/activate
+          ./configure --with-curl
+          make dist
+          chmod -R a+w * .
+        '''
+      }
+      post {
+        success {
+          stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
+        }
+        cleanup {
+          // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894
+          sh 'rm -rf ${WORKSPACE}/*'
+        }
+      }
+    } // stage Build Release Tarball
+
+    // TODO Rework once Improved Docker Pipeline Engine is released
+    // https://issues.jenkins-ci.org/browse/JENKINS-47962
+    // https://issues.jenkins-ci.org/browse/JENKINS-48050
+
+    stage('Make Check') {
+
+      matrix {
+        axes {
+          axis {
+            name 'ERLANG_VERSION'
+            values "20.3.8.11", "20.3.8.24", "22.2"
+          }
+        }
+
+        stages {
+          stage('Build and Test') {
+            agent {
+              docker {
+                image "${DOCKER_IMAGE}"
+                args "${DOCKER_ARGS}"
+                alwaysPull true
+              }
+            }
+            environment {
+              KERL_VER = "${ERLANG_VERSION}"
+            }
+            options {
+              skipDefaultCheckout()
+              timeout(time: 90, unit: "MINUTES")
+            }
+            steps {
+              unstash 'tarball'
+              sh( script: build_and_test )
+            }
+            post {
+              always {
+                junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml'
+              }
+              cleanup {
+                sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}'
+              }
+            }
+          } // stage
+        } // stages
+      } // matrix
+    } // stage "Make Check"
+  } // stages
+} // pipeline
diff --git a/build-aux/README.Jenkins b/build-aux/README.Jenkins
new file mode 100644
index 000000000..014fbaad8
--- /dev/null
+++ b/build-aux/README.Jenkins
@@ -0,0 +1,149 @@
+# Building with a dedicated Jenkins CI instance
+
+## History
+
+Around April 2019, once Travis CI performance became so abysmal that the
+development team couldn't tolerate it any longer. We decided to move off
+of it entirely and onto our own dedicated Jenkins instance. Performance
+was bad because:
+
+* The Travis CI VMs are severely underpowered, leading to many
+  unexpected test failures
+* The ASF wait queue for Travis was exceedingly long, with jobs
+  sometimes taking 30-45 minutes just to _launch_
+* Our test suites (all 3 of Erlang, JavaScript and Elixir) had
+  sporadic failures, requiring many re-launches of builds before
+  we could move forward
+
+We also had other reasons to want to move to a dedicated server:
+
+* Credentials on the main Jenkins instance could be reused by *anyone*
+  in the ASF
+* Credentials on the Travis CI setup should not contain credentials
+  for things such as building binaries, to ensure the best possible
+  security
+
+Other options we explored:
+
+* ASF BuildBot. This required a lot more work and couldn't guarantee
+  availability of sufficient workers.
+* Circle CI, GitHub CI, etc.: Lack of availability of alternate binary
+  platforms we've been asked to support (arm64v8, ppc64le) or alternate
+  OSes (OSX, FreeBSD, Windows), though there is some support for either
+  or both.
+
+## Cloudbees Core
+
+End of November 2019, ASF and CloudBees reached an agreement to allow
+the Foundation to use CloudBees Core to have a farm of managed Jenkins
+masters. This allows the ASF to give larger projects their own dedicated
+Jenkins master, which can be custom configured for the project. They can
+readily manage this farm of Jenkins masters centrally, including push
+updates to all masters and their plugins. Naturally, this also reduces
+contention, as well as providing increased security for project-level
+credentials.  CouchDB is the first project to use this setup, via
+https://ci-couchdb.apache.org/ (aka https://jenkins-cm1.apache.org/)
+
+Only members of the ASF LDAP group `couchdb-pmc` have write access to
+the Jenkins CI server, and all jobs are set to not trust changes to the
+Jenkinsfile from forked repos (see below).
+
+Further, IBM is sponsoring CouchDB with cloud-based worker nodes, for
+the project's exclusive use. Combined with the FreeBSD and OSX nodes the
+project already had, this will provide the necessary compute resources
+to meet the needs of the project for some time to come.
+
+# Jenkins configuration
+
+All jobs on the new Jenkins master are contained in a CouchDB folder.
+Credentials for the project are placed within this folder; this is a
+unique capability of CloudBees Core at this time.
+
+## Pull Request job configuration
+
+Blue Ocean link: https://ci-couchdb.apache.org/blue/organizations/jenkins/jenkins-cm1%2FPullRequests/activity/
+
+To implement build-a-PR functionality in the same way that Travis
+performs builds, Jenkins offers the Multibranch Pipeline job. Here's how
+it's configured for CouchDB through the GUI:
+
+* Job name: PullRequests (jobs shouldn't have spaces in them, because
+  the job name is used for workspace path naming.)
+* Display Name: "Pull Requests"
+* Description: "This job builds all GitHub pull requests against
+  apache/couchdb."
+* Branch sources: Github
+  * Credentials: a GitHub API key from wohali. These credentials are
+    stored in the top-level Jenkins CouchDB folder on the server.
+    The API token credentials are `user:email` and `repo:status`.
+  * URL https://github.com/apache/couchdb
+  * Behaviors
+    * Discover branches: Exclude branches that are also filed as PRs
+    * Discover PRs from origin: Merging the PR with the current target
+      branch revision
+    * Discover PRs from works: Merging the PR with the current target
+      branch revision, trust Nobody [2]
+    * Advanced clone behaviours:
+      * Fetch tags
+      * Clear "Shallow clone" [1]
+    * Clean before checkout [1]
+    * Prune stale remote-tracking branches
+  * Property strategy: All branches get the same properties
+* Build Configuration
+  * Mode: by Jenkinsfile
+  * Script path: `build-aux/Jenkinsfile.pr`
+* Scan Repository Triggers
+  * Periodically if not other wise run
+  * Interval: 1 day (do not set this too high, GitHub has an API token
+  throttle that can cause issues!)
+* Orphaned Item Strategy
+  * Discard old items
+  * Days to keep old items: <blank>
+  * Max # of old items to keep: 10
+* Everything else set as defaults.
+
+[1]: https://issues.jenkins-ci.org/browse/JENKINS-44598 explains why we have the build set to Clean Before Checkout every time, and why clones are not shallow.
+
+[2]: https://issues.apache.org/jira/browse/INFRA-17449 explains why "Discover pull requests from forks/Trust" has been set to "Nobody."
+
+## `master` and release branch configuration
+
+Our old Jenkins job (formerly `/Jenkinsfile`) is now
+`build-aux/Jenkinsfile.full`. This builds CouchDB on `master`, all of
+our release branches (`2.*`, `3.*`, etc.) as well as `jenkins-*` for
+testing.
+
+Settings are as follows:
+
+* Job name: FullPlatformMatrix (jobs shouldn't have spaces in them, because
+  the job name is used for workspace path naming.)
+* Display Name: "Full Platform Builds"
+* Description: "This job builds on our master and release branches,
+  and builds packages on all."
+* Branch sources: Github
+  * Credentials: a GitHub API key from wohali. These credentials are
+    stored in the top-level Jenkins CouchDB folder on the server.
+    The API token credentials are `user:email` and `repo:status`.
+  * URL https://github.com/apache/couchdb
+  * Behaviors
+    * Discover branches: All branches
+    * Filter by name (with wildcards): Include: master 2.*.x 3.*.x 4.*.x jenkins-*
+    * Advanced clone behaviours:
+      * Fetch tags
+      * Clear "Shallow clone" [1]
+    * Clean before checkout [1]
+    * Prune stale remote-tracking branches
+  * Property strategy: All branches get the same properties
+* Build Configuration
+  * Mode: by Jenkinsfile
+  * Script path: `build-aux/Jenkinsfile.pull`
+* Scan Repository Triggers
+  * none
+* Orphaned Item Strategy
+  * Discard old items
+  * Days to keep old items: <blank>
+  * Max # of old items to keep: 10
+* Everything else set as defaults.
+
+
+