Dynamically generate parallel stages

1 files changed, 199 insertions, 197 deletions

diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full
index f1eb35e08..dcdf803d4 100644
--- a/build-aux/Jenkinsfile.full
+++ b/build-aux/Jenkinsfile.full
@@ -49,18 +49,189 @@ docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 
 ERLANG_VERSION = '24.2'
 
-platform_images = [
-  'centos7': "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}",
-  'centos8': "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}",
-  'bionic': "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}",
-  'focal': "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}",
-  'stretch': "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}",
-  'buster': "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}",
-  'bullseye': "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+meta = [
+  'centos7': [
+    name: 'CentOS 7',
+    spidermonkey_vsn: '1.8.5',
+    image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}"
+  ],
+
+  'centos8': [
+    name: 'CentOS 8',
+    spidermonkey_vsn: '60',
+    image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}"
+  ],
+
+  'bionic': [
+    name: 'Ubuntu 18.04',
+    spidermonkey_vsn: '1.8.5',
+    image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}"
+  ],
+
+  'focal': [
+    name: 'Ubuntu 20.04',
+    spidermonkey_vsn: '68',
+    image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}"
+  ],
+
+  'stretch': [
+    name: 'Debian 9',
+    spidermonkey_vsn: '1.8.5',
+    image: "apache/couchdbci-debian:stretch-erlang-${ERLANG_VERSION}"
+  ],
+
+  'buster': [
+    name: 'Debian 10',
+    spidermonkey_vsn: '60',
+    image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}"
+  ],
+
+  'bullseye': [
+    name: 'Debian 11',
+    spidermonkey_vsn: '78',
+    image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}"
+  ],
+
+  'freebsd': [
+    name: 'FreeBSD',
+    spidermonkey_vsn: '1.8.5',
+    gnu_make: 'gmake'
+  ],
+
+  'macos': [
+    name: 'macOS',
+    spidermonkey_vsn: '60',
+    gnu_make: 'make'
+  ]
 ]
 
 gnu_make = ['freebsd': 'gmake', 'macos': 'make']
 
+// Credit to https://stackoverflow.com/a/69222555 for this technique.
+// We use an initial stage to dynamically generate the list of parallel stages
+// to run afterwards. This keeps our Jenkinsfile DRY and gives us more
+// flexibility and better UX than the matrix pipeline.
+def parallelStagesMap
+
+def generateNativeStage(platform) {
+  return {
+    stage(meta[platform].name) {
+      agent {
+        label platform
+      }
+
+      options {
+        skipDefaultCheckout()
+        timeout(time: 90, unit: "MINUTES")
+      }
+
+      steps {
+        // deleteDir is OK here because we're not inside of a Docker container!
+        deleteDir()
+        unstash 'tarball'
+        withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+meta[platform].gnu_make]) {
+          sh( script: unpack, label: 'Unpack tarball' )
+          dir( "${platform}/build/couchdb" ) {
+            sh 'pwd'
+            sh 'ls -l'
+            sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+            sh '$MAKE'
+            sh '$MAKE eunit'
+            sh '$MAKE elixir-suite'
+            sh '$MAKE exunit'
+            sh '$MAKE mango-test'
+          }
+        }
+      }
+
+      post {
+        always {
+          junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+        }
+        failure {
+          dir( "${platform}/build/couchdb" ) {
+            sh 'make build-report'
+          }
+        }
+        cleanup {
+          sh 'killall -9 beam.smp || true'
+          sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
+        }
+      }
+    }
+  }
+}
+
+def generateContainerStage(platform) {
+  return {
+    stage(meta[platform].name) {
+      agent {
+        docker {
+          image meta[platform].image
+          label 'docker'
+          args "${DOCKER_ARGS}"
+          registryUrl 'https://docker.io/'
+          registryCredentialsId 'dockerhub_creds'
+        }
+      }
+
+      options {
+        skipDefaultCheckout()
+        timeout(time: 90, unit: "MINUTES")
+      }
+
+      stages {
+        stage('Build from tarball & test') {
+          steps {
+            sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
+            unstash 'tarball'
+            sh( script: unpack, label: 'Unpack tarball' )
+            dir( "${platform}/build/couchdb" ) {
+              sh 'pwd'
+              sh 'ls -l'
+              sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}"
+              sh 'make'
+              sh 'make eunit'
+              sh 'make elixir-suite'
+              sh 'make exunit'
+              sh 'make mango-test'
+            }
+          }
+          post {
+            always {
+              junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
+            }
+            failure {
+              dir( "${platform}/build/couchdb" ) {
+                sh 'make build-report'
+              }
+            }
+          }
+        }
+
+        stage('Build CouchDB packages') {
+          steps {
+            unstash 'tarball'
+            sh( script: make_packages, label: 'Build packages' )
+            sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
+          }
+          post {
+            success {
+              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            }
+          }
+        }
+      }
+
+      post {
+        cleanup {
+          sh 'rm -rf ${WORKSPACE}/*'
+        }
+      }
+    }
+  }
+}
+
 pipeline {
 
   // no top-level agent; agents must be declared for each stage
@@ -127,200 +298,31 @@ pipeline {
       }
     } // stage Build Release Tarball
 
-    // TODO Rework once Improved Docker Pipeline Engine is released
-    // https://issues.jenkins-ci.org/browse/JENKINS-47962
-    // https://issues.jenkins-ci.org/browse/JENKINS-48050
-
-    stage('Test and Package') {
-      matrix {
-        axes {
-          axis {
-            name 'platform'
-            values 'freebsd', 'macos', 'centos7', 'centos8', 'bionic', 'focal', 'stretch', 'buster', 'bullseye'
-          }
-          axis {
-            name 'spidermonkey'
-            values '1.8.5', '60', '68', '78'
-          } 
-        }
-        excludes {
-          exclude {
-            axis {
-              name 'platform'
-              values 'centos7', 'bionic', 'stretch', 'freebsd'
-            }
-            axis {
-              name 'spidermonkey'
-              notValues '1.8.5'
-            }
-          }
-          exclude {
-            axis {
-              name 'platform'
-              values 'centos8', 'buster', 'macos'
-            }
-            axis {
-              name 'spidermonkey'
-              notValues '60'
-            }
-          }
-          exclude {
-            axis {
-              name 'platform'
-              values 'focal'
-            }
-            axis {
-              name 'spidermonkey'
-              notValues '68'
-            }
-          }
-          exclude {
-            axis {
-              name 'platform'
-              values 'bullseye'
+    stage('Generate Test Matrix') {
+      agent {
+        label 'master'
+      }
+      steps {
+        script {
+          parallelStagesMap = meta.collectEntries { key, platform ->
+            if (platform.image) {
+              [key: generateContainerStage(key)]
             }
-            axis {
-              name 'spidermonkey'
-              notValues '78'
+            else {
+              [key: generateNativeStage(key)]
             }
           }
         }
+      }
+    }
 
-        stages {
-          stage('Other Build') {
-            when {
-              beforeAgent true
-              anyOf {
-                environment name: 'platform', value: 'macos'
-                environment name: 'platform', value: 'freebsd'
-              }
-            }
-
-            agent {
-              label env.platform
-            }
-
-            options {
-              skipDefaultCheckout()
-              timeout(time: 90, unit: "MINUTES")
-            }
-
-            steps {
-              // deleteDir is OK here because we're not inside of a Docker container!
-              deleteDir()
-              unstash 'tarball'
-              withEnv(['HOME='+pwd(), 'PATH+USRLOCAL=/usr/local/bin', 'MAKE='+gnu_make[env.platform]]) {
-                sh( script: unpack, label: 'Unpack tarball' )
-                dir( "${platform}/build/couchdb" ) {
-                  sh 'pwd'
-                  sh 'ls -l'
-                  sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
-                  sh '$MAKE'
-                  sh '$MAKE eunit'
-                  sh '$MAKE elixir-suite'
-                  sh '$MAKE exunit'
-                  sh '$MAKE mango-test'
-                }
-              }
-            }
-
-            post {
-              always {
-                junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
-              }
-              failure {
-                dir( "${platform}/build/couchdb" ) {
-                  sh 'make build-report'
-                }
-              }
-              cleanup {
-                sh 'killall -9 beam.smp || true'
-                sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR} || true'
-              }
-            } // post
-
-          } // non-containerized platform
-
-          stage('Container Build') {
-            when {
-              beforeAgent true
-              not {
-                anyOf {
-                  environment name: 'platform', value: 'macos'
-                  environment name: 'platform', value: 'freebsd'
-                }
-              }
-            }
-
-            agent {
-              docker {
-                image platform_images["${platform}"]
-                label 'docker'
-                args "${DOCKER_ARGS}"
-                registryUrl 'https://docker.io/'
-                registryCredentialsId 'dockerhub_creds'
-              }
-            }
-
-            options {
-              skipDefaultCheckout()
-              timeout(time: 90, unit: "MINUTES")
-            }
-
-            stages {
-              stage('Build from tarball & test') {
-                steps {
-                  sh( script: 'rm -rf apache-couchdb-*', label: 'Clean workspace' )
-                  unstash 'tarball'
-                  sh( script: unpack, label: 'Unpack tarball' )
-                  dir( "${platform}/build/couchdb" ) {
-                    sh 'pwd'
-                    sh 'ls -l'
-                    sh "./configure --skip-deps --spidermonkey-version ${spidermonkey}"
-                    sh 'make'
-                    sh 'make eunit'
-                    sh 'make elixir-suite'
-                    sh 'make exunit'
-                    sh 'make mango-test'
-                  }
-                }
-                post {
-                  always {
-                    junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml'
-                  }
-                  failure {
-                    dir( "${platform}/build/couchdb" ) {
-                      sh 'make build-report'
-                    }
-                  }
-                }
-              }
-
-              stage('Build CouchDB packages') {
-                steps {
-                  unstash 'tarball'
-                  sh( script: make_packages, label: 'Build packages' )
-                  sh( script: cleanup_and_save, label: 'Stage package artifacts for archival' )
-                }
-                post {
-                  success {
-                    archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
-                  }
-                }
-              }
-            }
-
-            post {
-              cleanup {
-                sh 'rm -rf ${WORKSPACE}/*'
-              }
-            } // post
-          } // containerized platform
-        } // stages
-
-      } // matrix
-
-    } // stage 'Test and Package'
+    stage('Test and Package') {
+      steps {
+        script {
+          parallel parallelStagesMap
+        }
+      }
+    }