Refactor using sequential stages, in workspace

This work moves the builds back into the workspace, using a separate
sub-directory per platform to avoid clashes between builds caused by
JENKINS-57454. It also breaks out the steps into a pair of sequential
stages within each each parallel stage of the build, which gives us
better visibility into the progress of the build, and also sets us up
to capture test results and expose them directly via Jenkins UI for
faster problem determination.

1 files changed, 177 insertions, 72 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index 3c49b7672..b2deb8ef8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -14,35 +14,33 @@
 // the License.
 
 // DRYing out the Jenkinsfile...
-build_script = '''
-mkdir -p ${COUCHDB_IO_LOG_DIR}
 
-echo
-echo "Build CouchDB from tarball & test"
-builddir=$(mktemp -d)
-cd ${builddir}
+build_and_test = '''
+mkdir -p ${COUCHDB_IO_LOG_DIR}
+rm -rf build
+mkdir build
+cd build
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz
 cd apache-couchdb-*
 ./configure --with-curl
 make check || (build-aux/logfile-uploader.py && false)
+'''
 
-echo
-echo "Build CouchDB packages"
-cd ${builddir}
+make_packages = '''
 git clone https://github.com/apache/couchdb-pkg
+rm -rf couchdb
 mkdir couchdb
 cp ${WORKSPACE}/apache-couchdb-*.tar.gz couchdb
 tar -xf ${WORKSPACE}/apache-couchdb-*.tar.gz -C couchdb
 cd couchdb-pkg
 make ${platform} PLATFORM=${platform}
+'''
 
-echo
-echo "Cleanup & save for posterity"
+cleanup_and_save = '''
 rm -rf ${WORKSPACE}/pkgs/${platform}
 mkdir -p ${WORKSPACE}/pkgs/${platform}
-mv ../rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
-mv ../couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
-rm -rf ${builddir} ${COUCHDB_IO_LOG_DIR}
+mv ${WORKSPACE}/rpmbuild/RPMS/$(arch)/*rpm ${WORKSPACE}/pkgs/${platform} || true
+mv ${WORKSPACE}/couchdb/*.deb ${WORKSPACE}/pkgs/${platform} || true
 '''
 
 pipeline {
@@ -113,11 +111,7 @@ pipeline {
     // https://issues.jenkins-ci.org/browse/JENKINS-47962
     // https://issues.jenkins-ci.org/browse/JENKINS-48050
 
-    // The builddir stuff is to prevent all the builds from live syncing
-    // their build results to each other during the build, which ACTUALLY
-    // HAPPENS. Ugh.
-
-    stage('make check') {
+    stage('Test and Package') {
 
       parallel {
 
@@ -138,18 +132,22 @@ pipeline {
                 mkdir -p $COUCHDB_IO_LOG_DIR
 
                 # Build CouchDB from tarball & test
-                builddir=$(mktemp -d)
-                cd $builddir
+                mkdir build
+                cd build
                 tar -xf $WORKSPACE/apache-couchdb-*.tar.gz
                 cd apache-couchdb-*
                 ./configure --with-curl
                 gmake check || (build-aux/logfile-uploader.py && false)
 
                 # No package build for FreeBSD at this time
-                rm -rf $builddir $COUCHDB_IO_LOG_DIR
               '''
             } // withEnv
           } // steps
+          post {
+            cleanup {
+              sh 'rm -rf $COUCHDB_IO_LOG_DIR'
+            }
+          } // post
         } // stage FreeBSD
 
         stage('CentOS 6') {
@@ -158,6 +156,8 @@ pipeline {
               image 'couchdbdev/centos-6-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              // this keeps builds landing on the same host from clashing with each other
+              customWorkspace pwd() + '/centos6'
             }
           }
           options {
@@ -167,14 +167,28 @@ pipeline {
           environment {
             platform = 'centos6'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -185,6 +199,7 @@ pipeline {
               image 'couchdbdev/centos-7-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/centos7'
             }
           }
           options {
@@ -194,14 +209,29 @@ pipeline {
           environment {
             platform = 'centos7'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                unstash 'tarball'
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -212,6 +242,7 @@ pipeline {
               image 'couchdbdev/ubuntu-xenial-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/xenial'
             }
           }
           options {
@@ -221,14 +252,28 @@ pipeline {
           environment {
             platform = 'xenial'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -239,6 +284,7 @@ pipeline {
               image 'couchdbdev/ubuntu-bionic-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/bionic'
             }
           }
           options {
@@ -248,14 +294,28 @@ pipeline {
           environment {
             platform = 'bionic'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -266,6 +326,7 @@ pipeline {
               image 'couchdbdev/debian-jessie-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/jessie'
             }
           }
           options {
@@ -275,14 +336,28 @@ pipeline {
           environment {
             platform = 'jessie'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -293,6 +368,7 @@ pipeline {
               image 'couchdbdev/debian-stretch-erlang-19.3.6:latest'
               alwaysPull true
               label 'ubuntu'
+              customWorkspace pwd() + '/stretch'
             }
           }
           options {
@@ -302,14 +378,28 @@ pipeline {
           environment {
             platform = 'stretch'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -320,6 +410,7 @@ pipeline {
               image 'couchdbdev/aarch64-debian-stretch-erlang-20.3.8.20:latest'
               alwaysPull true
               label 'arm'
+              customWorkspace pwd() + '/arm'
             }
           }
           options {
@@ -329,14 +420,28 @@ pipeline {
           environment {
             platform = 'aarch64-debian-stretch'
           }
-          steps {
-            sh 'rm -f apache-couchdb-*.tar.gz'
-            unstash 'tarball'
-            sh( script: build_script )
-          } // steps
+          stages {
+            stage('Build from tarball & test') {
+              steps {
+                unstash 'tarball'
+                sh( script: build_and_test )
+              }
+            }
+            stage('Build CouchDB packages') {
+              steps {
+                sh( script: make_packages )
+                sh( script: cleanup_and_save )
+              }
+              post {
+                success {
+                  archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+                }
+              }
+            }
+          } // stages
           post {
-            success {
-              archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            cleanup {
+              sh 'rm -rf ${WORKSPACE}/*'
             }
           } // post
         } // stage
@@ -442,7 +547,7 @@ pipeline {
         body: "Boo, we failed. ${env.RUN_DISPLAY_URL}"
     }
     cleanup {
-      sh 'rm -rf ${WORKSPACE}/*'
+      sh 'rm -rf ${COUCHDB_IO_LOG_DIR}'
     }
   }