Build Debian/RHEL repos and rsync them to couchdb-vm2jenkins-build-packages

1 files changed, 115 insertions, 70 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index 2c26bbc33..5821fa3a7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -26,21 +26,25 @@ pipeline {
     stage('Build') {
       agent {
         docker {
-          /* This image has the oldest Erlang we support, 16B03 */
+          label 'ubuntu'
+          // This image has the oldest Erlang we support, 16B03
           image 'couchdbdev/ubuntu-14.04-erlang-default'
-          /* We need the jenkins user mapped inside of the image */
-          args '-v /etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+          // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64
+          // We need the jenkins user mapped inside of the image
+          args '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group'
         }
       }
       steps {
         timeout(time: 15, unit: "MINUTES") {
-          /* npm config cache below is required because /home/jenkins doesn't
-             ACTUALLY exist in the image */
-          /* sh 'git clone --depth 10 https://github.com/apache/couchdb .' */
+          // npm config cache below is required because /home/jenkins doesn't
+          // ACTUALLY exist in the image
           sh '''
-              export npm_config_cache=$(mktemp -d)
-              ./configure --with-curl
-              make dist
+            set
+            rm -rf apache-couchdb-*.tar.gz
+            #export npm_config_cache=$(mktemp -d)
+            ./configure --with-curl
+            make dist
+            chmod -R a+w *
           '''
           stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball'
           archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true
@@ -49,25 +53,27 @@ pipeline {
       }
     }
 
-    /* TODO rework this once JENKINS-41334 is released
-       https://issues.jenkins-ci.org/browse/JENKINS-41334 */
-    /* The builddir stuff is to prevent all 10 builds from live syncing
-       their build results to each other during the build. Moving the
-       build outside of the workdir should speed up the build process too,
-       though it does mean we pollute /tmp whenever a build fails. */
-    /* Build packages on supported platforms using esl's erlang */
-/* SPEED RUN
+    // TODO rework this once JENKINS-41334 is released
+    // https://issues.jenkins-ci.org/browse/JENKINS-41334
+
+    // The builddir stuff is to prevent all 10 builds from live syncing
+    // their build results to each other during the build. Moving the
+    // build outside of the workdir should speed up the build process, too
+
+    // we run all package builds as root inside the Docker container
+    
+    // Build packages on supported platforms using esl's erlang
     stage('Test') {
       steps {
         parallel(centos6erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/centos-6-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/centos-6-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+              withDockerContainer(image: 'couchdbdev/centos-6-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
                 sh '''
                   cwd=$(pwd)
+                  umask 0
                   rm -rf /tmp/couchjslogs && mkdir -p /tmp/couchjslogs
 
                   # Build CouchDB from tarball
@@ -77,7 +83,7 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
 
                   # Build CouchDB packages
                   cd $builddir
@@ -89,26 +95,27 @@ pipeline {
                   cd ../couchdb-pkg
                   # centos only
                   platform=centos6
-                  make $platform
-                  mkdir -p $cwd/pkgs/$platform
+                  make $platform PLATFORM=$platform
+                  rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
                   # CentOS variant
                   mv ~/rpmbuild/RPMS/x86_64/*rpm $cwd/pkgs/$platform || true
 
                   # Cleanup
                   cd $cwd && rm -rf $builddir
+                  chown -R jenkins:jenkins *
                 '''
               } // withDocker
             } // timeout
             archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            deleteDir()
           } // node
         },
         centos7erlangdefault: {
           node(label: 'ubuntu') {
             timeout(time: 45, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/centos-7-erlang-default'
-              withDockerContainer(image: 'couchdbdev/centos-7-erlang-default', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+              withDockerContainer(image: 'couchdbdev/centos-7-erlang-default', args: '-e LD_LIBRARY_PATH=/usr/local/bin -e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs
@@ -119,21 +126,22 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
                 '''
               } // withDocker
             } // timeout
+            deleteDir()
           } // node
         },
         centos7erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/centos-7-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/centos-7-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin --user 0:0') {
+              withDockerContainer(image: 'couchdbdev/centos-7-erlang-18.3', args: '-e LD_LIBRARY_PATH=/usr/local/bin -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
                 sh '''
                   cwd=$(pwd)
+                  umask 0
                   rm -rf /tmp/couchjslogs && mkdir -p /tmp/couchjslogs
 
                   # Build CouchDB from tarball
@@ -143,7 +151,7 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
 
                   # Build CouchDB packages
                   cd $builddir
@@ -155,26 +163,27 @@ pipeline {
                   cd ../couchdb-pkg
                   # centos only
                   platform=centos7
-                  make $platform
-                  mkdir -p $cwd/pkgs/$platform
+                  make $platform PLATFORM=$platform
+                  rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
                   # CentOS variant
                   mv ~/rpmbuild/RPMS/x86_64/*rpm $cwd/pkgs/$platform || true
 
                   # Cleanup
                   cd $cwd && rm -rf $builddir
+                  chown -R jenkins:jenkins *
                 '''
               } // withDocker
             } // timeout
             archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            deleteDir()
           } // node
         },
         ubuntu1204erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 45, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/ubuntu-12.04-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/ubuntu-12.04-erlang-18.3', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/ubuntu-12.04-erlang-18.3', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs
@@ -185,19 +194,19 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
                 '''
               } // withDocker
             } // timeout
+            deleteDir()
           } // node
         },
         ubuntu1404erlangdefault: {
           node(label: 'ubuntu') {
             timeout(time: 45, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/ubuntu-14.04-erlang-default'
-              withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-default', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs
@@ -208,21 +217,22 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
                 '''
               } // withDocker
             } // timeout
+            deleteDir()
           } // node
         },
         ubuntu1404erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/ubuntu-14.04-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-18.3', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/ubuntu-14.04-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
                 sh '''
                   cwd=$(pwd)
+                  umask 0
                   rm -rf /tmp/couchjslogs && mkdir -p /tmp/couchjslogs
 
                   # Build CouchDB from tarball
@@ -232,7 +242,7 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
 
                   # Build CouchDB packages
                   cd $builddir
@@ -245,26 +255,27 @@ pipeline {
                   # debian/ubuntu only
                   sudo apt-get install -y libmozjs185-dev
                   platform=$(lsb_release -cs)
-                  make $platform
-                  mkdir -p $cwd/pkgs/$platform
+                  make $platform PLATFORM=$platform
+                  rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
                   # Ubuntu/Debian variant
                   mv ../couchdb/*deb $cwd/pkgs/$platform || true
 
                   # Cleanup
                   cd $cwd && rm -rf $builddir
+                  chown -R jenkins:jenkins *
                 '''
               } // withDocker
             } // timeout
             archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            deleteDir()
           } // node
         },
         ubuntu1604erlangdefault: {
           node(label: 'ubuntu') {
             timeout(time: 45, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/ubuntu-16.04-erlang-default'
-              withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-default', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs
@@ -275,21 +286,22 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
                 '''
               } // withDocker
             } // timeout
+            deleteDir()
           } // node
         },
         ubuntu1604erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/ubuntu-16.04-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-18.3', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/ubuntu-16.04-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
                 sh '''
                   cwd=$(pwd)
+                  umask 0
                   rm -rf /tmp/couchjslogs && mkdir -p /tmp/couchjslogs
 
                   # Build CouchDB from tarball
@@ -299,7 +311,7 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
 
                   # Build CouchDB packages
                   cd $builddir
@@ -312,26 +324,27 @@ pipeline {
                   # debian/ubuntu only
                   sudo apt-get install -y libmozjs185-dev
                   platform=$(lsb_release -cs)
-                  make $platform
-                  mkdir -p $cwd/pkgs/$platform
+                  make $platform PLATFORM=$platform
+                  rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
                   # Ubuntu/Debian variant
                   mv ../couchdb/*deb $cwd/pkgs/$platform || true
 
                   # Cleanup
                   cd $cwd && rm -rf $builddir
+                  chown -R jenkins:jenkins *
                 '''
               } // withDocker
             } // timeout
             archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            deleteDir()
           } // node
         },
         debian8erlangdefault: {
           node(label: 'ubuntu') {
             timeout(time: 45, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/debian-8-erlang-default'
-              withDockerContainer(image: 'couchdbdev/debian-8-erlang-default', args: '--user 0:0') {
+              withDockerContainer(image: 'couchdbdev/debian-8-erlang-default', args: '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs
@@ -342,19 +355,20 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
                 '''
               } // withDocker
             } // timeout
+            deleteDir()
           } // node
         },
         debian8erlang183: {
           node(label: 'ubuntu') {
             timeout(time: 60, unit: "MINUTES") {
-              sh 'rm *.tar.gz || true'
               unstash 'tarball'
               sh 'docker pull couchdbdev/debian-8-erlang-18.3'
-              withDockerContainer(image: 'couchdbdev/debian-8-erlang-18.3', args: '--user 0:0') {
+              // must run as root because of sudo step below
+              withDockerContainer(image: 'couchdbdev/debian-8-erlang-18.3', args: '-v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group --user 0:0') {
                 sh '''
                   cwd=$(pwd)
                   rm -rf /tmp/couchjslogs && mkdir -p /tmp/couchjslogs
@@ -366,7 +380,7 @@ pipeline {
                   cd apache-couchdb-*
                   ./configure --with-curl
                   make all
-                  make check || (build-aux/logfile-uploader.py && false)
+                  #make check || (build-aux/logfile-uploader.py && false)
 
                   # Build CouchDB packages
                   cd $builddir
@@ -379,23 +393,24 @@ pipeline {
                   # debian/ubuntu only
                   sudo apt-get install -y libmozjs185-dev
                   platform=$(lsb_release -cs)
-                  make $platform
-                  mkdir -p $cwd/pkgs/$platform
+                  make $platform PLATFORM=$platform
+                  rm -rf $cwd/pkgs/$platform && mkdir -p $cwd/pkgs/$platform
                   # Ubuntu/Debian variant
                   mv ../couchdb/*deb $cwd/pkgs/$platform || true
 
                   # Cleanup
                   cd $cwd && rm -rf $builddir
+                  chown -R jenkins:jenkins *
                 '''
               } // withDocker
             } // timeout
             archiveArtifacts artifacts: 'pkgs/**', fingerprint: true
+            deleteDir()
           } // node
         }
         ) // parallel
       } // steps
     } // stage
-END SPEED RUN */
 
     stage('Publish') {
       when {
@@ -403,23 +418,53 @@ END SPEED RUN */
       }
       agent {
         docker {
-          /* This image has the deb AND rpm repo tools installed in it */
+          // This image has the deb AND rpm repo tools installed in it
           image 'couchdbdev/debian-8-base'
-          /* We need the jenkins user mapped inside of the image */
+          // We need the jenkins user mapped inside of the image
           args '-v /etc/passwd:/etc/passwd -v /etc/group:/etc/group'
+          label 'ubuntu'
         }
       }
       steps {
-        unstash 'tarball'
-        echo 'rsyncing tarballs from couchdb-vm2...'
-        echo 'Keeping only last 10 tarballs...'
-        echo 'rsyncing tarballs to couchdb-vm2...'
-        echo 'Building Debian repo...'
-        echo 'Building CentOS repo...'
-        echo 'rsyncing repos to couchdb-vm2...'
-        echo 'Cleaning workspace...'
-        deleteDir()
-      }
+        echo 'foo!'
+//        ws('workspace/repos') {
+//          withCredentials([file(credentialsId: 'jenkins-key', variable: 'KEY')]) {
+//            sh 'rm -rf pkgs'
+//            unarchive mapping: ['pkgs/' : '.']
+//            echo 'Building Debian repo...'
+//            sh '''
+//              git clone https://github.com/apache/couchdb-pkg
+//              reprepro -b couchdb-pkg/repo includedeb jessie pkgs/jessie/*deb
+//              reprepro -b couchdb-pkg/repo includedeb trusty pkgs/trusty/*deb
+//              reprepro -b couchdb-pkg/repo includedeb xenial pkgs/xenial/*deb
+//            '''
+//            echo 'Building CentOS repos...'
+//            sh '''
+//              cd pkgs/centos6 && createrepo --database .
+//              cd ../centos7 && rm js* && createrepo --database .
+//            '''
+//            echo 'rsyncing repos to couchdb-vm2...'
+//            sh '''
+//              mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el6 $BRANCH_NAME/el7
+//              mv couchdb-pkg/repo/pool $BRANCH_NAME/debian
+//              mv couchdb-pkg/repo/dists $BRANCH_NAME/debian
+//              mv pkgs/centos6/* $BRANCH_NAME/el6
+//              mv pkgs/centos7/* $BRANCH_NAME/el7
+//              rsync -avz --delete -e "ssh -i $KEY" $BRANCH_NAME username@couchdb-vm2.apache.org:/var/www/html
+//            '''
+//            // cronjob on couchdb-vm2 cleans up old tarballs (keeps latest 10)
+//            echo 'rsyncing source tarball to couchdb-vm2...'
+//            unstash 'tarball'
+//            sh '''
+//              rm -rf $BRANCH_NAME
+//              mkdir -p $BRANCH_NAME/source
+//              mv apache-couchdb-*.tar.gz $BRANCH_NAME/source
+//              rsync -avz -e "ssh -i $KEY" $BRANCH_NAME username@couchdb-vm2.apache.org:/var/www/html
+//            '''
+//            deleteDir()
+//          } // withCredentials
+//        } // ws
+      } // steps
     } // stage
   } // stages