diff options
Diffstat (limited to 'build-aux/Jenkinsfile.full')
| -rw-r--r-- | build-aux/Jenkinsfile.full | 466 |
1 files changed, 0 insertions, 466 deletions
diff --git a/build-aux/Jenkinsfile.full b/build-aux/Jenkinsfile.full deleted file mode 100644 index aeba9e57e..000000000 --- a/build-aux/Jenkinsfile.full +++ /dev/null @@ -1,466 +0,0 @@ -#!groovy -// -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not -// use this file except in compliance with the License. You may obtain a copy of -// the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations under -// the License. - -// Erlang version embedded in binary packages -ERLANG_VERSION = '23' - -// Erlang version used for rebar in release process. CouchDB will not build from -// the release tarball on Erlang versions older than this -MINIMUM_ERLANG_VERSION = '20' - -// We create parallel build / test / package stages for each OS using the metadata -// in this map. Adding a new OS should ideally only involve adding a new entry here. -meta = [ - 'centos7': [ - name: 'CentOS 7', - spidermonkey_vsn: '1.8.5', - image: "apache/couchdbci-centos:7-erlang-${ERLANG_VERSION}" - ], - - 'centos8': [ - name: 'CentOS 8', - spidermonkey_vsn: '60', - image: "apache/couchdbci-centos:8-erlang-${ERLANG_VERSION}" - ], - - 'bionic': [ - name: 'Ubuntu 18.04', - spidermonkey_vsn: '1.8.5', - image: "apache/couchdbci-ubuntu:bionic-erlang-${ERLANG_VERSION}" - ], - - 'focal': [ - name: 'Ubuntu 20.04', - spidermonkey_vsn: '68', - image: "apache/couchdbci-ubuntu:focal-erlang-${ERLANG_VERSION}" - ], - - 'buster': [ - name: 'Debian 10', - spidermonkey_vsn: '60', - image: "apache/couchdbci-debian:buster-erlang-${ERLANG_VERSION}" - ], - - 'bullseye-arm64': [ - name: 'Debian 11 ARM', - spidermonkey_vsn: '78', - image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}", - node_label: 'arm64v8' - ], - - 'bullseye-ppc64': [ - name: 'Debian 11 POWER', - spidermonkey_vsn: '78', - image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}", - node_label: 'ppc64le' - ], - - 'bullseye': [ - name: 'Debian 11', - spidermonkey_vsn: '78', - image: "apache/couchdbci-debian:bullseye-erlang-${ERLANG_VERSION}" - ], - - // Skip freebsd builds for now as adviced by node owner - // 'freebsd': [ - // name: 'FreeBSD', - // spidermonkey_vsn: '1.8.5', - // gnu_make: 'gmake' - // ], - - 'macos': [ - name: 'macOS', - spidermonkey_vsn: '60', - gnu_make: 'make' - ] -] - -// Credit to https://stackoverflow.com/a/69222555 for this technique. -// We can use the scripted pipeline syntax to dynamically generate stages, -// and inject them into a map that we pass to the `parallel` step in a script. -// While the scripting approach is very flexible, it's not able to use some -// functionality specific to Declarative Pipelines, like the `agent` and `post` -// directives, so you'll see alternatives like try-catch-finally used for flow -// control and the nested `node` and `docker` blocks in the container stage to -// configure the worker environment. - -// Returns a build stage suitable for non-containerized environments (currently -// macOS and FreeBSD). Coincidentally we do not currently support automated -// package generation on these platforms. This method in invoked when we create -// `parallelStagesMap` below. -def generateNativeStage(platform) { - return { - stage(platform) { - node(platform) { - timeout(time: 90, unit: "MINUTES") { - try { - // deleteDir is OK here because we're not inside of a Docker container! - deleteDir() - unstash 'tarball' - withEnv([ - 'HOME='+pwd(), - 'PATH+USRLOCAL=/usr/local/bin', - 'MAKE='+meta[platform].gnu_make - ]) { - sh( script: "mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}/build", label: 'Create build directories' ) - sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) - dir( "${platform}/build" ) { - sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}" - sh '$MAKE' - sh '$MAKE eunit' - sh '$MAKE elixir-suite' - sh '$MAKE exunit' - sh '$MAKE mango-test' - sh '$MAKE weatherreport-test' - } - } - } - catch (err) { - sh 'ls -l ${WORKSPACE}' - withEnv([ - 'HOME='+pwd(), - 'PATH+USRLOCAL=/usr/local/bin', - 'MAKE='+meta[platform].gnu_make - ]) { - dir( "${platform}/build" ) { - sh 'ls -l' - sh '${MAKE} build-report' - } - } - error("Build step failed with error: ${err.getMessage()}") - } - finally { - junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' - sh 'killall -9 beam.smp || true' - sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}' - } - } - } - } - } -} - -// Returns a build stage suitable for container-based deployments. This method -// is invoked when we create the `parallelStagesMap` in the pipeline below. -def generateContainerStage(platform) { - return { - // Important: the stage name here must match the parallelStagesMap key for the - // Jenkins UI to render the pipeline stages correctly. Don't ask why. -APK - stage(platform) { - node(meta[platform].get('node_label', 'docker')) { - docker.withRegistry('https://docker.io/', 'dockerhub_creds') { - docker.image(meta[platform].image).inside("${DOCKER_ARGS}") { - timeout(time: 90, unit: "MINUTES") { - stage("${meta[platform].name} - build & test") { - try { - sh( script: "rm -rf ${platform} apache-couchdb-*", label: 'Clean workspace' ) - unstash 'tarball' - sh( script: "mkdir -p ${COUCHDB_IO_LOG_DIR} ${platform}/build", label: 'Create build directories' ) - sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/build --strip-components=1", label: 'Unpack release' ) - dir( "${platform}/build" ) { - sh "./configure --skip-deps --spidermonkey-version ${meta[platform].spidermonkey_vsn}" - sh 'make' - sh 'make eunit' - sh 'make elixir-suite' - sh 'make exunit' - sh 'make mango-test' - sh 'make weatherreport-test' - } - } - catch (err) { - sh 'ls -l ${WORKSPACE}' - dir( "${platform}/build" ) { - sh 'ls -l' - sh 'make build-report' - } - error("Build step failed with error: ${err.getMessage()}") - } - finally { - junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' - sh 'rm -rf ${WORKSPACE}/* ${COUCHDB_IO_LOG_DIR}' - } - } - - stage("${meta[platform].name} - package") { - try { - unstash 'tarball' - sh( script: "mkdir -p ${platform}/couchdb", label: 'Create build directory' ) - sh( script: "tar -xf apache-couchdb-*.tar.gz -C ${platform}/couchdb", label: 'Unpack release' ) - sh( script: "cd ${platform} && git clone https://github.com/apache/couchdb-pkg", label: 'Clone packaging helper repo' ) - dir( "${platform}/couchdb-pkg" ) { - sh( script: 'make', label: 'Build packages' ) - } - sh( label: 'Stage package artifacts for archival', script: """ - rm -rf pkgs/${platform} - mkdir -p pkgs/${platform} - mv ${platform}/rpmbuild/RPMS/\$(arch)/*rpm pkgs/${platform} || true - mv ${platform}/couchdb/*.deb pkgs/${platform} || true - """ ) - archiveArtifacts artifacts: 'pkgs/**', fingerprint: true, onlyIfSuccessful: true - } - catch (err) { - sh 'ls -l ${WORKSPACE}' - error("Build step failed with error: ${err.getMessage()}") - } - finally { - sh 'rm -rf ${WORKSPACE}/*' - } - } - } - } - } - } - } - } -} - -// Finally we have the actual Pipeline. It's mostly a Declarative Pipeline, -// except for the 'Test and Package' stage where we use the `script` step as an -// "escape hatch" to dynamically generate a set of parallel stages to execute. -pipeline { - - // no top-level agent; agents must be declared for each stage - agent none - - environment { - COUCHAUTH = credentials('couchdb_vm2_couchdb') - COUCHDB_IO_LOG_DIR = '/tmp/couchjslogs' - // Following fix an issue with git <= 2.6.5 where no committer - // name or email are present for reflog, required for git clone - GIT_COMMITTER_NAME = 'Jenkins User' - GIT_COMMITTER_EMAIL = 'couchdb@apache.org' - // https://github.com/jenkins-infra/jenkins.io/blob/master/Jenkinsfile#64 - // We need the jenkins user mapped inside of the image - // npm config cache below deals with /home/jenkins not mapping correctly - // inside the image - DOCKER_ARGS = '-e npm_config_cache=npm-cache -e HOME=. -v=/etc/passwd:/etc/passwd -v /etc/group:/etc/group' - } - - options { - buildDiscarder(logRotator(numToKeepStr: '10', artifactNumToKeepStr: '10')) - preserveStashes(buildCount: 10) - timeout(time: 3, unit: 'HOURS') - timestamps() - } - - stages { - stage('Build Release Tarball') { - agent { - docker { - label 'docker' - image "apache/couchdbci-debian:erlang-${MINIMUM_ERLANG_VERSION}" - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - environment { - // TODO find a way to avoid setting this explicitly - spidermonkey = '60' - } - steps { - timeout(time: 15, unit: "MINUTES") { - sh (script: 'rm -rf apache-couchdb-*', label: 'Clean workspace of any previous release artifacts' ) - sh "./configure --spidermonkey-version ${spidermonkey}" - sh 'make erlfmt-check' - sh 'make elixir-check-formatted' - sh 'make dist' - } - } - post { - success { - stash includes: 'apache-couchdb-*.tar.gz', name: 'tarball' - archiveArtifacts artifacts: 'apache-couchdb-*.tar.gz', fingerprint: true - } - failure { - sh 'ls -l ${WORKSPACE}' - } - cleanup { - // UGH see https://issues.jenkins-ci.org/browse/JENKINS-41894 - sh 'rm -rf ${WORKSPACE}/*' - } - } - } // stage Build Release Tarball - - stage('Test and Package') { - steps { - script { - // Including failFast: true in map fails the build immediately if any parallel step fails - parallelStagesMap = meta.collectEntries( [failFast: false] ) { key, values -> - if (values.image) { - ["${key}": generateContainerStage(key)] - } - else { - ["${key}": generateNativeStage(key)] - } - } - parallel parallelStagesMap - } - } - } - - /* - * Example of how to do a qemu-based run, please leave here - */ - -/* - stage('Debian Buster arm64v8') { - // the process is convoluted to ensure we have the latest qemu static binaries on the node first - // before trying to run a foreign docker container type. Alternately ensuring the `update_qemu` - // container is run on every Jenkins agent *after every restart of the Docker daemon* would work. - agent { - any { - } - } - options { - timeout(time: 120, unit: "MINUTES") - } - environment { - platform = 'aarch64-debian-stretch' - sm_ver = '60' - } - stages { - stage('Install latest qemu binaries') { - steps { - sh( script: 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes' ) - } - } - stage('Pull latest docker image') { - steps { - sh "docker pull apache/couchdbci-debian:arm64v8-buster-erlang-${ERLANG_VERSION}" - } - } - stage('Build from tarball & test & packages') { - steps { - withDockerContainer(image: "apache/couchdbci-debian:arm64v8-buster-erlang-${ERLANG_VERSION}", args: "${DOCKER_ARGS}") { - unstash 'tarball' - withEnv(['MIX_HOME='+pwd(), 'HEX_HOME='+pwd()]) { - sh( script: build_and_test ) - sh( script: make_packages ) - sh( script: cleanup_and_save ) - } - } - } - post { - always { -*/ -// junit '**/.eunit/*.xml, **/_build/*/lib/couchdbtest/*.xml, **/src/mango/nosetests.xml, **/test/javascript/junit.xml' -/* - } - success { - archiveArtifacts artifacts: 'pkgs/**', fingerprint: true - } - } - } - } // stages - post { - cleanup { - sh 'rm -rf ${WORKSPACE}/*' - } - } // post - } // stage -*/ - - stage('Publish') { - - when { - expression { return env.BRANCH_NAME ==~ /main|2.*.x|3.*.x|4.*.x|jenkins-.*/ } - } - - agent { - docker { - image "apache/couchdbci-debian:erlang-${ERLANG_VERSION}" - label 'docker' - args "${DOCKER_ARGS}" - registryUrl 'https://docker.io/' - registryCredentialsId 'dockerhub_creds' - } - } - options { - skipDefaultCheckout() - timeout(time: 90, unit: "MINUTES") - } - - steps { - withCredentials([sshUserPrivateKey(credentialsId: 'jenkins-key', keyFileVariable: 'KEY')]) { - sh 'rm -rf ${WORKSPACE}/*' - unstash 'tarball' - unarchive mapping: ['pkgs/' : '.'] - - sh( label: 'Retrieve & clean current repo-nightly tree', script: ''' - rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@repo-nightly.couchdb.org:/var/www/html/$BRANCH_NAME . || mkdir -p $BRANCH_NAME - rm -rf $BRANCH_NAME/debian/* $BRANCH_NAME/el6 $BRANCH_NAME/el7/* $BRANCH_NAME/el8/* - mkdir -p $BRANCH_NAME/debian $BRANCH_NAME/el7 $BRANCH_NAME/el8 $BRANCH_NAME/source - rsync -avz -e "ssh -o StrictHostKeyChecking=no -i $KEY" jenkins@repo-nightly.couchdb.org:/var/www/html/js . - ''' ) - - sh( label: 'Build Debian repo', script: ''' - git clone https://github.com/apache/couchdb-pkg - cp js/ubuntu-bionic/*.deb pkgs/bionic - for plat in buster bullseye bionic focal - do - reprepro -b couchdb-pkg/repo includedeb $plat pkgs/$plat/*.deb - done - ''' ) - - sh( label: 'Build CentOS repos', script: ''' - cp js/centos-7/*rpm pkgs/centos7 - cp js/centos-8/*rpm pkgs/centos8 - cd pkgs/centos7 && createrepo_c --database . - cd ../centos8 && createrepo_c --database . - ''' ) - - sh( label: 'Build tree to upload', script: ''' - mv couchdb-pkg/repo/pool $BRANCH_NAME/debian - mv couchdb-pkg/repo/dists $BRANCH_NAME/debian - mv pkgs/centos7/* $BRANCH_NAME/el7 - mv pkgs/centos8/* $BRANCH_NAME/el8 - mv apache-couchdb-*.tar.gz $BRANCH_NAME/source - cd $BRANCH_NAME/source - ls -1tr | head -n -10 | xargs -d '\n' rm -f -- - cd ../.. - ''' ) - - sh( label: 'Sync tree back to repo-nightly', script: ''' - rsync -avz --delete -e "ssh -o StrictHostKeyChecking=no -i $KEY" $BRANCH_NAME jenkins@repo-nightly.couchdb.org:/var/www/html - rm -rf $BRANCH_NAME couchdb-pkg *.tar.gz - ''' ) - } // withCredentials - } // steps - } // stage - } // stages - - post { - success { - mail to: 'notifications@couchdb.apache.org', - replyTo: 'notifications@couchdb.apache.org', - subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", - body: "Yay, we passed. ${env.RUN_DISPLAY_URL}" - } - unstable { - mail to: 'notifications@couchdb.apache.org', - replyTo: 'notifications@couchdb.apache.org', - subject: "[Jenkins] SUCCESS: ${currentBuild.fullDisplayName}", - body: "Eep! Build is unstable... ${env.RUN_DISPLAY_URL}" - } - failure { - mail to: 'notifications@couchdb.apache.org', - replyTo: 'notifications@couchdb.apache.org', - subject: "[Jenkins] FAILURE: ${currentBuild.fullDisplayName}", - body: "Boo, we failed. ${env.RUN_DISPLAY_URL}" - } - } - -} // pipeline |