From 856c88812b1b7ec135eb339751c6f9e7581bbf25 Mon Sep 17 00:00:00 2001 From: Blair Hester Date: Wed, 15 Oct 2014 23:36:37 -0700 Subject: Major cleanup, fixes for UNOFFICIAL_RELEASE, get DEPLOY_ vars from release.mk, and fix issues preventing set -e use --- packaging/debs/apt-repository/Makefile | 9 +- packaging/debs/apt-repository/aptly_wrapper.sh | 116 ++++++++++++++++--------- 2 files changed, 77 insertions(+), 48 deletions(-) diff --git a/packaging/debs/apt-repository/Makefile b/packaging/debs/apt-repository/Makefile index 1ae30c22..7c639d79 100644 --- a/packaging/debs/apt-repository/Makefile +++ b/packaging/debs/apt-repository/Makefile @@ -9,11 +9,8 @@ CAN_HAS_APTLY=$(shell [ -f /usr/bin/aptly ] && echo true) ifeq ($(CAN_HAS_APTLY), true) debian_apt_repository: - ./aptly_wrapper.sh -o fetch - for FILE in ../Debian/*.deb ; do \ - ./aptly_wrapper.sh -o add -f $${FILE} ; \ - done - for FILE in ../Debian/*.dsc ; do \ + ./aptly_wrapper.sh -o fetch -d $(DEPLOY_HOST) -p $(DEPLOY_PATH) + for FILE in ../Debian/*.d?? ; do \ ./aptly_wrapper.sh -o add -f $${FILE} ; \ done ifeq ($(UNOFFICIAL_RELEASE), true) @@ -24,7 +21,7 @@ else sed -i '/gpgDisableSign/ s/true/false/' aptly.conf ./aptly_wrapper.sh -o publish -h $(HOME_ARG) -g $(SIGNING_KEY) endif - ./aptly_wrapper.sh -o push + ./aptly_wrapper.sh -o push -d $(DEPLOY_HOST) -p $(DEPLOY_PATH) else debian_apt_repository: @echo Not building APT repository as aptly could not be found diff --git a/packaging/debs/apt-repository/aptly_wrapper.sh b/packaging/debs/apt-repository/aptly_wrapper.sh index 71bde6d0..7341f39c 100755 --- a/packaging/debs/apt-repository/aptly_wrapper.sh +++ b/packaging/debs/apt-repository/aptly_wrapper.sh @@ -2,12 +2,9 @@ # aptly_wrapper.sh -# Commenting set -e, as it will cause the script to bail when handling -# certain errors, such as a non-exisiting remote aptly dir when performing -# a fetch. -#set -e +set -e -DEBUG=yes +DEBUG=no DATE=`date +%y%d-%H%M%S` @@ -25,12 +22,6 @@ REPO_COMPONENT=main REPO_LABEL="RabbitMQ Repository for Debian / Ubuntu etc" REPO_ORIGIN=RabbitMQ -# These would ideally come from the release.mk -# Need to determine actual dir layout for prod server where we can put -# aptly/ -DEPLOY_HOST=localhost -DEPLOY_PATH=/tmp/rabbitmq/extras/releases -DEPLOY_DEST=${DEPLOY_HOST}:${DEPLOY_PATH} # Rsync needs to preserve hardlinks which Aptly uses to map packages in the # metadata pool with the public pool @@ -51,17 +42,19 @@ APTLY="aptly -config=${APTLY_CONF}" Debug() { # Debug "this will appear when debug = yes only" if [ "$DEBUG" = "yes" ]; then - echo $* + #echo " $*" + printf "\t%s\n" "$*" fi } Usage() { echo "" - echo "`basename $0` -o [add|publish|fetch|push|diff] -h GPG_HOME -g GPG_KEY_ID -f \"pkg_full_name another_pkg_full_name\"" + echo "`basename $0` -o [add|publish|fetch|push|diff] -h GPG_HOME -g GPG_KEY_ID" + echo " -d DEPLOY_HOST -p DEPLOY_PATH -f \"pkg_full_name another_pkg_full_name\"" echo "" echo "For example:" echo " To pull the persistence files from the server:" - echo " ./`basename $0` -o fetch" + echo " ./`basename $0` -o fetch -d server.example.com -p /tmp/rabbitmq/extras/releases" echo "" echo " To add a package to a repo:" echo " ./`basename $0` -o add -f rabbitmq-server_3.3.5-1_all.deb" @@ -73,7 +66,10 @@ Usage() { echo " ./`basename $0` -o publish -h HOME=/tmp -g 57F929F7" echo "" echo " To push the persistence files and public repo data to server:" - echo " ./`basename $0` -o push" + echo " ./`basename $0` -o push -d server.example.com -p /tmp/rabbitmq/extras/releases" + echo "" + echo " To compare the remote persistence files to local files:" + echo " ./`basename $0` -o diff -d server.example.com -p /tmp/rabbitmq/extras/releases" echo "" exit } @@ -84,12 +80,14 @@ if [ "$#" -lt "2" ]; then fi -while getopts ":o:f:g:h:" opt; do +while getopts ":o:f:g:h:d:p:" opt; do case $opt in o ) OP=$OPTARG ;; f ) PKG_NAME_FULL=$OPTARG ;; g ) GPG_KEY=$OPTARG ;; h ) GPG_HOME=$OPTARG ;; + d ) DEPLOY_HOST=$OPTARG ;; + p ) DEPLOY_PATH=$OPTARG ;; \? ) Usage ;; esac done @@ -100,6 +98,7 @@ if [ -z "$OP" ]; then Usage fi + # If the GPG_HOME value is passed in then export it here for aptly to use # to find the gpg key. # This, for example, is expected to be HOME=/tmp if .gnupg/* is in /tmp/.gnupg. @@ -117,24 +116,60 @@ if [ ! -z "$GPG_KEY" ]; then export GPG_KEY_FLAG fi +# Use the same case statement for fetch and push +if [ "$OP" = "fetch" ]; then + SYNC_OPT=fetch + OP=sync +fi +if [ "$OP" = "push" ]; then + SYNC_OPT=push + OP=sync +fi + +# Override passed in value from release.mk for testing purposes +#DEPLOY_HOST= + +# Add the aptly dir to the path. The public repo files and internal aptly files +# reside in the aptly dir. +DEPLOY_PATH=${DEPLOY_PATH}/aptly +DEPLOY_DEST=${DEPLOY_HOST}:${DEPLOY_PATH} + case $OP in - fetch) - # Pull via ssh the Aptly persistence files from the server - - # Check existence of remote persistence files expected to be in the - # aptly directory. - APTLY_DIR_EXISTS=`ssh ${DEPLOY_HOST} "ls $DEPLOY_PATH 2>/dev/null | grep -c aptly"` - - # Use -gt to handle existence of more than one instance of an aptly - # named directory, such as aptly and aptly.old. - if [ "$APTLY_DIR_EXISTS" -gt "0" ]; then - echo "aptly dir does exist on $DEPLOY_HOST" - Debug "$RSYNC_CMD ${DEPLOY_DEST}/aptly/* aptly/" - $RSYNC_CMD ${DEPLOY_DEST}/aptly/* aptly/ - else - echo "aptly dir does NOT exist on $DEPLOY_HOST" - echo "Nothing to fetch. Add operation will automatically create" - echo "the repo." + sync) + if [ -z "$DEPLOY_HOST" ] || [ -z "$DEPLOY_PATH" ]; then + echo "Both -p DEPLOY_PATH and -d DEPLOY_DEST are required for -o $SYNC_OPT" + Usage + fi + # using rsync over ssh, check that we can ssh to dest host + DEPLOY_SSH_CONNECT=`ssh -q ${DEPLOY_HOST} "exit" && echo yes || echo no` + + if [ "$DEPLOY_SSH_CONNECT" = "no" ]; then + echo "Unable to access ${DEPLOY_HOST}" + echo "Using local repo files only." + else + DEPLOY_DIR_EXISTS=`ssh ${DEPLOY_HOST} "[ -d $DEPLOY_PATH ] && echo yes || echo no"` + if [ "$DEPLOY_DIR_EXISTS" = "no" ]; then + if [ "$SYNC_OPT" = "fetch" ]; then + echo "Aptly repo metadata files do NOT exist on $DEPLOY_HOST" + else + # push operation, attempt to create dest dir + # The set -e will prevent the rsync if the mkdir fails + ssh $DEPLOY_HOST "mkdir -p $DEPLOY_PATH" || echo "Failed to create $DEPLOY_PATH. Unable to push local repo to $DEPLOY_HOST." + Debug "$RSYNC_CMD aptly/ ${DEPLOY_DEST}/" + $RSYNC_CMD aptly/ ${DEPLOY_DEST}/ + fi + else + if [ "$SYNC_OPT" = "fetch" ]; then + echo "Performing Aptly metadata sync from $DEPLOY_HOST" +# removed unneeded /* + Debug "$RSYNC_CMD ${DEPLOY_DEST}/ aptly/" + $RSYNC_CMD ${DEPLOY_DEST}/ aptly/ + else + # push operation + Debug "$RSYNC_CMD aptly/ ${DEPLOY_DEST}/" + $RSYNC_CMD aptly/ ${DEPLOY_DEST}/ + fi + fi fi ;; @@ -142,8 +177,12 @@ case $OP in # perform a dry-run fetch of the remote repo with rsync so # itemize-changes can be used to compare the remote persistence # files to what is local. - Debug "$RSYNC_CMD_DIFF ${DEPLOY_DEST}/aptly/* aptly/" - $RSYNC_CMD_DIFF ${DEPLOY_DEST}/aptly/* aptly/ + if [ -z "$DEPLOY_HOST" ] || [ -z "$DEPLOY_PATH" ]; then + echo "Both -p DEPLOY_PATH and -d DEPLOY_DEST are required for -o diff" + Usage + fi + Debug "$RSYNC_CMD_DIFF ${DEPLOY_DEST}/ aptly/" + $RSYNC_CMD_DIFF ${DEPLOY_DEST}/ aptly/ ;; add) if [ -z "$PKG_NAME_FULL" ]; then @@ -243,11 +282,4 @@ case $OP in $GPG_KEY_FLAG $SNAPSHOT_NAME debian fi ;; - - push) - # Push via rsync over ssh the Aptly persistence files and public - # repo data. - Debug "$RSYNC_CMD aptly ${DEPLOY_DEST}/" - $RSYNC_CMD aptly ${DEPLOY_DEST}/ - ;; esac -- cgit v1.2.1