workflow for deb packages (#718)

6 files changed, 86 insertions, 148 deletions

diff --git a/daisy_workflows/README.md b/daisy_workflows/README.md
index e93f50f..3ae86b1 100644
--- a/daisy_workflows/README.md
+++ b/daisy_workflows/README.md
@@ -12,21 +12,16 @@ For more information on Daisy and how workflows work, refer to the
         -gcs_path YOUR_GCS_PATCH \
         -var:package_version=2.6.0 \
         -var:github_branch=development \
+        -var:output_path=YOUR_GS_BUCKET \
         build_debian.wf.json
 
-# Builds EL6 packages.
+# Builds EL packages.
 ./daisy -project YOUR_PROJECT \
         -zone ZONE \
         -gcs_path YOUR_GCS_PATCH \
-        -var:package_version=2.6.0 \
-        build_el6.wf.json
+        -var:output_path=YOUR_GS_BUCKET \
+        build_el.wf.json
 
-# Builds EL7 packages.
-./daisy -project YOUR_PROJECT \
-        -zone ZONE \
-        -gcs_path YOUR_GCS_PATCH \
-        -var:package_version=2.6.0 \
-        build_el7.wf.json
 ```
 
 # Variables
@@ -34,5 +29,3 @@ For more information on Daisy and how workflows work, refer to the
 * `output_path` Specify a different GCS path to save resulting packages to.
 * `github_repo` Specify a different github repo (for example a forked repo).
 * `github_branch` Specify a different github branch.
-* `package_version` The version of the package. This version has to match the
-  version of the python `setup.py` files, spec files, and Debian changelog.
diff --git a/daisy_workflows/build_deb_packages.sh b/daisy_workflows/build_deb_packages.sh
index 38c41d9..0eb1408 100644
--- a/daisy_workflows/build_deb_packages.sh
+++ b/daisy_workflows/build_deb_packages.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2017 Google Inc. All Rights Reserved.
+# Copyright 2018 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,49 +14,49 @@
 # limitations under the License.
 
 URL="http://metadata/computeMetadata/v1/instance/attributes"
+
 BRANCH="$(curl -f -H Metadata-Flavor:Google ${URL}/github_branch)"
 GIT_REPO="$(curl -f -H Metadata-Flavor:Google ${URL}/github_repo)"
-VERSION="$(curl -f -H Metadata-Flavor:Google ${URL}/package_version)"
 OUTPUT="$(curl -f -H Metadata-Flavor:Google ${URL}/output_path)"
-DEB_VER="$(cut -d "." -f 1 /etc/debian_version)"
 
-if [ -z ${OUTPUT} ]; then
+if [ -z $OUTPUT ]; then
   OUTPUT="$(curl -f -H Metadata-Flavor:Google ${URL}/daisy-outs-path)"
 fi
 
-# Install build dependencies.
-apt-get update
-apt-get -y install git debhelper devscripts dh-python dh-systemd \
-  python-all python-boto python-setuptools python-pytest python-mock \
-  python3-all python3-boto python3-setuptools python3-pytest python3-mock \
-  python3-distro \
-  libcurl4-openssl-dev libjson-c-dev libpam-dev build-essential
-if [ $? -ne 0 ]; then
-  echo "BuildFailed: Unable to install build dependencies."
+if [[ ! -e /etc/debian_version ]]; then
+  echo "BuildFailed: not a debian host!"
   exit 1
 fi
 
+workdir=$(pwd)
+mkdir output
+
+sudo apt-get install -y git
+
 # Clone the github repo.
-git clone ${GIT_REPO} -b ${BRANCH}
+git clone ${GIT_REPO} -b ${BRANCH} compute-image-packages
 if [ $? -ne 0 ]; then
   echo "BuildFailed: Unable to clone github repo ${GIT_REPO} and branch ${BRANCH}"
   exit 1
 fi
 
-# Create tar's for package builds.
-tar -czvf google-compute-image-packages_${VERSION}.orig.tar.gz --exclude .git compute-image-packages
-
-# Build the deb's
-pushd compute-image-packages
-debuild -us -uc
-if [ $? -ne 0 ]; then
-  echo "BuildFailed: debuild failed."
-  exit 1
-fi
-popd
+# Build packages.
+cd compute-image-packages/packages
+for package in *; do
+  pushd "$package"
+  ./packaging/setup_deb.sh
+  if [[ $? -ne 0 ]]; then
+    echo "BuildFailed: Unable to build $package"
+    exit 1
+  fi
+  find /tmp/debpackage \( -iname '*.deb' -o -iname '*.dsc' \) \
+    -exec mv '{}' "${workdir}/output/" \;
+  popd
+done
 
 # Copy the deb and dsc files to the output.
-gsutil cp *.dsc *.deb ${OUTPUT}/
+cd "${workdir}/output"
+gsutil cp *.dsc *.deb ${OUTPUT}
 if [ $? -ne 0 ]; then
   echo "BuildFailed: copying to ${OUTPUT} failed."
   exit 1
diff --git a/daisy_workflows/build_debian.wf.json b/daisy_workflows/build_debian.wf.json
index f3fad62..c47fdcc 100644
--- a/daisy_workflows/build_debian.wf.json
+++ b/daisy_workflows/build_debian.wf.json
@@ -2,7 +2,6 @@
   "Name": "build-debian-packages",
   "Vars": {
     "output_path": {
-      "Value": "${OUTSPATH}",
       "Description": "GCS output path for packages."
     },
     "github_repo": {
@@ -12,10 +11,6 @@
     "github_branch": {
       "Value": "master",
       "Description": "Github branch to build packages from."
-    },
-    "package_version": {
-      "Required": true,
-      "Description": "The package version to build."
     }
   },
   "Sources": {
@@ -43,8 +38,7 @@
           "Metadata": {
             "github_branch": "${github_branch}",
             "github_repo": "${github_repo}",
-            "output_path": "${output_path}",
-            "package_version": "${package_version}"
+            "output_path": "${output_path}"
           },
           "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
           "StartupScript": "build_deb_packages.sh"
diff --git a/daisy_workflows/build_el6.wf.json b/daisy_workflows/build_el.wf.json
index 07e8471..1f577f9 100644
--- a/daisy_workflows/build_el6.wf.json
+++ b/daisy_workflows/build_el.wf.json
@@ -1,5 +1,5 @@
 {
-  "Name": "build-el6-packages",
+  "Name": "build-el-packages",
   "Vars": {
     "output_path": {
       "Value": "${OUTSPATH}",
@@ -12,10 +12,6 @@
     "github_branch": {
       "Value": "master",
       "Description": "Github branch to build packages from."
-    },
-    "package_version": {
-      "Required": true,
-      "Description": "The package version to build."
     }
   },
   "Sources": {
@@ -25,6 +21,12 @@
     "setup-disk": {
       "CreateDisks": [
         {
+          "Name": "disk-el7-build",
+          "SourceImage": "projects/centos-cloud/global/images/family/centos-7",
+          "SizeGb": "10",
+          "Type": "pd-ssd"
+        },
+        {
           "Name": "disk-el6-build",
           "SourceImage": "projects/centos-cloud/global/images/family/centos-6",
           "SizeGb": "10",
@@ -35,6 +37,20 @@
     "package-build": {
       "CreateInstances": [
         {
+          "Name": "inst-el7-build",
+          "Disks": [
+            {"Source": "disk-el7-build"}
+          ],
+          "MachineType": "n1-standard-2",
+          "Metadata": {
+            "github_branch": "${github_branch}",
+            "github_repo": "${github_repo}",
+            "output_path": "${output_path}"
+          },
+          "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
+          "StartupScript": "build_el_packages.sh"
+        },
+        {
           "Name": "inst-el6-build",
           "Disks": [
             {"Source": "disk-el6-build"}
@@ -43,8 +59,7 @@
           "Metadata": {
             "github_branch": "${github_branch}",
             "github_repo": "${github_repo}",
-            "output_path": "${output_path}",
-            "package_version": "${package_version}"
+            "output_path": "${output_path}"
           },
           "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
           "StartupScript": "build_el_packages.sh"
@@ -54,6 +69,14 @@
     "wait-for-build": {
       "WaitForInstancesSignal": [
         {
+          "Name": "inst-el7-build",
+          "SerialOutput": {
+            "Port": 1,
+            "SuccessMatch": "BuildSuccess:",
+            "FailureMatch": "BuildFailed:"
+          }
+        },
+        {
           "Name": "inst-el6-build",
           "SerialOutput": {
             "Port": 1,
diff --git a/daisy_workflows/build_el7.wf.json b/daisy_workflows/build_el7.wf.json
deleted file mode 100644
index 10700b6..0000000
--- a/daisy_workflows/build_el7.wf.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-  "Name": "build-el7-packages",
-  "Vars": {
-    "output_path": {
-      "Value": "${OUTSPATH}",
-      "Description": "GCS output path for packages."
-    },
-    "github_repo": {
-      "Value": "https://github.com/GoogleCloudPlatform/compute-image-packages.git",
-      "Description": "Github repo to build packages from."
-    },
-    "github_branch": {
-      "Value": "master",
-      "Description": "Github branch to build packages from."
-    },
-    "package_version": {
-      "Required": true,
-      "Description": "The package version to build."
-    }
-  },
-  "Sources": {
-    "build_el_packages.sh": "./build_el_packages.sh"
-  },
-  "Steps": {
-    "setup-disk": {
-      "CreateDisks": [
-        {
-          "Name": "disk-el7-build",
-          "SourceImage": "projects/centos-cloud/global/images/family/centos-7",
-          "SizeGb": "10",
-          "Type": "pd-ssd"
-        }
-      ]
-    },
-    "package-build": {
-      "CreateInstances": [
-        {
-          "Name": "inst-el7-build",
-          "Disks": [
-            {"Source": "disk-el7-build"}
-          ],
-          "MachineType": "n1-standard-2",
-          "Metadata": {
-            "github_branch": "${github_branch}",
-            "github_repo": "${github_repo}",
-            "output_path": "${output_path}",
-            "package_version": "${package_version}"
-          },
-          "Scopes": ["https://www.googleapis.com/auth/devstorage.read_write"],
-          "StartupScript": "build_el_packages.sh"
-        }
-      ]
-    },
-    "wait-for-build": {
-      "WaitForInstancesSignal": [
-        {
-          "Name": "inst-el7-build",
-          "SerialOutput": {
-            "Port": 1,
-            "SuccessMatch": "BuildSuccess:",
-            "FailureMatch": "BuildFailed:"
-          }
-        }
-      ]
-    }
-  },
-  "Dependencies": {
-    "package-build": ["setup-disk"],
-    "wait-for-build": ["package-build"]
-  }
-}
diff --git a/daisy_workflows/build_el_packages.sh b/daisy_workflows/build_el_packages.sh
index a91a468..d02f1ed 100644
--- a/daisy_workflows/build_el_packages.sh
+++ b/daisy_workflows/build_el_packages.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright 2017 Google Inc. All Rights Reserved.
+# Copyright 2018 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,49 +14,48 @@
 # limitations under the License.
 
 URL="http://metadata/computeMetadata/v1/instance/attributes"
+
 BRANCH="$(curl -f -H Metadata-Flavor:Google ${URL}/github_branch)"
 GIT_REPO="$(curl -f -H Metadata-Flavor:Google ${URL}/github_repo)"
-VERSION="$(curl -f -H Metadata-Flavor:Google ${URL}/package_version)"
 OUTPUT="$(curl -f -H Metadata-Flavor:Google ${URL}/output_path)"
 
-if [ -z ${OUTPUT} ]; then
+if [ -z $OUTPUT ]; then
   OUTPUT="$(curl -f -H Metadata-Flavor:Google ${URL}/daisy-outs-path)"
 fi
 
-# Install build dependencies.
-yum -y install git rpmdevtools \
-  python2-devel python-setuptools python-boto \
-  make gcc-c++ libcurl-devel json-c json-c-devel pam-devel \
-  policycoreutils-python
+if [[ ! -e /etc/redhat-release ]]; then
+  echo "BuildFailed: not a RHEL host!"
+  exit 1
+fi
+
+workdir=$(pwd)
+mkdir output
+
+sudo yum install -y git
 
 # Clone the github repo.
-git clone ${GIT_REPO} -b ${BRANCH}
+git clone ${GIT_REPO} -b ${BRANCH} compute-image-packages
 if [ $? -ne 0 ]; then
   echo "BuildFailed: Unable to clone github repo ${GIT_REPO} and branch ${BRANCH}"
   exit 1
 fi
 
-# Create tar's for package builds.
-tar -czvf google-compute-engine_${VERSION}.orig.tar.gz --exclude .git compute-image-packages
-
-# Setup rpmbuild tree.
-for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do
-  mkdir -p /rpmbuild/$d
-done
-cp compute-image-packages/specs/*.spec /rpmbuild/SPECS/
-cp google-compute-engine_${VERSION}.orig.tar.gz /rpmbuild/SOURCES/
-
-# Build the RPM's
-for spec in $(ls /rpmbuild/SPECS/*.spec); do
- rpmbuild --define "_topdir /rpmbuild" -ba $spec
-  if [ $? -ne 0 ]; then
-    echo "BuildFailed: rpmbuild for $spec failed."
+# Build packages.
+cd compute-image-packages/packages
+for package in *; do
+  pushd "$package"
+  ./packaging/setup_rpm.sh
+  if [[ $? -ne 0 ]]; then
+    echo "BuildFailed: Unable to build $package"
     exit 1
   fi
+  find /tmp/rpmpackage -iname '*.rpm' -exec mv '{}' "${workdir}/output/" \;
+  popd
 done
 
-# Copy the rpm and srpms to the output.
-gsutil cp /rpmbuild/RPMS/*/*.rpm /rpmbuild/SRPMS/*.src.rpm ${OUTPUT}/
+# Copy the rpm files to the output.
+cd "${workdir}/output"
+gsutil cp *.rpm ${OUTPUT}
 if [ $? -ne 0 ]; then
   echo "BuildFailed: copying to ${OUTPUT} failed."
   exit 1