Enforce upper-constraints when building ramdisks

Currently, building ramdisks installs ironic-python-agent without any
upper-constraints. This causes the package to be installed with newer,
untested dependencies.

This commits introduces a tool to generate a local upper-constraints
file based on predefined strategies (below). Additionally, the fallback
to the openstack/requirements uses the URL defined in tox.ini instead of
redefining it. This prevents having to keep track of two separate
variables when releasing.

upper-constraints lookup strategies (in order):

  * UPPER_CONSTRAINTS_FILE points to a local file
  * UPPER_CONSTRAINTS_FILE points to a URL
  * /opt/stack/new/requirements/upper-constraints.txt
  * upper-constraints.txt from openstack/requirements git repository

Partial-bug: #1616554
Change-Id: Ib5c0c57cafdb6ffd7456e61f3b1bb5fa57520e5a
(cherry picked from commit a0ca6ce157b8a84f282d8abbdd27c75f48d991ff)

7 files changed, 108 insertions, 9 deletions

diff --git a/.gitignore b/.gitignore
index bfa6b137..fcc3c555 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,6 @@ doc/build
 
 # release notes build
 releasenotes/build
+
+# upper-constraints handling for image builds
+upper-constraints.txt
diff --git a/Dockerfile b/Dockerfile
index bfab2b41..c062a790 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -36,11 +36,10 @@ RUN apt-mark manual python-minimal
 # Install requirements separately, because pip understands a git+https url
 # while setuptools doesn't
 RUN proxy.sh pip install --upgrade pip
-# TODO(jroll) use upper-constraints here
-RUN proxy.sh pip install --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt
+RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt
 
 # This will succeed because all the dependencies were installed previously
-RUN proxy.sh pip install --no-cache-dir /tmp/ironic-python-agent
+RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir /tmp/ironic-python-agent
 
 # Remove no longer needed packages
 # NOTE(jroll) leave git to avoid strange apt issues in downstream Dockerfiles
diff --git a/imagebuild/common/extract_upper_constraints_from_tox_ini.sh b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh
new file mode 100755
index 00000000..86bd9973
--- /dev/null
+++ b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# NOTE(mmitchell): This extracts the URL defined as the default value for
+#                  UPPER_CONSTRAINTS_FILE in tox.ini. This is used by image
+#                  builders to avoid duplicating the default value in multiple
+#                  scripts. This is specially done to leverage the release
+#                  tools that automatically update the tox.ini when projects
+#                  are released.
+sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' $1 | head -n1
+
diff --git a/imagebuild/common/generate_upper_constraints.sh b/imagebuild/common/generate_upper_constraints.sh
new file mode 100755
index 00000000..cd61b784
--- /dev/null
+++ b/imagebuild/common/generate_upper_constraints.sh
@@ -0,0 +1,79 @@
+#!/bin/bash -eu
+
+SCRIPT_NAME=$(basename $0)
+COMMON_ROOT=$(dirname $0)
+DESTINATION="$1"
+TOX_INI_UPPER_CONSTRAINT_URL="$(${COMMON_ROOT}/extract_upper_constraints_from_tox_ini.sh ${COMMON_ROOT}/../../tox.ini)"
+
+copy() {
+  local src=$1
+  local destination=$2
+
+  if test -z "${src}"; then
+    return 1
+  fi
+
+  if test -e "${src}"; then
+    log "File '${src}' exists. Using as upper-constraints."
+    cp "${src}" "${destination}"
+  else
+    log "File '${src}' not found. Skipping local file strategy."
+    return 1
+  fi
+  return 0
+}
+
+download() {
+  local url=$1
+  local destination=$2
+
+  if test -z "${url}"; then
+     return 1
+  else
+    log "Downloading from '${url}'"
+    curl ${url} -o "${destination}"
+  fi
+  return 0
+}
+
+log() {
+  echo "${SCRIPT_NAME}: ${@}"
+}
+
+fail() {
+  log ${@}
+  exit 1
+}
+
+upper_constraints_is_not_null() {
+  test "${UPPER_CONSTRAINTS_FILE:-""}" != ""
+}
+
+copy_uc() {
+  copy "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
+}
+
+download_uc() {
+  download "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}"
+}
+
+copy_new_requirements_uc() {
+  copy "/opt/stack/new/requirements/upper-constraints.txt" "${DESTINATION}"
+}
+
+download_from_tox_ini_url() {
+  log "tox.ini indicates '${TOX_INI_UPPER_CONSTRAINT_URL}' as fallback."
+  download "${TOX_INI_UPPER_CONSTRAINT_URL}" "${DESTINATION}"
+}
+
+log "Generating local constraints file..."
+
+if upper_constraints_is_not_null; then
+  log "UPPER_CONSTRAINTS_FILE is defined as '${UPPER_CONSTRAINTS_FILE:-""}'"
+  copy_uc || download_uc || fail "Failed to copy or download file indicated in UPPER_CONSTRAINTS_FILE."
+else
+  log "UPPER_CONSTRAINTS_FILE is not defined. Using fallback strategies."
+
+  copy_new_requirements_uc || \
+  download_from_tox_ini_url || fail "Failed to download upper-constraints.txt from '${TOX_INI_UPPER_CONSTRAINT_URL}'."
+fi
diff --git a/imagebuild/coreos/docker_build.bash b/imagebuild/coreos/docker_build.bash
index 9386ef67..5f503d37 100755
--- a/imagebuild/coreos/docker_build.bash
+++ b/imagebuild/coreos/docker_build.bash
@@ -7,6 +7,7 @@
 set -e
 
 OUTPUT_FILE="oem/container.tar.gz"
+IPA_ROOT=$(readlink -f $(dirname $0)/../../)
 
 # If there's already a container.tar.gz, don't overwrite it -- instead, bail
 if [[ -e "${OUTPUT_FILE}" ]]; then
@@ -15,7 +16,10 @@ if [[ -e "${OUTPUT_FILE}" ]]; then
 fi
 
 # Build the docker image
-cd ../../
+# Everything from ${IPA_ROOT} will be available under /tmp/ironic-python-agent in Docker
+cd ${IPA_ROOT}
+
+imagebuild/common/generate_upper_constraints.sh ${IPA_ROOT}/upper-constraints.txt
 
 # TODO(jlvilla): Once Docker 1.9 is widely deployed, switch to using the 'ARG'
 # command which was added in Docker 1.9. Currently Ubuntu 14.04 uses Docker
diff --git a/imagebuild/tinyipa/build-tinyipa.sh b/imagebuild/tinyipa/build-tinyipa.sh
index ca6b2e0f..4e26dd11 100755
--- a/imagebuild/tinyipa/build-tinyipa.sh
+++ b/imagebuild/tinyipa/build-tinyipa.sh
@@ -56,6 +56,9 @@ cd ../..
 rm -rf *.egg-info
 python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
 cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt
+
+imagebuild/common/generate_upper_constraints.sh upper-constraints.txt
+cp upper-constraints.txt $BUILDDIR/tmp/upper-constraints.txt
 cd $WORKDIR
 
 sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf
@@ -79,10 +82,10 @@ done < $WORKDIR/build_files/buildreqs.lst
 # Build python wheels
 $CHROOT_CMD python /tmp/get-pip.py
 $CHROOT_CMD pip install pbr
-$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels setuptools
-$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels pip
-$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
-$CHROOT_CMD pip wheel --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
+$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels setuptools
+$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels pip
+$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt
+$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent
 
 # Build tgt
 rm -rf $WORKDIR/build_files/tgt.tcz
diff --git a/imagebuild/tinyipa/finalise-tinyipa.sh b/imagebuild/tinyipa/finalise-tinyipa.sh
index 7d70b708..dafe4682 100755
--- a/imagebuild/tinyipa/finalise-tinyipa.sh
+++ b/imagebuild/tinyipa/finalise-tinyipa.sh
@@ -69,7 +69,9 @@ $TC_CHROOT_CMD tce-load -ic /tmp/builtin/optional/qemu-utils.tcz
 
 # If flag is set install the python now
 if $BUILD_AND_INSTALL_TINYIPA ; then
-    $CHROOT_CMD python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
+    cp -a $BUILDDIR/tmp/upper-constraints.txt $FINALDIR/tmp/upper-constraints.txt
+    $CHROOT_CMD python /tmp/get-pip.py -c /tmp/upper-constraints.txt --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
+    rm -rf $FINALDIR/tmp/upper-constraints.txt
     rm -rf $FINALDIR/tmp/wheelhouse
     rm -rf $FINALDIR/tmp/get-pip.py
 fi