diff options
author | Mathieu Mitchell <mmitchell@iweb.com> | 2016-08-22 15:43:39 -0400 |
---|---|---|
committer | Mathieu Mitchell <mmitchell@iweb.com> | 2016-08-26 09:06:24 -0400 |
commit | a0ca6ce157b8a84f282d8abbdd27c75f48d991ff (patch) | |
tree | a70c73bb03ee492db3d691f533df506b55209d72 | |
parent | 007654a9b667db8baa11e43358878a17b55b214f (diff) | |
download | ironic-python-agent-a0ca6ce157b8a84f282d8abbdd27c75f48d991ff.tar.gz |
Enforce upper-constraints when building ramdisks
Currently, building ramdisks installs ironic-python-agent without any
upper-constraints. This causes the package to be installed with newer,
untested dependencies.
This commits introduces a tool to generate a local upper-constraints
file based on predefined strategies (below). Additionally, the fallback
to the openstack/requirements uses the URL defined in tox.ini instead of
redefining it. This prevents having to keep track of two separate
variables when releasing.
upper-constraints lookup strategies (in order):
* UPPER_CONSTRAINTS_FILE points to a local file
* UPPER_CONSTRAINTS_FILE points to a URL
* /opt/stack/new/requirements/upper-constraints.txt
* upper-constraints.txt from openstack/requirements git repository
Partial-bug: #1616554
Change-Id: Ib5c0c57cafdb6ffd7456e61f3b1bb5fa57520e5a
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | Dockerfile | 5 | ||||
-rwxr-xr-x | imagebuild/common/extract_upper_constraints_from_tox_ini.sh | 9 | ||||
-rwxr-xr-x | imagebuild/common/generate_upper_constraints.sh | 79 | ||||
-rwxr-xr-x | imagebuild/coreos/docker_build.bash | 6 | ||||
-rwxr-xr-x | imagebuild/tinyipa/build-tinyipa.sh | 11 | ||||
-rwxr-xr-x | imagebuild/tinyipa/finalise-tinyipa.sh | 4 |
7 files changed, 108 insertions, 9 deletions
@@ -24,3 +24,6 @@ doc/build # release notes build releasenotes/build + +# upper-constraints handling for image builds +upper-constraints.txt @@ -39,11 +39,10 @@ RUN apt-mark manual python-minimal # Install requirements separately, because pip understands a git+https url # while setuptools doesn't RUN proxy.sh pip install --upgrade pip -# TODO(jroll) use upper-constraints here -RUN proxy.sh pip install --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt +RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir -r /tmp/ironic-python-agent/requirements.txt # This will succeed because all the dependencies were installed previously -RUN proxy.sh pip install --no-cache-dir /tmp/ironic-python-agent +RUN proxy.sh pip install -c /tmp/ironic-python-agent/upper-constraints.txt --no-cache-dir /tmp/ironic-python-agent # Remove no longer needed packages # NOTE(jroll) leave git to avoid strange apt issues in downstream Dockerfiles diff --git a/imagebuild/common/extract_upper_constraints_from_tox_ini.sh b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh new file mode 100755 index 00000000..86bd9973 --- /dev/null +++ b/imagebuild/common/extract_upper_constraints_from_tox_ini.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# NOTE(mmitchell): This extracts the URL defined as the default value for +# UPPER_CONSTRAINTS_FILE in tox.ini. This is used by image +# builders to avoid duplicating the default value in multiple +# scripts. This is specially done to leverage the release +# tools that automatically update the tox.ini when projects +# are released. +sed -n 's/^.*{env:UPPER_CONSTRAINTS_FILE\:\([^}]*\)}.*$/\1/p' $1 | head -n1 + diff --git a/imagebuild/common/generate_upper_constraints.sh b/imagebuild/common/generate_upper_constraints.sh new file mode 100755 index 00000000..cd61b784 --- /dev/null +++ b/imagebuild/common/generate_upper_constraints.sh @@ -0,0 +1,79 @@ +#!/bin/bash -eu + +SCRIPT_NAME=$(basename $0) +COMMON_ROOT=$(dirname $0) +DESTINATION="$1" +TOX_INI_UPPER_CONSTRAINT_URL="$(${COMMON_ROOT}/extract_upper_constraints_from_tox_ini.sh ${COMMON_ROOT}/../../tox.ini)" + +copy() { + local src=$1 + local destination=$2 + + if test -z "${src}"; then + return 1 + fi + + if test -e "${src}"; then + log "File '${src}' exists. Using as upper-constraints." + cp "${src}" "${destination}" + else + log "File '${src}' not found. Skipping local file strategy." + return 1 + fi + return 0 +} + +download() { + local url=$1 + local destination=$2 + + if test -z "${url}"; then + return 1 + else + log "Downloading from '${url}'" + curl ${url} -o "${destination}" + fi + return 0 +} + +log() { + echo "${SCRIPT_NAME}: ${@}" +} + +fail() { + log ${@} + exit 1 +} + +upper_constraints_is_not_null() { + test "${UPPER_CONSTRAINTS_FILE:-""}" != "" +} + +copy_uc() { + copy "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}" +} + +download_uc() { + download "${UPPER_CONSTRAINTS_FILE:-""}" "${DESTINATION}" +} + +copy_new_requirements_uc() { + copy "/opt/stack/new/requirements/upper-constraints.txt" "${DESTINATION}" +} + +download_from_tox_ini_url() { + log "tox.ini indicates '${TOX_INI_UPPER_CONSTRAINT_URL}' as fallback." + download "${TOX_INI_UPPER_CONSTRAINT_URL}" "${DESTINATION}" +} + +log "Generating local constraints file..." + +if upper_constraints_is_not_null; then + log "UPPER_CONSTRAINTS_FILE is defined as '${UPPER_CONSTRAINTS_FILE:-""}'" + copy_uc || download_uc || fail "Failed to copy or download file indicated in UPPER_CONSTRAINTS_FILE." +else + log "UPPER_CONSTRAINTS_FILE is not defined. Using fallback strategies." + + copy_new_requirements_uc || \ + download_from_tox_ini_url || fail "Failed to download upper-constraints.txt from '${TOX_INI_UPPER_CONSTRAINT_URL}'." +fi diff --git a/imagebuild/coreos/docker_build.bash b/imagebuild/coreos/docker_build.bash index 9386ef67..5f503d37 100755 --- a/imagebuild/coreos/docker_build.bash +++ b/imagebuild/coreos/docker_build.bash @@ -7,6 +7,7 @@ set -e OUTPUT_FILE="oem/container.tar.gz" +IPA_ROOT=$(readlink -f $(dirname $0)/../../) # If there's already a container.tar.gz, don't overwrite it -- instead, bail if [[ -e "${OUTPUT_FILE}" ]]; then @@ -15,7 +16,10 @@ if [[ -e "${OUTPUT_FILE}" ]]; then fi # Build the docker image -cd ../../ +# Everything from ${IPA_ROOT} will be available under /tmp/ironic-python-agent in Docker +cd ${IPA_ROOT} + +imagebuild/common/generate_upper_constraints.sh ${IPA_ROOT}/upper-constraints.txt # TODO(jlvilla): Once Docker 1.9 is widely deployed, switch to using the 'ARG' # command which was added in Docker 1.9. Currently Ubuntu 14.04 uses Docker diff --git a/imagebuild/tinyipa/build-tinyipa.sh b/imagebuild/tinyipa/build-tinyipa.sh index 0c605953..33dae7b4 100755 --- a/imagebuild/tinyipa/build-tinyipa.sh +++ b/imagebuild/tinyipa/build-tinyipa.sh @@ -65,6 +65,9 @@ cd ../.. rm -rf *.egg-info python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet cp requirements.txt $BUILDDIR/tmp/ipa-requirements.txt + +imagebuild/common/generate_upper_constraints.sh upper-constraints.txt +cp upper-constraints.txt $BUILDDIR/tmp/upper-constraints.txt cd $WORKDIR sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf @@ -88,10 +91,10 @@ done < $WORKDIR/build_files/buildreqs.lst # Build python wheels $CHROOT_CMD python /tmp/get-pip.py $CHROOT_CMD pip install pbr -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels setuptools -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels pip -$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt -$CHROOT_CMD pip wheel --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels setuptools +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels pip +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --wheel-dir /tmp/wheels -r /tmp/ipa-requirements.txt +$CHROOT_CMD pip wheel -c /tmp/upper-constraints.txt --no-index --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip --find-links=/tmp/wheels ironic-python-agent # Build tgt rm -rf $WORKDIR/build_files/tgt.tcz diff --git a/imagebuild/tinyipa/finalise-tinyipa.sh b/imagebuild/tinyipa/finalise-tinyipa.sh index 6de82232..dca127df 100755 --- a/imagebuild/tinyipa/finalise-tinyipa.sh +++ b/imagebuild/tinyipa/finalise-tinyipa.sh @@ -76,7 +76,9 @@ $CHROOT_CMD depmod -a `$WORKDIR/build_files/fakeuname -r` # If flag is set install the python now if $BUILD_AND_INSTALL_TINYIPA ; then - $CHROOT_CMD python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent + cp -a $BUILDDIR/tmp/upper-constraints.txt $FINALDIR/tmp/upper-constraints.txt + $CHROOT_CMD python /tmp/get-pip.py -c /tmp/upper-constraints.txt --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent + rm -rf $FINALDIR/tmp/upper-constraints.txt rm -rf $FINALDIR/tmp/wheelhouse rm -rf $FINALDIR/tmp/get-pip.py fi |