Fix integration script

Don't fail on inability to write new ChangeLog or AUTHORS files.
Ignore comments in requirements files

Change-Id: I0c017b493ca1ef3f1c770daf25d8778bd98f02ef
Fixes: bug #1192889

3 files changed, 139 insertions, 79 deletions

diff --git a/pbr/packaging.py b/pbr/packaging.py
index a64c2b0..f91842b 100644
--- a/pbr/packaging.py
+++ b/pbr/packaging.py
@@ -137,6 +137,10 @@ def parse_requirements(requirements_files=REQUIREMENTS_FILES):
 
     requirements = []
     for line in get_reqs_from_files(requirements_files):
+        # Ignore comments
+        if (not line.strip()) or line.startswith('#'):
+            continue
+
         # For the requirements list, we need to inject only the portion
         # after egg= so that distutils knows the package it's looking for
         # such as:
@@ -224,8 +228,12 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir,
     should_skip = get_boolean_option(option_dict, 'skip_changelog',
                                      'SKIP_WRITE_GIT_CHANGELOG')
     if not should_skip:
-        log.info('[pbr] Writing ChangeLog')
         new_changelog = os.path.join(dest_dir, 'ChangeLog')
+        # If there's already a ChangeLog and it's not writable, just use it
+        if (os.path.exists(new_changelog)
+                and not os.access(new_changelog, os.W_OK)):
+            return
+        log.info('[pbr] Writing ChangeLog')
         if git_dir is None:
             git_dir = _get_git_directory()
         if git_dir:
@@ -242,10 +250,14 @@ def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()):
     should_skip = get_boolean_option(option_dict, 'skip_authors',
                                      'SKIP_GENERATE_AUTHORS')
     if not should_skip:
-        log.info('[pbr] Generating AUTHORS')
-        jenkins_email = 'jenkins@review'
         old_authors = os.path.join(dest_dir, 'AUTHORS.in')
         new_authors = os.path.join(dest_dir, 'AUTHORS')
+        # If there's already a ChangeLog and it's not writable, just use it
+        if (os.path.exists(new_authors)
+                and not os.access(new_authors, os.W_OK)):
+            return
+        log.info('[pbr] Generating AUTHORS')
+        jenkins_email = 'jenkins@review'
         if git_dir is None:
             git_dir = _get_git_directory()
         if git_dir:
diff --git a/pbr/tests/test_setup.py b/pbr/tests/test_setup.py
index 4bf70cb..4fca00d 100644
--- a/pbr/tests/test_setup.py
+++ b/pbr/tests/test_setup.py
@@ -318,6 +318,12 @@ class ParseRequirementsTest(tests.BaseTestCase):
         actual = packaging.get_reqs_from_files([])
         self.assertEqual([], actual)
 
+    def test_parse_requirements_with_comments(self):
+        with open(self.tmp_file, 'w') as fh:
+            fh.write("# this is a comment\nfoobar\n# and another one\nfoobaz")
+        self.assertEqual(['foobar', 'foobaz'],
+                         packaging.parse_requirements([self.tmp_file]))
+
 
 class ParseDependencyLinksTest(tests.BaseTestCase):
 
diff --git a/tools/integration.sh b/tools/integration.sh
index a824224..f4be1d1 100644
--- a/tools/integration.sh
+++ b/tools/integration.sh
@@ -1,42 +1,76 @@
 #!/bin/bash -xe
 
+function mkvenv {
+    venv=$1
+    setuptools=$2
+    pip=$3
+
+    rm -rf $venv
+    if [ "$setuptools" == 'distribute' ] ; then
+        virtualenv --distribute $venv
+    elif [ "$setuptools" == 'setuptools' ] ; then
+        virtualenv $venv
+    else
+        virtualenv $venv
+        $venv/bin/pip install -v -U $setuptools
+    fi
+    $venv/bin/pip install $pip
+}
+
 # PROJECTS is a list of projects that we're testing
-PROJECTS=$1
+PROJECTS=$*
 
 # BASE should be a directory with a subdir called "new" and in that
 #      dir, there should be a git repository for every entry in PROJECTS
-BASE=$2
+BASE=${BASE:-/opt/stack}
 
 REPODIR=${REPODIR:-$BASE/new}
 
-tmpdir=`mkdtemp`
+# TODO: Figure out how to get this on to the box properly
+sudo apt-get install -y --force-yes libxml2-dev libxslt-dev libmysqlclient-dev libpq-dev libnspr4-dev pkg-config libsqlite3-dev libzmq-dev
+
+tmpdir=`mktemp -d`
+
+whoami=`whoami`
+tmpdownload=$tmpdir/download
+mkdir -p $tmpdownload
 
-tmpdownload=`mktemp -d`
-tmpvenv=$tmpdownload/venv
-virtualenv $tempvenv
-cd $REPODIR/pbr
-$tempvenv/bin/pip install -d $tmpdownload -r requirements.txt
-$tempvenv/bin/python setup.py sdist -d $tmpdownload
+pypidir=$tmpdir/pypi
+mkdir -p $pypidir
+
+jeepybvenv=$tmpdir/jeepyb
+
+rm -f ~/.pip/pip.conf ~/.pydistutils.cfg
+
+mkvenv $jeepybvenv distribute pip
+$jeepybvenv/bin/pip install -U git+https://review.openstack.org/p/openstack-infra/jeepyb.git
+
+cat <<EOF > $tmpdir/mirror.yaml
+cache-root: $tmpdownload
+
+mirrors:
+  - name: openstack
+    projects:
+      - https://github.com/openstack/requirements
+    output: $pypidir
+EOF
+
+pbrsdistdir=$tmpdir/pbrsdist
+git clone $REPODIR/pbr $pbrsdistdir
+cd $pbrsdistdir
+
+$jeepybvenv/bin/run-mirror -b remotes/origin/master --verbose -c $tmpdir/mirror.yaml --no-process
+
+$jeepybvenv/bin/pip install -i http://pypi.python.org/simple -d $tmpdownload/pip/openstack 'pip==1.0' 'setuptools>=0.7'
+$jeepybvenv/bin/pip install -i http://pypi.python.org/simple -d $tmpdownload/pip/openstack 'setuptools<0.7'
+
+$jeepybvenv/bin/pip install -i http://pypi.python.org/simple -d $tmpdownload/pip/openstack -r requirements.txt
+$jeepybvenv/bin/python setup.py sdist -d $tmpdownload/pip/openstack
+
+$jeepybvenv/bin/run-mirror -b remotes/origin/master --verbose -c $tmpdir/mirror.yaml --no-download
 
 # Make pypi thing
-pypidir=`mktemp -d`
 pypiurl=file://$pypidir
-echo "<html><body>" > $pypidir/index.html
-for fulltarball in $tmpdownload/*.tar.gz ; do
-    tarball=`basename $fulltarball`
-    name=`echo $tarball | sed 's/-[^-]*.tar.gz//'`
-    md5=`md5sum $fulltarball | awk '{print $1}'`
-    subdir=$pypidir/$name
-    mkdir -p $subdir
-    mv $fulltarball $subdir
-    echo "<a href='$tarball#md5=$md5'>$tarball</a>" >>$subdir/index.html
-    if ! grep $name $pypidir/index.html >/dev/null 2>&1 ; then
-        echo "<a href='$name'>$name</a>" >>$pypidir/index.html
-    fi
-done
-echo "</body></html>" >> $pypidir/index.html
-rm -rf $tmpdownload
-
 
 cat <<EOF > ~/.pydistutils.cfg
 [easy_install]
@@ -50,56 +84,64 @@ index-url = $pypiurl
 extra-index-url = http://pypi.openstack.org/openstack
 EOF
 
-function mkvenv {
-    venv=$1
-    setuptools=$2
-    pip=$3
+projectdir=$tmpdir/projects
+mkdir -p $projectdir
 
-    rm -rf $venv
-    if [ "$setuptools" == 'distribute' ] ; then
-        virtualenv --distribute $venv
-    elif [ "$setuptools" == 'setuptools' ] ; then
-        virtualenv $venv
-    else
-        virtualenv $venv
-        $venv/bin/pip install -v -U $setuptools
+for PROJECT in $PROJECTS ; do
+    SHORT_PROJECT=`basename $PROJECT`
+    if ! grep 'pbr' $REPODIR/$SHORT_PROJECT/requirements.txt >/dev/null 2>&1
+    then
+        # project doesn't use pbr
+        continue
     fi
-    $venv/bin/pip install $pip
-}
-
-# Test that pbr installs in different combinations
-for setuptools in 'setuptools' 'setuptools>=0.7' 'distribute' ; do
-    for pip in 'pip==1.0' 'pip>=1.3,<1.4' ; do
-        for PROJECT in $PROJECTS ; do
-            SHORT_PROJECT=`basename $PROJECT`
-            tmpdir=`mktemp -d`
-            tmpvenv=$tmpdir/venv
-
-            # Test pip installing
-            mkvenv $tmpvenv $setuptools $pip
-            cd $tmpdir
-            $tempvenv/bin/pip install git+file://$REPODIR/$SHORT_PROJECT
-
-            # Test python setup.py install
-            mkvenv $tmpvenv $setuptools $pip
-            cd $REPODIR/$SHORT_PROJECT
-            $tempvenv/bin/python setup.py install
-
-            # Because install will have caused eggs to be locally downloaded
-            # pbr and d2to1 can get excluded from being in the actual venv
-            # test that this did not happen
-            $tempvenv/bin/python -c 'import pkg_resources as p; import sys; pbr=p.working_set.find(p.Requirement.parse("pbr")) is None; sys.exit(pbr or 0)'
-
-            # Test that we can make a tarball from scratch
-            mkvenv $tmpvenv $setuptools $pip
-            $tempvenv/bin/python setup.py sdist
-
-            # Test that the tarball installs
-            cd $tmpdir
-            mkvenv $tmpvenv $setuptools $pip
-            $tempvenv/bin/pip install $REPODIR/$SHORT_PROJECT/dist/*tar.gz
-
-            rm -rf $tmpdir
-        done
-    done
+    if [ $SHORT_PROJECT = 'tempest' ]; then
+        # Tempest doesn't really install
+        continue
+    fi
+    shortprojectdir=$projectdir/$SHORT_PROJECT
+    git clone $REPODIR/$SHORT_PROJECT $shortprojectdir
+
+    sdistvenv=$tmpdir/sdist
+
+    # Test that we can make a tarball from scratch
+    mkvenv $sdistvenv distribute pip
+    cd $shortprojectdir
+    $sdistvenv/bin/python setup.py sdist
+
+    # Test that the tarball installs
+    cd $tmpdir
+    tarballvenv=$tmpdir/tarball
+    mkvenv $tarballvenv 'setuptools>=0.7' 'pip>=1.3,<1.4'
+    $tarballvenv/bin/pip install $shortprojectdir/dist/*tar.gz
+
+    # Test pip installing
+    pipvenv=$tmpdir/pip
+    mkvenv $pipvenv setuptools 'pip==1.0'
+    cd $tmpdir
+    echo $pipvenv/bin/pip install git+file://$REPODIR/$SHORT_PROJECT
+    $pipvenv/bin/pip install git+file://$REPODIR/$SHORT_PROJECT
+
+    # Test python setup.py install
+    installvenv=$tmpdir/install
+    mkvenv $installvenv setuptools pip
+    installprojectdir=$projectdir/install$SHORT_PROJECT
+    git clone $REPODIR/$SHORT_PROJECT $installprojectdir
+    cd $installprojectdir
+    $installvenv/bin/python setup.py install
+
+    # TODO(mordred): extend script to do a better job with the mirrir
+    # easy_install to a file:/// can't handle name case insensitivity
+    # Test python setup.py develop
+    # developvenv=$tmpdir/develop
+    # mkvenv $developvenv setuptools pip
+    # developprojectdir=$projectdir/develop$SHORT_PROJECT
+    # git clone $REPODIR/$SHORT_PROJECT $developprojectdir
+    # cd $developprojectdir
+    # $developvenv/bin/python setup.py develop
+
+    # TODO(mordred): need to implement egg filtering
+    # Because install will have caused eggs to be locally downloaded
+    # pbr and d2to1 can get excluded from being in the actual venv
+    # test that this did not happen
+    # $tempvenv/bin/python -c 'import pkg_resources as p; import sys; pbr=p.working_set.find(p.Requirement.parse("pbr")) is None; sys.exit(pbr or 0)'
 done