WIP: Resolve CI issues for stable-2.3. (#32605)

* Revert "If pip install requirements.txt fails, upgrade pip (#32399)"

This reverts commit 2fb4f547a915755e21be401bb7fd3f4817505e89.

* Avoid pep8 user warning in 1.7.1.
* Limit cryptography version for Windows tests.
* Upgrade pip in virtualenv for pip test.
* Upgrade pip in virtualenv for groupby_filter test.
* Upgrade pip in virtualenv for template_jinja2_latest test.

11 files changed, 33 insertions, 19 deletions

diff --git a/CODING_GUIDELINES.md b/CODING_GUIDELINES.md
index f11af31d78..60bf41ea65 100644
--- a/CODING_GUIDELINES.md
+++ b/CODING_GUIDELINES.md
@@ -14,15 +14,15 @@ Language
   * While not all components of Ansible must be in Python, core contributions to the Ansible repo must be written in Python.  This is to maximize the ability of everyone to contribute.
   * If you want to write non-Python ansible modules or inventory scripts, that's fine, but they are not going to get merged in most likely.  Sorry!!  
 
-PEP 8 and basic style checks
+PEP8 and basic style checks
 ===========================
 
-  * [PEP 8](https://www.python.org/dev/peps/pep-0008/) is a great Python style guide, which you should read.
-  * PEP 8 must not be strictly followed in all aspects, but most of it is good advice
+  * PEP8 is a great Python style guide, which you should read.
+  * PEP8 must not be strictly followed in all aspects, but most of it is good advice
   * In particular, we don't really care about line lengths.  Buy a bigger monitor!
-  * To run checks for things we care about, use [ansible-test](http://docs.ansible.com/ansible/dev_guide/testing_pep8.html#running-locally).
+  * To run checks for things we care about, run "make pep8"
   * Similarly, additional checks can be made with "make pyflakes"
-  * There is no need to submit code changes for PEP 8 and pyflakes fixes, as these break attribution history.  Project leadership will make these periodically.
+  * There is no need to submit code changes for pep8 and pyflake fixes, as these break attribution history.  Project leadership will make these periodically.
   * Do not submit pull requests that simply adjust whitespace in the code
 
 Testing
diff --git a/lib/ansible/modules/packaging/os/apt_repository.py b/lib/ansible/modules/packaging/os/apt_repository.py
index 6de23bb88c..b29dd69c49 100644
--- a/lib/ansible/modules/packaging/os/apt_repository.py
+++ b/lib/ansible/modules/packaging/os/apt_repository.py
@@ -210,7 +210,6 @@ class SourcesList(object):
             if filename is not None:
                 return filename
             return '_'.join(re.sub('[^a-zA-Z0-9]', ' ', s).split())
-
         def _strip_username_password(s):
             if '@' in s:
                 s = s.split('@', 1)
diff --git a/test/integration/targets/groupby_filter/runme.sh b/test/integration/targets/groupby_filter/runme.sh
index 8f9fce9b56..b058ce637b 100755
--- a/test/integration/targets/groupby_filter/runme.sh
+++ b/test/integration/targets/groupby_filter/runme.sh
@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2==2.9.4
 
 ansible-playbook -i ../../inventory test_jinja2_groupby.yml -v "$@"
diff --git a/test/integration/targets/pip/tasks/pip.yml b/test/integration/targets/pip/tasks/pip.yml
index b6df5ec297..c6d398015d 100644
--- a/test/integration/targets/pip/tasks/pip.yml
+++ b/test/integration/targets/pip/tasks/pip.yml
@@ -151,6 +151,15 @@
       - "not setuptools_check_mode.changed"
 
 
+# Upgrade pip in virtualenv
+- name: download get-pip.py
+  get_url:
+    url: https://bootstrap.pypa.io/get-pip.py
+    dest: "{{ output_dir }}"
+
+- name: install pip
+  command: "{{ output_dir }}/pipenv/bin/python {{ output_dir }}/get-pip.py"
+
 # Normal case
 - name: check for q package
   pip: name=q virtualenv={{ output_dir }}/pipenv state=present
diff --git a/test/integration/targets/template_jinja2_latest/runme.sh b/test/integration/targets/template_jinja2_latest/runme.sh
index 3c705e19a0..8704c5eac2 100755
--- a/test/integration/targets/template_jinja2_latest/runme.sh
+++ b/test/integration/targets/template_jinja2_latest/runme.sh
@@ -16,6 +16,9 @@ virtualenv --system-site-packages --python "${PYTHON}" "${MYTMPDIR}/jinja2"
 
 source "${MYTMPDIR}/jinja2/bin/activate"
 
+curl https://bootstrap.pypa.io/get-pip.py > "${MYTMPDIR}/get-pip.py"
+python "${MYTMPDIR}/get-pip.py"
+
 pip install -U jinja2
 
 ansible-playbook -i ../../inventory main.yml -e @../../integration_config.yml -v "$@"
diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py
index b760832945..848cb3adc6 100644
--- a/test/runner/lib/executor.py
+++ b/test/runner/lib/executor.py
@@ -156,11 +156,16 @@ def install_command_requirements(args):
 
     try:
         run_command(args, cmd)
-    except SubprocessError:
-        # Gundalow/Mattclay: Revisit logic here https://github.com/ansible/ansible/issues/32398
-        # If the install fails for any reason, upgrade pip and retry.
-        # We don't wish to unconditionally upgrade pip, as that would reduce test coverage
+    except SubprocessError as ex:
+        if ex.status != 2:
+            raise
+
+        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.
 
+        # Installing "coverage" on ubuntu 16.04 fails with the error:
+        # AttributeError: 'Requirement' object has no attribute 'project_name'
+        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
+        # Upgrading pip works around the issue.
         run_command(args, ['pip', 'install', '--upgrade', 'pip'])
         run_command(args, cmd)
 
diff --git a/test/runner/lib/sanity.py b/test/runner/lib/sanity.py
index 134111d636..93405b23ae 100644
--- a/test/runner/lib/sanity.py
+++ b/test/runner/lib/sanity.py
@@ -316,7 +316,7 @@ def command_sanity_pep8(args, targets):
         return SanitySkipped(test)
 
     cmd = [
-        'pycodestyle',
+        'pep8',
         '--max-line-length', '160',
         '--config', '/dev/null',
         '--ignore', ','.join(sorted(current_ignore)),
diff --git a/test/runner/requirements/constraints.txt b/test/runner/requirements/constraints.txt
index d29516d6c6..9fea634946 100644
--- a/test/runner/requirements/constraints.txt
+++ b/test/runner/requirements/constraints.txt
@@ -5,3 +5,4 @@ sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7
 yamllint < 1.8 ; python_version < '2.7' # yamllint 1.8 and later require python 2.7 or later
 wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
 isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests
+pep8 <= 1.7.0 # pep8 1.7.0 is the last "real" release, avoid the user warning added in 1.7.1
diff --git a/test/runner/requirements/sanity.txt b/test/runner/requirements/sanity.txt
index df8c7c0713..98a92117c6 100644
--- a/test/runner/requirements/sanity.txt
+++ b/test/runner/requirements/sanity.txt
@@ -1,12 +1,9 @@
-cryptography
 jinja2
 mock
-paramiko
-pycodestyle
+pep8
 pylint
 pytest
 rstcheck
 sphinx
-virtualenv
 voluptuous
 yamllint
diff --git a/test/runner/requirements/windows-integration.txt b/test/runner/requirements/windows-integration.txt
index d6fcc566fc..464b565d66 100644
--- a/test/runner/requirements/windows-integration.txt
+++ b/test/runner/requirements/windows-integration.txt
@@ -1,3 +1,4 @@
+cryptography >= 1.3.4, < 2.1 # cryptography 2.1 requires pip 8.1.2+ (sanity_ok)
 jinja2
 junit-xml
 pywinrm
diff --git a/test/sanity/pep8/current-ignore.txt b/test/sanity/pep8/current-ignore.txt
index e87d7d0338..d09a3150be 100644
--- a/test/sanity/pep8/current-ignore.txt
+++ b/test/sanity/pep8/current-ignore.txt
@@ -22,13 +22,9 @@ E266
 E301
 E302
 E303
-E305
-E306
 E402
 E502
 E713
-E722
 E731
-E741
 W391
 W503