[stable-2.11] Add PyPI proxy container for tests on Python 2.6. (#74204)

(cherry picked from commit becf9416736dc911d3411b92f09512b4dae2955c)

Co-authored-by: Matt Clay <matt@mystile.com>

7 files changed, 168 insertions, 1 deletions

diff --git a/changelogs/fragments/ansible-test-pypi-test-container.yml b/changelogs/fragments/ansible-test-pypi-test-container.yml
new file mode 100644
index 0000000000..5c2f10c90a
--- /dev/null
+++ b/changelogs/fragments/ansible-test-pypi-test-container.yml
@@ -0,0 +1,3 @@
+major_changes:
+  - ansible-test - Tests run with the ``centos6`` and ``default`` test containers now use a PyPI proxy container to access PyPI when Python 2.6 is used.
+                   This allows tests running under Python 2.6 to continue functioning even though PyPI is discontinuing support for non-SNI capable clients.
diff --git a/test/lib/ansible_test/_data/quiet_pip.py b/test/lib/ansible_test/_data/quiet_pip.py
index 5b5db823e2..e1bb824646 100644
--- a/test/lib/ansible_test/_data/quiet_pip.py
+++ b/test/lib/ansible_test/_data/quiet_pip.py
@@ -13,6 +13,7 @@ LOGGING_MESSAGE_FILTER = re.compile("^("
                                     ".*Running pip install with root privileges is generally not a good idea.*|"  # custom Fedora patch [1]
                                     "DEPRECATION: Python 2.7 will reach the end of its life .*|"  # pip 19.2.3
                                     "Ignoring .*: markers .* don't match your environment|"
+                                    "Looking in indexes: .*|"  # pypi-test-container
                                     "Requirement already satisfied.*"
                                     ")$")
 
diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py
index 7e2650c702..15a235180b 100644
--- a/test/lib/ansible_test/_internal/cli.py
+++ b/test/lib/ansible_test/_internal/cli.py
@@ -39,6 +39,7 @@ from .executor import (
     Delegate,
     generate_pip_install,
     check_startup,
+    configure_pypi_proxy,
 )
 
 from .config import (
@@ -170,6 +171,7 @@ def main():
         display.info('MAXFD: %d' % MAXFD, verbosity=2)
 
         try:
+            configure_pypi_proxy(config)
             args.func(config)
             delegate_args = None
         except Delegate as ex:
@@ -236,6 +238,15 @@ def parse_args():
                         default=0,
                         help='display more output')
 
+    common.add_argument('--pypi-proxy',
+                        action='store_true',
+                        help=argparse.SUPPRESS)  # internal use only
+
+    common.add_argument('--pypi-endpoint',
+                        metavar='URI',
+                        default=None,
+                        help=argparse.SUPPRESS)  # internal use only
+
     common.add_argument('--color',
                         metavar='COLOR',
                         nargs='?',
diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py
index fcae668727..eb9c1739a9 100644
--- a/test/lib/ansible_test/_internal/config.py
+++ b/test/lib/ansible_test/_internal/config.py
@@ -68,6 +68,9 @@ class EnvironmentConfig(CommonConfig):
         """
         super(EnvironmentConfig, self).__init__(args, command)
 
+        self.pypi_endpoint = args.pypi_endpoint  # type: str
+        self.pypi_proxy = args.pypi_proxy  # type: bool
+
         self.local = args.local is True
         self.venv = args.venv
         self.venv_system_site_packages = args.venv_system_site_packages
diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py
index e2ab3217ad..250b9114af 100644
--- a/test/lib/ansible_test/_internal/delegation.py
+++ b/test/lib/ansible_test/_internal/delegation.py
@@ -19,6 +19,7 @@ from .executor import (
     HTTPTESTER_HOSTS,
     create_shell_command,
     run_httptester,
+    run_pypi_proxy,
     start_httptester,
     get_python_interpreter,
     get_python_version,
@@ -285,6 +286,11 @@ def delegate_docker(args, exclude, require, integration_targets):
     if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy):
         cmd_options.append('-it')
 
+    pypi_proxy_id, pypi_proxy_endpoint = run_pypi_proxy(args)
+
+    if pypi_proxy_endpoint:
+        cmd += ['--pypi-endpoint', pypi_proxy_endpoint]
+
     with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
         try:
             create_payload(args, local_source_fd.name)
@@ -406,6 +412,9 @@ def delegate_docker(args, exclude, require, integration_targets):
             if httptester_id:
                 docker_rm(args, httptester_id)
 
+            if pypi_proxy_id:
+                docker_rm(args, pypi_proxy_id)
+
             if test_id:
                 if args.docker_terminate == 'always' or (args.docker_terminate == 'success' and success):
                     docker_rm(args, test_id)
diff --git a/test/lib/ansible_test/_internal/docker_util.py b/test/lib/ansible_test/_internal/docker_util.py
index c04a2a02f9..3ad771bd41 100644
--- a/test/lib/ansible_test/_internal/docker_util.py
+++ b/test/lib/ansible_test/_internal/docker_util.py
@@ -112,7 +112,7 @@ def get_docker_container_ip(args, container_id):
     networks = network_settings.get('Networks')
 
     if networks:
-        network_name = get_docker_preferred_network_name(args)
+        network_name = get_docker_preferred_network_name(args) or 'bridge'
         ipaddress = networks[network_name]['IPAddress']
     else:
         # podman doesn't provide Networks, fall back to using IPAddress
diff --git a/test/lib/ansible_test/_internal/executor.py b/test/lib/ansible_test/_internal/executor.py
index 421354e24f..c3755a7113 100644
--- a/test/lib/ansible_test/_internal/executor.py
+++ b/test/lib/ansible_test/_internal/executor.py
@@ -2,6 +2,7 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+import atexit
 import json
 import os
 import datetime
@@ -81,6 +82,7 @@ from .util_common import (
     write_json_test_results,
     ResultType,
     handle_layout_messages,
+    CommonConfig,
 )
 
 from .docker_util import (
@@ -126,6 +128,8 @@ from .config import (
     ShellConfig,
     WindowsIntegrationConfig,
     TIntegrationConfig,
+    UnitsConfig,
+    SanityConfig,
 )
 
 from .metadata import (
@@ -145,6 +149,10 @@ from .data import (
     data_context,
 )
 
+from .http import (
+    urlparse,
+)
+
 HTTPTESTER_HOSTS = (
     'ansible.http.tests',
     'sni1.ansible.http.tests',
@@ -1404,6 +1412,138 @@ rdr pass inet proto tcp from any to any port 749 -> 127.0.0.1 port 8749
         raise ApplicationError('No supported port forwarding mechanism detected.')
 
 
+def run_pypi_proxy(args):  # type: (EnvironmentConfig) -> t.Tuple[t.Optional[str], t.Optional[str]]
+    """Run a PyPI proxy container, returning the container ID and proxy endpoint."""
+    use_proxy = False
+
+    if args.docker_raw == 'centos6':
+        use_proxy = True  # python 2.6 is the only version available
+
+    if args.docker_raw == 'default':
+        if args.python == '2.6':
+            use_proxy = True  # python 2.6 requested
+        elif not args.python and isinstance(args, (SanityConfig, UnitsConfig, ShellConfig)):
+            use_proxy = True  # multiple versions (including python 2.6) can be used
+
+    if args.docker_raw and args.pypi_proxy:
+        use_proxy = True  # manual override to force proxy usage
+
+    if not use_proxy:
+        return None, None
+
+    proxy_image = 'quay.io/ansible/pypi-test-container:1.0.0'
+    port = 3141
+
+    options = [
+        '--detach',
+        '-p', '%d:%d' % (port, port),
+    ]
+
+    docker_pull(args, proxy_image)
+
+    container_id = docker_run(args, proxy_image, options=options)[0]
+
+    if args.explain:
+        container_id = 'pypi_id'
+        container_ip = '127.0.0.1'
+    else:
+        container_id = container_id.strip()
+        container_ip = get_docker_container_ip(args, container_id)
+
+    endpoint = 'http://%s:%d/root/pypi/+simple/' % (container_ip, port)
+
+    return container_id, endpoint
+
+
+def configure_pypi_proxy(args):  # type: (CommonConfig) -> None
+    """Configure the environment to use a PyPI proxy, if present."""
+    if not isinstance(args, EnvironmentConfig):
+        return
+
+    if args.pypi_endpoint:
+        configure_pypi_block_access()
+        configure_pypi_proxy_pip(args)
+        configure_pypi_proxy_easy_install(args)
+
+
+def configure_pypi_block_access():  # type: () -> None
+    """Block direct access to PyPI to ensure proxy configurations are always used."""
+    if os.getuid() != 0:
+        display.warning('Skipping custom hosts block for PyPI for non-root user.')
+        return
+
+    hosts_path = '/etc/hosts'
+    hosts_block = '''
+127.0.0.1 pypi.org pypi.python.org files.pythonhosted.org
+'''
+
+    def hosts_cleanup():
+        display.info('Removing custom PyPI hosts entries: %s' % hosts_path, verbosity=1)
+
+        with open(hosts_path) as hosts_file_read:
+            content = hosts_file_read.read()
+
+        content = content.replace(hosts_block, '')
+
+        with open(hosts_path, 'w') as hosts_file_write:
+            hosts_file_write.write(content)
+
+    display.info('Injecting custom PyPI hosts entries: %s' % hosts_path, verbosity=1)
+    display.info('Config: %s\n%s' % (hosts_path, hosts_block), verbosity=3)
+
+    with open(hosts_path, 'a') as hosts_file:
+        hosts_file.write(hosts_block)
+
+    atexit.register(hosts_cleanup)
+
+
+def configure_pypi_proxy_pip(args):  # type: (EnvironmentConfig) -> None
+    """Configure a custom index for pip based installs."""
+    pypi_hostname = urlparse(args.pypi_endpoint)[1].split(':')[0]
+
+    pip_conf_path = os.path.expanduser('~/.pip/pip.conf')
+    pip_conf = '''
+[global]
+index-url = {0}
+trusted-host = {1}
+'''.format(args.pypi_endpoint, pypi_hostname).strip()
+
+    def pip_conf_cleanup():
+        display.info('Removing custom PyPI config: %s' % pip_conf_path, verbosity=1)
+        os.remove(pip_conf_path)
+
+    if os.path.exists(pip_conf_path):
+        raise ApplicationError('Refusing to overwrite existing file: %s' % pip_conf_path)
+
+    display.info('Injecting custom PyPI config: %s' % pip_conf_path, verbosity=1)
+    display.info('Config: %s\n%s' % (pip_conf_path, pip_conf), verbosity=3)
+
+    write_text_file(pip_conf_path, pip_conf, True)
+    atexit.register(pip_conf_cleanup)
+
+
+def configure_pypi_proxy_easy_install(args):  # type: (EnvironmentConfig) -> None
+    """Configure a custom index for easy_install based installs."""
+    pydistutils_cfg_path = os.path.expanduser('~/.pydistutils.cfg')
+    pydistutils_cfg = '''
+[easy_install]
+index_url = {0}
+'''.format(args.pypi_endpoint).strip()
+
+    if os.path.exists(pydistutils_cfg_path):
+        raise ApplicationError('Refusing to overwrite existing file: %s' % pydistutils_cfg_path)
+
+    def pydistutils_cfg_cleanup():
+        display.info('Removing custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1)
+        os.remove(pydistutils_cfg_path)
+
+    display.info('Injecting custom PyPI config: %s' % pydistutils_cfg_path, verbosity=1)
+    display.info('Config: %s\n%s' % (pydistutils_cfg_path, pydistutils_cfg), verbosity=3)
+
+    write_text_file(pydistutils_cfg_path, pydistutils_cfg, True)
+    atexit.register(pydistutils_cfg_cleanup)
+
+
 def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, inventory_path, temp_path, always):
     """
     :type args: IntegrationConfig