Remove 3.5 and xenial support (SC-711) (#1167)

Includes:
 - Update tox.ini and .travis.yml accordingly
 - Cleanup tox.ini with new tox syntax and cloud-init dependencies
 - Update documentation accordingly
 - Replace/remove xenial references where additional testing isn't required
 - Remove xenial checks in integration tests
 - Replace yield_fixture with fixture in pytest tests

Sections of code commented with lines like "Remove when Xenial is no
longer supported" still exist as they're require additional testing.

1 files changed, 77 insertions, 113 deletions

diff --git a/tox.ini b/tox.ini
index 034ee9a4..57d18cdb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,58 +1,52 @@
 [tox]
-envlist = py3, xenial-dev, flake8, pylint, black, isort
+envlist = py3, lowest-supported-dev, flake8, pylint, black, isort
 recreate = True
 
 [testenv]
-commands = {envpython} -m pytest {posargs:tests/unittests}
+basepython = python3
 setenv =
     LC_ALL = en_US.utf-8
 passenv=
     PYTEST_ADDOPTS
 
-[testenv:flake8]
-basepython = python3
+[flake_env]
+envdir = {toxworkdir}/.flake_env
 deps =
     flake8==3.9.2
-commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/ setup.py}
-
-# https://github.com/gabrielfalcao/HTTPretty/issues/223
-setenv =
-    LC_ALL = en_US.utf-8
-
-[testenv:pylint]
-basepython = python3
-deps =
-    # requirements
     pylint==2.11.1
-    # test-requirements because unit tests are now present in cloudinit tree
+    black==21.12b0
+    isort==5.10.1
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/integration-requirements.txt
+
+[testenv:flake8]
+envdir = {[flake_env]envdir}
+deps = {[flake_env]deps}
+commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/ setup.py}
+
+[testenv:pylint]
+envdir = {[flake_env]envdir}
+deps = {[flake_env]deps}
 commands = {envpython} -m pylint {posargs:cloudinit tests tools}
 
 [testenv:black]
-basepython = python3
-deps =
-    # requirements
-    black==21.12b0
+envdir = {[flake_env]envdir}
+deps = {[flake_env]deps}
 commands = {envpython} -m black . --check
 
 [testenv:isort]
-basepython = python3
-deps =
-    isort==5.10.1
+envdir = {[flake_env]envdir}
+deps = {[flake_env]deps}
 commands = {envpython} -m isort . --check-only
 
 [testenv:format]
-basepython = python3
-deps =
-    black==21.12b0
-    isort==5.10.1
+envdir = {[flake_env]envdir}
+deps = {[flake_env]deps}
 commands =
     {envpython} -m isort .
     {envpython} -m black .
 
 [testenv:py3]
-basepython = python3
 deps =
     -r{toxinidir}/test-requirements.txt
 commands = {envpython} -m pytest \
@@ -60,84 +54,56 @@ commands = {envpython} -m pytest \
             {posargs:--cov=cloudinit --cov-branch \
             tests/unittests}
 
-[testenv:py27]
-basepython = python2.7
-deps = -r{toxinidir}/test-requirements.txt
+[lowest-supported-deps]
+# Tox is going to install requirements from pip. This is fine for
+# testing python version compatibility, but when we build cloud-init, we are
+# building against the dependencies in the OS repo, not pip. The OS
+# dependencies will generally be older than what is found in pip.
 
-[flake8]
-# E203: whitespace before ':', doesn't adhere to pep8 or black formatting
-# W503: line break before binary operator
-ignore=E203,W503
-exclude = .venv,.tox,dist,doc,*egg,.git,build,tools
-per-file-ignores =
-    cloudinit/cmd/main.py:E402
+# To obtain these versions, check the versions of these libraries
+# in the oldest support Ubuntu distro.
 
-[testenv:doc]
-basepython = python3
-deps =
-    -r{toxinidir}/doc-requirements.txt
-commands =
-    {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html}
-    doc8 doc/rtd
-
-[xenial-shared-deps]
-# The version of pytest in xenial doesn't work with Python 3.8, so we define
-# two xenial environments: [testenv:xenial] runs the tests with exactly the
-# version of pytest present in xenial, and is used in CI.  [testenv:xenial-dev]
-# runs the tests with the lowest version of pytest that works with Python 3.8,
-# 3.0.7, but keeps the other dependencies at xenial's level.
-#
-# (This section is not a testenv, it is used to maintain a single definition of
-# the dependencies shared between the two xenial testenvs.)
+# httpretty isn't included here because python2.7 requires a higher version
+# than whats run on bionic, so we need two different definitions.
 deps =
-    # requirements
-    jinja2==2.8
-    pyyaml==3.11
-    oauthlib==1.0.3
-    pyserial==3.0.1
+    jinja2==2.10
+    oauthlib==2.0.6
+    pyserial==3.4
     configobj==5.0.6
-    requests==2.9.1
-    jsonschema
+    pyyaml==3.12
+    requests==2.18.4
+    jsonpatch==1.16
+    jsonschema==2.6.0
+    netifaces==0.10.4
     # test-requirements
-    pytest-catchlog==1.2.1
-
-[testenv:xenial]
-# When updating this commands definition, also update the definition in
-# [testenv:xenial-dev].  See the comment there for details.
-commands =
-  python ./tools/pipremove jsonschema
-  python -m pytest {posargs:tests/unittests}
-basepython = python3
+    pytest==3.3.2
+    pytest-cov==2.5.1
+    # Needed by pytest and default causes failures
+    attrs==17.4.0
+
+[testenv:lowest-supported]
+# This definition will run on bionic with the version of httpretty
+# that runs there
 deps =
-    # Refer to the comment in [xenial-shared-deps] for details
-    {[xenial-shared-deps]deps}
-    httpretty==0.8.6
-    jsonpatch==1.10
-    pytest==2.8.7
-
-[testenv:xenial-dev]
-# This should be:
-#   commands = {[testenv:xenial]commands}
-# but the version of pytest in xenial has a bug
-# (https://github.com/tox-dev/tox/issues/208) which means that the {posargs}
-# substitution variable is misparsed and causes a traceback.  Ensure that any
-# changes here are reflected in [testenv:xenial].
-commands =
-  python ./tools/pipremove jsonschema
-  python -m pytest {posargs:tests/unittests}
-basepython = {[testenv:xenial]basepython}
+    {[lowest-supported-deps]deps}
+    httpretty==0.8.14
+commands = {[testenv:py3]commands}
+
+[testenv:lowest-supported-dev]
+# The oldest httpretty version to work with Python 3.7+ is 0.9.5,
+# because it is the first to include this commit:
+# https://github.com/gabrielfalcao/HTTPretty/commit/5776d97da3992b9071db5e21faf175f6e8729060
 deps =
-    # Refer to the comment in [xenial-shared-deps] for details
-    {[xenial-shared-deps]deps}
-    # httpretty in xenial is 0.8.6, not 0.9.5.  The oldest version to work with
-    # Python 3.7+ is 0.9.5, because it is the first to include this commit:
-    # https://github.com/gabrielfalcao/HTTPretty/commit/5776d97da3992b9071db5e21faf175f6e8729060
+    {[lowest-supported-deps]deps}
     httpretty==0.9.5
-    # jsonpatch in xenial is 1.10, not 1.19 (#839779). The oldest version
-    # to work with python3.6 is 1.16 as found in Artful.  To keep default
-    # invocation of 'tox' happy, accept the difference in version here.
-    jsonpatch==1.16
-    pytest==3.0.7
+commands = {[testenv:py3]commands}
+
+[testenv:doc]
+deps =
+    -r{toxinidir}/doc-requirements.txt
+commands =
+    {envpython} -m sphinx {posargs:doc/rtd doc/rtd_html}
+    doc8 doc/rtd
 
 [testenv:tip-flake8]
 commands = {envpython} -m flake8 {posargs:cloudinit/ tests/ tools/ setup.py}
@@ -152,37 +118,36 @@ deps =
     -r{toxinidir}/test-requirements.txt
     -r{toxinidir}/integration-requirements.txt
 
-# Until Xenial tox support is dropped or bumps to tox:2.3.2, reflect changes to
-# deps into testenv:integration-tests-ci: commands, passenv and deps.
-# This is due to (https://github.com/tox-dev/tox/issues/208) which means that
-# the {posargs} handling and substitutions won't do what we want until tox 2.3.2
-# Once Xenial is dropped, integration-tests-ci can use proper substitution
-# commands = {[testenv:integration-tests]commands}
 [testenv:integration-tests]
-basepython = python3
 commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
-passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_*
 deps =
     -r{toxinidir}/integration-requirements.txt
+passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_*
 
 [testenv:integration-tests-ci]
-commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
+commands = {[testenv:integration-tests]commands}
+deps = {[testenv:integration-tests]deps}
 passenv = CLOUD_INIT_* SSH_AUTH_SOCK OS_* TRAVIS
-deps =
-    -r{toxinidir}/integration-requirements.txt
 setenv =
     PYTEST_ADDOPTS="-m ci and not adhoc"
 
 [testenv:integration-tests-jenkins]
-commands = {envpython} -m pytest --log-cli-level=INFO {posargs:tests/integration_tests}
+commands = {[testenv:integration-tests]commands}
+deps = {[testenv:integration-tests]deps}
 passenv = *_proxy CLOUD_INIT_* SSH_AUTH_SOCK OS_* GOOGLE_* GCP_*
-deps =
-    -r{toxinidir}/integration-requirements.txt
 setenv =
     PYTEST_ADDOPTS="-m not adhoc"
 
+[flake8]
+# E203: whitespace before ':', doesn't adhere to pep8 or black formatting
+# W503: line break before binary operator
+ignore=E203,W503
+exclude = .venv,.tox,dist,doc,*egg,.git,build,tools
+per-file-ignores =
+    cloudinit/cmd/main.py:E402
+
 [pytest]
-# TODO: s/--strict/--strict-markers/ once xenial support is dropped
+# TODO: s/--strict/--strict-markers/ once pytest version is high enough
 testpaths = tests/unittests
 addopts = --strict
 log_format = %(asctime)s %(levelname)-9s %(name)s:%(filename)s:%(lineno)d %(message)s
@@ -202,7 +167,6 @@ markers =
     lxd_setup: specify callable to be called between init and start
     lxd_use_exec: `execute` will use `lxc exec` instead of SSH
     lxd_vm: test will only run in LXD VM
-    not_xenial: test cannot run on the xenial release
     not_bionic: test cannot run on the bionic release
     no_container: test cannot run in a container
     user_data: the user data to be passed to the test instance