Relocate ansible-test self tests outside package. (#61255)

* Relocate ansible-test self tests outside package.

We don't want to include the tests for verifying ansible-test within the ansible-test package.

* Add `test/ansible_test/` to classification.py.

* Fix test invocation.

* Relocate tests in MANIFEST.in.

* Improve package-data sanity test error checking.

* Only use includes for ansible-test in MANIFEST.in.

* Improve readability of MANIFEST.in.

9 files changed, 36 insertions, 22 deletions

diff --git a/MANIFEST.in b/MANIFEST.in
index 5ec2feb94b..1eb821d91f 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -19,12 +19,13 @@ recursive-include lib/ansible/galaxy/data *.yml *.j2 README.md ansible.cfg inven
 recursive-include lib/ansible/config *.yml
 recursive-include licenses *.txt
 recursive-include packaging *
+recursive-include test/ansible_test *.py Makefile
 recursive-include test/cache .keep
 recursive-include test/integration *
 recursive-include test/lib/ansible_test/config *.template
-recursive-include test/lib/ansible_test/_data *
-recursive-include test/lib/ansible_test/tests *
-recursive-exclude test/lib/ansible_test *.pyc *.pyo *.bak *.orig *~ *.rej
+recursive-include test/lib/ansible_test/_data *.cfg *.ini *.json *.ps1 *.psd1 *.py *.sh *.txt *.yml coveragerc inventory
+recursive-include test/lib/ansible_test/_data/injector ansible ansible-config ansible-connection ansible-console ansible-doc ansible-galaxy ansible-playbook ansible-pull ansible-test ansible-vault pytest
+recursive-include test/lib/ansible_test/_data/sanity/validate-modules validate-modules
 recursive-include test/results .keep
 recursive-include test/sanity *.json *.py *.txt
 exclude test/sanity/code-smell/botmeta.*
diff --git a/test/ansible_test/Makefile b/test/ansible_test/Makefile
new file mode 100644
index 0000000000..7fb2a00760
--- /dev/null
+++ b/test/ansible_test/Makefile
@@ -0,0 +1,13 @@
+all: sanity unit validate-modules-unit
+
+.PHONY: sanity
+sanity:
+	$(abspath ${CURDIR}/../../bin/ansible-test) sanity test/lib/ ${FLAGS}
+
+.PHONY: unit
+unit:
+	PYTHONPATH=$(abspath ${CURDIR}/../lib) pytest unit ${FLAGS}
+
+.PHONY: validate-modules-unit
+validate-modules-unit:
+	PYTHONPATH=$(abspath ${CURDIR}/../lib/ansible_test/_data/sanity/validate-modules):$(abspath ${CURDIR}/../../lib) pytest validate-modules-unit ${FLAGS}
diff --git a/test/lib/ansible_test/tests/unit/test_diff.py b/test/ansible_test/unit/test_diff.py
index 93e7243b7a..1f2559d2b9 100644
--- a/test/lib/ansible_test/tests/unit/test_diff.py
+++ b/test/ansible_test/unit/test_diff.py
@@ -6,12 +6,12 @@ import os
 import subprocess
 import pytest
 
-from ..._internal.util import (
+from ansible_test._internal.util import (
     to_text,
     to_bytes,
 )
 
-from ..._internal.diff import (
+from ansible_test._internal.diff import (
     parse_diff,
     FileDiff,
 )
diff --git a/test/lib/ansible_test/tests/validate-modules-unit/test_validate_modules_regex.py b/test/ansible_test/validate-modules-unit/test_validate_modules_regex.py
index 8c0b45ca3e..8c0b45ca3e 100644
--- a/test/lib/ansible_test/tests/validate-modules-unit/test_validate_modules_regex.py
+++ b/test/ansible_test/validate-modules-unit/test_validate_modules_regex.py
diff --git a/test/lib/ansible_test/_internal/classification.py b/test/lib/ansible_test/_internal/classification.py
index 12f448e27e..58e34c2076 100644
--- a/test/lib/ansible_test/_internal/classification.py
+++ b/test/lib/ansible_test/_internal/classification.py
@@ -617,6 +617,9 @@ class PathMapper:
 
             return minimal
 
+        if path.startswith('test/ansible_test/'):
+            return minimal  # these tests are not invoked from ansible-test
+
         if path.startswith('test/cache/'):
             return minimal
 
diff --git a/test/lib/ansible_test/tests/Makefile b/test/lib/ansible_test/tests/Makefile
deleted file mode 100644
index 4ead9f1909..0000000000
--- a/test/lib/ansible_test/tests/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-all: sanity unit validate-modules-unit
-
-.PHONY: sanity
-sanity:
-	ansible-test sanity test/lib/ ${FLAGS}
-
-.PHONY: unit
-unit:
-	PYTHONPATH=$(abspath ${CURDIR}/../..) pytest unit ${FLAGS}
-
-.PHONY: validate-modules-unit
-validate-modules-unit:
-	PYTHONPATH=$(abspath ${CURDIR}/../_data/sanity/validate-modules):$(abspath ${CURDIR}/../../../../lib) pytest validate-modules-unit ${FLAGS}
diff --git a/test/lib/ansible_test/tests/__init__.py b/test/lib/ansible_test/tests/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/test/lib/ansible_test/tests/__init__.py
+++ /dev/null
diff --git a/test/lib/ansible_test/tests/unit/__init__.py b/test/lib/ansible_test/tests/unit/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/test/lib/ansible_test/tests/unit/__init__.py
+++ /dev/null
diff --git a/test/sanity/code-smell/package-data.py b/test/sanity/code-smell/package-data.py
index 5b0392c22c..d3a4d7a811 100755
--- a/test/sanity/code-smell/package-data.py
+++ b/test/sanity/code-smell/package-data.py
@@ -161,12 +161,17 @@ def clean_repository(file_list):
 
 def create_sdist(tmp_dir):
     """Create an sdist in the repository"""
-    dummy = subprocess.Popen(
+    create = subprocess.Popen(
         ['make', 'snapshot', 'SDIST_DIR=%s' % tmp_dir],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         universal_newlines=True,
-    ).communicate()
+    )
+
+    stderr = create.communicate()[1]
+
+    if create.returncode != 0:
+        raise Exception('make snapshot failed:\n%s' % stderr)
 
     # Determine path to sdist
     tmp_dir_files = os.listdir(tmp_dir)
@@ -204,13 +209,18 @@ def extract_sdist(sdist_path, tmp_dir):
 
 def install_sdist(tmp_dir, sdist_dir):
     """Install the extracted sdist into the temporary directory"""
-    stdout, _dummy = subprocess.Popen(
+    install = subprocess.Popen(
         ['python', 'setup.py', 'install', '--root=%s' % tmp_dir],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
         universal_newlines=True,
         cwd=os.path.join(tmp_dir, sdist_dir),
-    ).communicate()
+    )
+
+    stdout, stderr = install.communicate()
+
+    if install.returncode != 0:
+        raise Exception('sdist install failed:\n%s' % stderr)
 
     # Determine the prefix for the installed files
     match = re.search('^creating (%s/.*?/(?:site|dist)-packages)/ansible$' %