Fix various issues with missing interpreters (#2828)

fix https://github.com/tox-dev/tox/issues/2811

11 files changed, 80 insertions, 23 deletions

diff --git a/docs/changelog/2811.bugfix.rst b/docs/changelog/2811.bugfix.rst
new file mode 100644
index 00000000..24f53569
--- /dev/null
+++ b/docs/changelog/2811.bugfix.rst
@@ -0,0 +1,2 @@
+The combination of ``usedevelop = true`` and ``--skip-missing-interpreters=false`` will no longer fail for environments
+that were *not* invoked - by :user:`stephenfin`.
diff --git a/docs/changelog/2826.bugfix.rst b/docs/changelog/2826.bugfix.rst
new file mode 100644
index 00000000..1d0ec7d4
--- /dev/null
+++ b/docs/changelog/2826.bugfix.rst
@@ -0,0 +1,2 @@
+Fix an attribute error when ``use_develop = true`` is set and an unsupported interpreter version is requested - by
+:user:`stephenfin`.
diff --git a/docs/changelog/2827.bugfix.rst b/docs/changelog/2827.bugfix.rst
new file mode 100644
index 00000000..1fa28363
--- /dev/null
+++ b/docs/changelog/2827.bugfix.rst
@@ -0,0 +1,2 @@
+tox returns a non-zero error code if all envs are skipped. It will now correctly do this if only a single env was
+requested and this was skipped - by :user:`stephenfin`.
diff --git a/docs/upgrading.rst b/docs/upgrading.rst
index 554c76db..3ac77c19 100644
--- a/docs/upgrading.rst
+++ b/docs/upgrading.rst
@@ -88,6 +88,18 @@ moving to the newly added ``py_impl`` and ``py_dot_ver`` variables, for example:
 
    deps = -r{py_impl}{py_dot_ver}-req.txt
 
+Failure when all environments are skipped
+-----------------------------------------
+
+A run that results in all environments being skipped will no longer result in success. Instead, a failure will be
+reported. For example, consider a host that does not support Python 3.5:
+
+.. code-block:: bash
+
+   tox run --skip-missing-interpreters=true -e py35
+
+This will now result in a failure.
+
 Substitutions removed
 ---------------------
 
@@ -134,7 +146,6 @@ Re-use of environments
     [testenv:b]
     deps = pytest<7
 
-
 CLI command compatibility
 -------------------------
 
diff --git a/src/tox/session/cmd/run/common.py b/src/tox/session/cmd/run/common.py
index 02797bfb..73facf62 100644
--- a/src/tox/session/cmd/run/common.py
+++ b/src/tox/session/cmd/run/common.py
@@ -187,7 +187,9 @@ def report(start: float, runs: list[ToxEnvRunResult], is_colored: bool, verbosit
         _print(Fore.GREEN, f"  congratulations :) ({duration:.2f} seconds)")
         return Outcome.OK
     _print(Fore.RED, f"  evaluation failed :( ({duration:.2f} seconds)")
-    return runs[0].code if len(runs) == 1 else -1
+    if len(runs) == 1:
+        return runs[0].code if not runs[0].skipped else -1
+    return -1
 
 
 def _get_outcome_message(run: ToxEnvRunResult) -> tuple[str, int]:
diff --git a/src/tox/tox_env/python/api.py b/src/tox/tox_env/python/api.py
index 58e73ca2..9aa1f839 100644
--- a/src/tox/tox_env/python/api.py
+++ b/src/tox/tox_env/python/api.py
@@ -231,27 +231,29 @@ class Python(ToxEnv, ABC):
     @property
     def base_python(self) -> PythonInfo:
         """Resolve base python"""
+        base_pythons: list[str] = self.conf["base_python"]
+
         if self._base_python_searched is False:
-            base_pythons: list[str] = self.conf["base_python"]
             self._base_python_searched = True
             self._base_python = self._get_python(base_pythons)
-            if self._base_python is None:
-                if self.core["skip_missing_interpreters"]:
-                    raise Skip(f"could not find python interpreter with spec(s): {', '.join(base_pythons)}")
-                raise NoInterpreter(base_pythons)
-            if self.journal:
+            if self._base_python is not None and self.journal:
                 value = self._get_env_journal_python()
                 self.journal["python"] = value
+
+        if self._base_python is None:
+            if self.core["skip_missing_interpreters"]:
+                raise Skip(f"could not find python interpreter with spec(s): {', '.join(base_pythons)}")
+            raise NoInterpreter(base_pythons)
+
         return cast(PythonInfo, self._base_python)
 
     def _get_env_journal_python(self) -> dict[str, Any]:
-        assert self._base_python is not None
         return {
-            "implementation": self._base_python.implementation,
+            "implementation": self.base_python.implementation,
             "version_info": tuple(self.base_python.version_info),
-            "version": self._base_python.version,
-            "is_64": self._base_python.is_64,
-            "sysplatform": self._base_python.platform,
+            "version": self.base_python.version,
+            "is_64": self.base_python.is_64,
+            "sysplatform": self.base_python.platform,
             "extra_version_info": None,
         }
 
diff --git a/src/tox/tox_env/python/package.py b/src/tox/tox_env/python/package.py
index 289bd774..d671550d 100644
--- a/src/tox/tox_env/python/package.py
+++ b/src/tox/tox_env/python/package.py
@@ -14,7 +14,7 @@ from ..api import ToxEnvCreateArgs
 from ..errors import Skip
 from ..package import Package, PackageToxEnv, PathPackage
 from ..runner import RunToxEnv
-from .api import Python
+from .api import NoInterpreter, Python
 from .pip.req_file import PythonDeps
 
 if TYPE_CHECKING:
@@ -87,7 +87,11 @@ class PythonPackageToxEnv(Python, PackageToxEnv, ABC):
             # python only code are often compatible at major level (unless universal wheel in which case both 2/3)
             # c-extension codes are trickier, but as of today both poetry/setuptools uses pypa/wheels logic
             # https://github.com/pypa/wheel/blob/master/src/wheel/bdist_wheel.py#L234-L280
-            run_py = cast(Python, run_env).base_python
+            try:
+                run_py = cast(Python, run_env).base_python
+            except NoInterpreter:
+                run_py = None
+
             if run_py is None:
                 base = ",".join(run_env.conf["base_python"])
                 raise Skip(f"could not resolve base python with {base}")
diff --git a/tests/session/cmd/test_depends.py b/tests/session/cmd/test_depends.py
index 6b27d3bc..7c9040ea 100644
--- a/tests/session/cmd/test_depends.py
+++ b/tests/session/cmd/test_depends.py
@@ -34,18 +34,18 @@ def test_depends(tox_project: ToxProjectCreator, patch_prev_py: Callable[[bool],
        py ~ .pkg
        {py} ~ .pkg
        {prev_py} ~ .pkg | .pkg-{impl}{prev_ver}
-       py31 ~ .pkg | ... (could not resolve base python with py31)
+       py31 ~ .pkg | ... (could not find python interpreter with spec(s): py31)
        cov2
           cov
              py ~ .pkg
              {py} ~ .pkg
              {prev_py} ~ .pkg | .pkg-{impl}{prev_ver}
-             py31 ~ .pkg | ... (could not resolve base python with py31)
+             py31 ~ .pkg | ... (could not find python interpreter with spec(s): py31)
        cov
           py ~ .pkg
           {py} ~ .pkg
           {prev_py} ~ .pkg | .pkg-{impl}{prev_ver}
-          py31 ~ .pkg | ... (could not resolve base python with py31)
+          py31 ~ .pkg | ... (could not find python interpreter with spec(s): py31)
     """
     assert outcome.out == dedent(expected).lstrip()
 
diff --git a/tests/session/cmd/test_sequential.py b/tests/session/cmd/test_sequential.py
index 0825110e..8672234a 100644
--- a/tests/session/cmd/test_sequential.py
+++ b/tests/session/cmd/test_sequential.py
@@ -224,7 +224,7 @@ def test_missing_interpreter_skip_on(tox_project: ToxProjectCreator) -> None:
     proj = tox_project({"tox.ini": ini})
 
     result = proj.run("r")
-    result.assert_success()
+    result.assert_failed()
     assert "py: SKIP" in result.out
 
 
@@ -367,7 +367,7 @@ def test_platform_does_not_match_run_env(tox_project: ToxProjectCreator) -> None
     proj = tox_project({"tox.ini": ini})
 
     result = proj.run("r")
-    result.assert_success()
+    result.assert_failed()
     exp = f"py: skipped because platform {sys.platform} does not match wrong_platform"
     assert exp in result.out
 
diff --git a/tests/tox_env/python/test_python_runner.py b/tests/tox_env/python/test_python_runner.py
index eca65b2d..33de626a 100644
--- a/tests/tox_env/python/test_python_runner.py
+++ b/tests/tox_env/python/test_python_runner.py
@@ -128,8 +128,39 @@ def test_extras_are_normalized(
     ("config", "cli", "expected"),
     [("false", "true", True), ("true", "false", False), ("false", "config", False), ("true", "config", True)],
 )
-def test_config_skip_missing_interpreters(tox_project: ToxProjectCreator, config: str, cli: str, expected: str) -> None:
+def test_config_skip_missing_interpreters(
+    tox_project: ToxProjectCreator,
+    config: str,
+    cli: str,
+    expected: bool,
+) -> None:
     py_ver = ".".join(str(i) for i in sys.version_info[0:2])
     project = tox_project({"tox.ini": f"[tox]\nenvlist=py4,py{py_ver}\nskip_missing_interpreters={config}"})
-    result = project.run("--skip-missing-interpreters", cli)
-    assert result.code == 0 if expected else 1
+    result = project.run(f"--skip-missing-interpreters={cli}")
+    assert result.code == (0 if expected else -1)
+
+
+@pytest.mark.parametrize(
+    ("skip", "env", "retcode"),
+    [
+        ("true", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0),
+        ("false", f"py{''.join(str(i) for i in sys.version_info[0:2])}", 0),
+        ("true", "py31", -1),
+        ("false", "py31", 1),
+        ("true", None, 0),
+        ("false", None, -1),
+    ],
+)
+def test_skip_missing_interpreters_specified_env(
+    tox_project: ToxProjectCreator,
+    skip: str,
+    env: str | None,
+    retcode: int,
+) -> None:
+    py_ver = "".join(str(i) for i in sys.version_info[0:2])
+    project = tox_project({"tox.ini": f"[tox]\nenvlist=py31,py{py_ver}\n[testenv]\nusedevelop=true"})
+    args = [f"--skip-missing-interpreters={skip}"]
+    if env:
+        args += ["-e", env]
+    result = project.run(*args)
+    assert result.code == retcode
diff --git a/whitelist.txt b/whitelist.txt
index c79f0c76..b0a61999 100644
--- a/whitelist.txt
+++ b/whitelist.txt
@@ -142,6 +142,7 @@ replacer
 repo
 reqs
 retann
+retcode
 rfind
 rpartition
 rreq