diff options
author | Ned Batchelder <ned@nedbatchelder.com> | 2023-01-05 12:32:32 -0500 |
---|---|---|
committer | Ned Batchelder <ned@nedbatchelder.com> | 2023-01-05 12:32:32 -0500 |
commit | 097e13177ecc97638426a8e3a276ddeaae5422a1 (patch) | |
tree | ab2c633ca8aeceb7c8a02ddbd4f9fe99c3becfd8 | |
parent | 3a02703108831a554ec893b9031dcebe377fdd89 (diff) | |
download | python-coveragepy-git-097e13177ecc97638426a8e3a276ddeaae5422a1.tar.gz |
perf: some quick refactoring to test #1527
-rw-r--r-- | lab/benchmark.py | 167 |
1 files changed, 135 insertions, 32 deletions
diff --git a/lab/benchmark.py b/lab/benchmark.py index ca94f28c..f3bc86d7 100644 --- a/lab/benchmark.py +++ b/lab/benchmark.py @@ -11,7 +11,7 @@ import sys import time from pathlib import Path -from typing import Dict, Iterable, Iterator, List, Optional, Tuple +from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple class ShellSession: @@ -116,17 +116,21 @@ class ProjectToTest: # Where can we clone the project from? git_url: Optional[str] = None + slug: Optional[str] = None def __init__(self): - if self.git_url: - self.slug = self.git_url.split("/")[-1] - self.dir = Path(self.slug) + if not self.slug: + if self.git_url: + self.slug = self.git_url.split("/")[-1] - def get_source(self, shell): - """Get the source of the project.""" + def make_dir(self): + self.dir = Path(f"work_{self.slug}") if self.dir.exists(): rmrf(self.dir) - shell.run_command(f"git clone {self.git_url}") + + def get_source(self, shell): + """Get the source of the project.""" + shell.run_command(f"git clone {self.git_url} {self.dir}") def prep_environment(self, env): """Prepare the environment to run the test suite. @@ -135,20 +139,41 @@ class ProjectToTest: """ pass + def tweak_coverage_settings(self, settings: Iterable[Tuple[str, Any]]) -> Iterator[None]: + """Tweak the coverage settings. + + NOTE: This is not properly factored, and is only used by ToxProject now!!! + """ + pass + def run_no_coverage(self, env): """Run the test suite with no coverage measurement.""" pass - def run_with_coverage(self, env, pip_args, cov_options): + def run_with_coverage(self, env, pip_args, cov_tweaks): """Run the test suite with coverage measurement.""" pass +class EmptyProject(ProjectToTest): + """A dummy project for testing other parts of this code.""" + def __init__(self, slug: str="empty", fake_durations: Iterable[float]=(1.23,)): + self.slug = slug + self.durations = iter(itertools.cycle(fake_durations)) + + def get_source(self, shell): + pass + + def run_with_coverage(self, env, pip_args, cov_tweaks): + """Run the test suite with coverage measurement.""" + return next(self.durations) + + class ToxProject(ProjectToTest): """A project using tox to run the test suite.""" def prep_environment(self, env): - env.shell.run_command(f"{env.python} -m pip install tox") + env.shell.run_command(f"{env.python} -m pip install 'tox<4'") self.run_tox(env, env.pyver.toxenv, "--notest") def run_tox(self, env, toxenv, toxargs=""): @@ -159,13 +184,16 @@ class ToxProject(ProjectToTest): def run_no_coverage(self, env): return self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install") - def run_with_coverage(self, env, pip_args, cov_options): - assert not cov_options, f"ToxProject.run_with_coverage can't take cov_options={cov_options!r}" + def run_with_coverage(self, env, pip_args, cov_tweaks): self.run_tox(env, env.pyver.toxenv, "--notest") env.shell.run_command( f".tox/{env.pyver.toxenv}/bin/python -m pip install {pip_args}" ) - return self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install") + with self.tweak_coverage_settings(cov_tweaks): + self.pre_check(env) # NOTE: Not properly factored, and only used from here. + duration = self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install") + self.post_check(env) # NOTE: Not properly factored, and only used from here. + return duration class ProjectPytestHtml(ToxProject): @@ -173,12 +201,13 @@ class ProjectPytestHtml(ToxProject): git_url = "https://github.com/pytest-dev/pytest-html" - def run_with_coverage(self, env, pip_args, cov_options): + def run_with_coverage(self, env, pip_args, cov_tweaks): + raise Exception("This doesn't work because options changed to tweaks") covenv = env.pyver.toxenv + "-cov" self.run_tox(env, covenv, "--notest") env.shell.run_command(f".tox/{covenv}/bin/python -m pip install {pip_args}") - if cov_options: - replace = ("# reference: https", f"[run]\n{cov_options}\n#") + if cov_tweaks: + replace = ("# reference: https", f"[run]\n{cov_tweaks}\n#") else: replace = ("", "") with file_replace(Path(".coveragerc"), *replace): @@ -206,6 +235,34 @@ class ProjectAttrs(ToxProject): git_url = "https://github.com/python-attrs/attrs" + def tweak_coverage_settings(self, tweaks: Iterable[Tuple[str, Any]]) -> Iterator[None]: + return tweak_toml_coverage_settings("pyproject.toml", tweaks) + + def pre_check(self, env): + env.shell.run_command("cat pyproject.toml") + + def post_check(self, env): + env.shell.run_command("ls -al") + + +def tweak_toml_coverage_settings(toml_file: str, tweaks: Iterable[Tuple[str, Any]]) -> Iterator[None]: + if tweaks: + toml_inserts = [] + for name, value in tweaks: + if isinstance(value, bool): + toml_inserts.append(f"{name} = {str(value).lower()}") + elif isinstance(value, str): + toml_inserts.append(f"{name} = '{value}'") + else: + raise Exception(f"Can't tweak toml setting: {name} = {value!r}") + header = "[tool.coverage.run]\n" + insert = header + "\n".join(toml_inserts) + "\n" + else: + header = insert = "" + return file_replace(Path(toml_file), header, insert) + + + class AdHocProject(ProjectToTest): """A standalone program to run locally.""" @@ -232,7 +289,7 @@ class AdHocProject(ProjectToTest): env.shell.run_command(f"{env.python} {self.python_file}") return env.shell.last_duration - def run_with_coverage(self, env, pip_args, cov_options): + def run_with_coverage(self, env, pip_args, cov_tweaks): env.shell.run_command(f"{env.python} -m pip install {pip_args}") with change_dir(self.cur_dir): env.shell.run_command( @@ -265,7 +322,6 @@ class PyVersion: # The tox environment to run this Python toxenv: str - class Python(PyVersion): """A version of CPython to use.""" @@ -273,7 +329,6 @@ class Python(PyVersion): self.command = self.slug = f"python{major}.{minor}" self.toxenv = f"py{major}{minor}" - class PyPy(PyVersion): """A version of PyPy to use.""" @@ -288,6 +343,7 @@ class AdHocPython(PyVersion): self.slug = slug self.toxenv = None + @dataclasses.dataclass class Coverage: """A version of coverage.py to use, maybe None.""" @@ -296,33 +352,33 @@ class Coverage: # Arguments for "pip install ..." pip_args: Optional[str] = None # Tweaks to the .coveragerc file - options: Optional[str] = None + tweaks: Optional[Iterable[Tuple[str, Any]]] = None class CoveragePR(Coverage): """A version of coverage.py from a pull request.""" - def __init__(self, number, options=None): + def __init__(self, number, tweaks=None): super().__init__( slug=f"#{number}", pip_args=f"git+https://github.com/nedbat/coveragepy.git@refs/pull/{number}/merge", - options=options, + tweaks=tweaks, ) class CoverageCommit(Coverage): """A version of coverage.py from a specific commit.""" - def __init__(self, sha, options=None): + def __init__(self, sha, tweaks=None): super().__init__( slug=sha, pip_args=f"git+https://github.com/nedbat/coveragepy.git@{sha}", - options=options, + tweaks=tweaks, ) class CoverageSource(Coverage): """The coverage.py in a working tree.""" - def __init__(self, directory, options=None): + def __init__(self, directory, tweaks=None): super().__init__( slug="source", pip_args=directory, - options=options, + tweaks=tweaks, ) @@ -337,6 +393,8 @@ class Env: ResultData = Dict[Tuple[str, str, str], float] +DIMENSION_NAMES = ["proj", "pyver", "cov"] + class Experiment: """A particular time experiment to run.""" @@ -353,9 +411,18 @@ class Experiment: def run(self, num_runs: int = 3) -> None: results = [] + total_runs = ( + len(self.projects) * + len(self.py_versions) * + len(self.cov_versions) * + num_runs + ) + total_run_nums = iter(itertools.count(start=1)) + for proj in self.projects: print(f"Testing with {proj.slug}") with ShellSession(f"output_{proj.slug}.log") as shell: + proj.make_dir() proj.get_source(shell) for pyver in self.py_versions: @@ -366,20 +433,23 @@ class Experiment: shell.run_command(f"{python} -V") env = Env(pyver, python, shell) - with change_dir(Path(proj.slug)): + with change_dir(proj.dir): print(f"Prepping for {proj.slug} {pyver.slug}") proj.prep_environment(env) for cov_ver in self.cov_versions: durations = [] for run_num in range(num_runs): + total_run_num = next(total_run_nums) print( - f"Running tests, cov={cov_ver.slug}, {run_num+1} of {num_runs}" + f"Running tests, cov={cov_ver.slug}, " + + f"{run_num+1} of {num_runs}, " + + f"total {total_run_num}/{total_runs}" ) if cov_ver.pip_args is None: dur = proj.run_no_coverage(env) else: dur = proj.run_with_coverage( - env, cov_ver.pip_args, cov_ver.options, + env, cov_ver.pip_args, cov_ver.tweaks, ) print(f"Tests took {dur:.3f}s") durations.append(dur) @@ -411,7 +481,7 @@ class Experiment: table_axes = [dimensions[rowname] for rowname in rows] data_order = [*rows, column] - remap = [data_order.index(datum) for datum in ["proj", "pyver", "cov"]] + remap = [data_order.index(datum) for datum in DIMENSION_NAMES] WIDTH = 20 def as_table_row(vals): @@ -445,8 +515,12 @@ class Experiment: PERF_DIR = Path("/tmp/covperf") def run_experiment( - py_versions: List[PyVersion], cov_versions: List[Coverage], projects: List[ProjectToTest], - rows: List[str], column: str, ratios: Iterable[Tuple[str, str, str]] = (), + py_versions: List[PyVersion], + cov_versions: List[Coverage], + projects: List[ProjectToTest], + rows: List[str], + column: str, + ratios: Iterable[Tuple[str, str, str]] = (), ): slugs = [v.slug for v in py_versions + cov_versions + projects] if len(set(slugs)) != len(slugs): @@ -456,6 +530,8 @@ def run_experiment( ratio_slugs = [rslug for ratio in ratios for rslug in ratio[1:]] if any(rslug not in slugs for rslug in ratio_slugs): raise Exception(f"Ratio slug doesn't match a slug: {ratio_slugs}, {slugs}") + if set(rows + [column]) != set(DIMENSION_NAMES): + raise Exception(f"All of these must be in rows or column: {', '.join(DIMENSION_NAMES)}") print(f"Removing and re-making {PERF_DIR}") rmrf(PERF_DIR) @@ -466,7 +542,7 @@ def run_experiment( exp.show_results(rows=rows, column=column, ratios=ratios) -if 1: +if 0: run_experiment( py_versions=[ #Python(3, 11), @@ -489,3 +565,30 @@ if 1: ("94231 vs 3.10", "94231", "v3.10.5"), ], ) + + +if 1: + run_experiment( + py_versions=[ + Python(3, 11), + ], + cov_versions=[ + Coverage("701", "coverage==7.0.1"), + Coverage("701.dynctx", "coverage==7.0.1", [("dynamic_context", "test_function")]), + Coverage("702", "coverage==7.0.2"), + Coverage("702.dynctx", "coverage==7.0.2", [("dynamic_context", "test_function")]), + ], + projects=[ + #EmptyProject("empty", [1.2, 3.4]), + #EmptyProject("dummy", [6.9, 7.1]), + #ProjectDateutil(), + ProjectAttrs(), + ], + rows=["proj", "pyver"], + column="cov", + ratios=[ + (".2 vs .1", "702", "701"), + (".1 dynctx cost", "701.dynctx", "701"), + (".2 dynctx cost", "702.dynctx", "702"), + ], + ) |