From 8bf75b6686690f672652ed55b010a73cc45b5f31 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 1 May 2023 09:41:14 -0700 Subject: [stable-2.13] Release tool improvements (#80641) (#80681) * Provide reproducible sdist builds. * Use reproducible wheel builds. * Add PyPI artifact checks. (cherry picked from commit d37678c5ff221ada71afc0cad3ff7b70e0a0ec2f) --- packaging/release.py | 51 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/packaging/release.py b/packaging/release.py index 1d1ba37117..795e3336b0 100755 --- a/packaging/release.py +++ b/packaging/release.py @@ -10,6 +10,7 @@ import dataclasses import datetime import enum import functools +import gzip import hashlib import http.client import inspect @@ -21,6 +22,7 @@ import re import secrets import shlex import shutil +import stat import subprocess import sys import tarfile @@ -765,6 +767,41 @@ def set_ansible_version(current_version: Version, requested_version: Version) -> ANSIBLE_RELEASE_FILE.write_text(updated) +def create_reproducible_sdist(original_path: pathlib.Path, output_path: pathlib.Path, mtime: int) -> None: + """Read the specified sdist and write out a new copy with uniform file metadata at the specified location.""" + with tarfile.open(original_path) as original_archive: + with tempfile.TemporaryDirectory() as temp_dir: + tar_file = pathlib.Path(temp_dir) / "sdist.tar" + + with tarfile.open(tar_file, mode="w") as tar_archive: + for original_info in original_archive.getmembers(): # type: tarfile.TarInfo + tar_archive.addfile(create_reproducible_tar_info(original_info, mtime), original_archive.extractfile(original_info)) + + with tar_file.open("rb") as tar_archive: + with gzip.GzipFile(output_path, "wb", mtime=mtime) as output_archive: + shutil.copyfileobj(tar_archive, output_archive) + + +def create_reproducible_tar_info(original: tarfile.TarInfo, mtime: int) -> tarfile.TarInfo: + """Return a copy of the given TarInfo with uniform file metadata.""" + sanitized = tarfile.TarInfo() + sanitized.name = original.name + sanitized.size = original.size + sanitized.mtime = mtime + sanitized.mode = (original.mode & ~(stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)) | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR + sanitized.type = original.type + sanitized.linkname = original.linkname + sanitized.uid = 0 + sanitized.gid = 0 + sanitized.uname = "root" + sanitized.gname = "root" + + if original.mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH): + sanitized.mode |= stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + + return sanitized + + def test_built_artifact(path: pathlib.Path) -> None: """Test the specified built artifact by installing it in a venv and running some basic commands.""" with tempfile.TemporaryDirectory() as temp_dir_name: @@ -823,6 +860,12 @@ def get_release_artifact_details(repository: str, version: Version, validate: bo artifacts = [describe_release_artifact(version, item, validate) for item in data["urls"]] + expected_artifact_types = {"bdist_wheel", "sdist"} + found_artifact_types = set(artifact.package_type for artifact in artifacts) + + if found_artifact_types != expected_artifact_types: + raise RuntimeError(f"Expected {expected_artifact_types} artifact types, but found {found_artifact_types} instead.") + return artifacts @@ -1228,12 +1271,18 @@ def build(allow_dirty: bool = False) -> None: temp_dir = pathlib.Path(temp_dir_name) dist_dir = temp_dir / "dist" + commit_time = int(git("show", "-s", "--format=%ct", capture_output=True).stdout) + + env.update( + SOURCE_DATE_EPOCH=str(commit_time), + ) + git("worktree", "add", "-d", temp_dir) try: run("python", "-m", "build", "--config-setting=--build-manpages", env=env, cwd=temp_dir) - get_sdist_path(version, dist_dir).rename(sdist_file) + create_reproducible_sdist(get_sdist_path(version, dist_dir), sdist_file, commit_time) get_wheel_path(version, dist_dir).rename(wheel_file) finally: git("worktree", "remove", temp_dir) -- cgit v1.2.1