diff --git a/.github/workflows/build-cross-action.yml b/.github/workflows/build-cross-action.yml index 7d660658..096bc95d 100644 --- a/.github/workflows/build-cross-action.yml +++ b/.github/workflows/build-cross-action.yml @@ -23,6 +23,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' host: - x86_64 - aarch64 @@ -116,6 +117,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - x86_64 @@ -178,6 +180,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - amd64 - x86 diff --git a/.github/workflows/build-native-action.yml b/.github/workflows/build-native-action.yml index 017bcd59..aefe1eed 100644 --- a/.github/workflows/build-native-action.yml +++ b/.github/workflows/build-native-action.yml @@ -42,6 +42,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' host: - x86_64 - aarch64 @@ -131,6 +132,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - x86_64 outputs: @@ -201,6 +203,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - arm64 outputs: @@ -269,6 +272,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - amd64 - x86 diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml index 23c97e7f..37a2ea7b 100644 --- a/.github/workflows/github-release.yml +++ b/.github/workflows/github-release.yml @@ -87,6 +87,9 @@ jobs: - platform: linux-gnu arch: x86_64 python: '3.13' + - platform: linux-gnu + arch: x86_64 + python: '3.14' - platform: linux-gnu arch: aarch64 python: '3.10' @@ -99,6 +102,9 @@ jobs: - platform: linux-gnu arch: aarch64 python: '3.13' + - platform: linux-gnu + arch: aarch64 + python: '3.14' - platform: win arch: x86 @@ -112,6 +118,9 @@ jobs: - platform: win arch: x86 python: '3.13' + - platform: win + arch: x86 + python: '3.14' - platform: win arch: amd64 python: '3.10' @@ -124,6 +133,9 @@ jobs: - platform: win arch: amd64 python: '3.13' + - platform: win + arch: amd64 + python: '3.14' - platform: macos arch: x86_64 @@ -137,6 +149,9 @@ jobs: - platform: macos arch: x86_64 python: '3.13' + - platform: macos + arch: x86_64 + python: '3.14' - platform: macos arch: arm64 python: '3.10' @@ -149,6 +164,9 @@ jobs: - platform: macos arch: arm64 python: '3.13' + - platform: macos + arch: arm64 + python: '3.14' steps: - uses: actions/checkout@v3 - name: Set up Python 3.11 diff --git a/.github/workflows/test-fips-action.yml b/.github/workflows/test-fips-action.yml index 7b88ec11..0fcc483e 100644 --- a/.github/workflows/test-fips-action.yml +++ b/.github/workflows/test-fips-action.yml @@ -18,6 +18,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - x86_64 outputs: diff --git a/.github/workflows/verify-build-action.yml b/.github/workflows/verify-build-action.yml index 4540a63e..558ac7ff 100644 --- a/.github/workflows/verify-build-action.yml +++ b/.github/workflows/verify-build-action.yml @@ -19,6 +19,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' host: - x86_64 - aarch64 @@ -94,6 +95,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - x86_64 @@ -154,6 +156,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - arm64 @@ -219,6 +222,7 @@ jobs: - '3.11' - '3.12' - '3.13' + - '3.14' arch: - amd64 - x86 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1fb041d5..d9617067 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,23 +15,12 @@ repos: exclude: setup\.py entry: python3 .pre-commit-hooks/copyright_headers.py language: system -- repo: https://github.com/timothycrosley/isort - rev: 5.12.0 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.9.10 hooks: - - id: isort -- repo: https://github.com/psf/black - rev: 22.6.0 - hooks: - - id: black -- repo: https://github.com/pycqa/flake8 - rev: 5.0.4 - hooks: - - id: flake8 - exclude: ^(\.pre-commit-hooks/.*\.py)$ - additional_dependencies: - - flake8-mypy-fork - - flake8-docstrings - - flake8-typing-imports + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.11.1 hooks: diff --git a/.pre-commit-hooks/check_changelog_entries.py b/.pre-commit-hooks/check_changelog_entries.py index ce77d0e2..3db7ae2e 100644 --- a/.pre-commit-hooks/check_changelog_entries.py +++ b/.pre-commit-hooks/check_changelog_entries.py @@ -25,7 +25,6 @@ def check_changelog_entries(files): - exitcode = 0 for entry in files: path = pathlib.Path(entry).resolve() @@ -40,13 +39,10 @@ def check_changelog_entries(files): # Does it end in .rst if path.suffix != ".rst": exitcode = 1 - print( - "The changelog entry '{}' should have '.rst' as it's file extension".format( - path.relative_to(CODE_ROOT), - ), - file=sys.stderr, - flush=True, + msg = ( + f"The changelog entry '{path.relative_to(CODE_ROOT)}' should have '.rst' as it's file extension" ) + print(msg, file=sys.stderr, flush=True) continue print( "The changelog entry '{}' should have one of the following extensions: {}.".format( @@ -65,19 +61,14 @@ def check_changelog_entries(files): if CHANGELOG_ENTRY_RE.match(path.name): # So, this IS a changelog entry, but it's misplaced.... exitcode = 1 - print( - "The changelog entry '{}' should be placed under '{}/', not '{}'".format( - path.relative_to(CODE_ROOT), - CHANGELOG_ENTRIES_PATH.relative_to(CODE_ROOT), - path.relative_to(CODE_ROOT).parent, - ), - file=sys.stderr, - flush=True, + msg = ( + f"The changelog entry '{path.relative_to(CODE_ROOT)}' " + f"should be placed under '{CHANGELOG_ENTRIES_PATH.relative_to(CODE_ROOT)}/', " + f"not '{path.relative_to(CODE_ROOT).parent}'" ) + print(msg, file=sys.stderr, flush=True) continue - elif CHANGELOG_LIKE_RE.match(path.name) and not CHANGELOG_ENTRY_RE.match( - path.name - ): + elif CHANGELOG_LIKE_RE.match(path.name) and not CHANGELOG_ENTRY_RE.match(path.name): # Does it look like a changelog entry print( "The changelog entry '{}' should have one of the following extensions: {}.".format( @@ -90,18 +81,14 @@ def check_changelog_entries(files): exitcode = 1 continue - elif not CHANGELOG_LIKE_RE.match( - path.name - ) and not CHANGELOG_ENTRY_RE.match(path.name): + elif not CHANGELOG_LIKE_RE.match(path.name) and not CHANGELOG_ENTRY_RE.match(path.name): # Does not look like, and it's not a changelog entry continue # Does it end in .rst if path.suffix != ".rst": exitcode = 1 print( - "The changelog entry '{}' should have '.rst' as it's file extension".format( - path.relative_to(CODE_ROOT), - ), + f"The changelog entry '{path.relative_to(CODE_ROOT)}' should have '.rst' as it's file extension", file=sys.stderr, flush=True, ) @@ -111,21 +98,22 @@ def check_changelog_entries(files): def check_changelog_entry_contents(entry): contents = entry.read_text().splitlines() if len(contents) > 1: - # More than one line. - # If the second line starts with '*' it's a bullet list and we need to add an - # empty line before it. - if contents[1].strip().startswith("*"): - contents.insert(1, "") - entry.write_text("{}\n".format("\n".join(contents))) + if contents[1].strip() and contents[1].strip()[0] not in ("-", "=", "~", "^", "*", "+", "#", "<", ">"): + # This is not a heading + print( + f"The changelog entry '{entry.relative_to(CODE_ROOT)}' should have a heading.", + file=sys.stderr, + flush=True, + ) + sys.exit(1) def main(argv): parser = argparse.ArgumentParser(prog=__name__) parser.add_argument("files", nargs="+") - - options = parser.parse_args(argv) - return check_changelog_entries(options.files) + args = parser.parse_args(argv) + return check_changelog_entries(args.files) if __name__ == "__main__": - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv[1:])) diff --git a/.pre-commit-hooks/copyright_headers.py b/.pre-commit-hooks/copyright_headers.py index 560fe8e2..14928913 100644 --- a/.pre-commit-hooks/copyright_headers.py +++ b/.pre-commit-hooks/copyright_headers.py @@ -12,9 +12,7 @@ CODE_ROOT = pathlib.Path(__file__).resolve().parent.parent SPDX_HEADER = "# SPDX-License-Identifier: Apache-2.0" COPYRIGHT_HEADER = "# Copyright {year} Broadcom." -COPYRIGHT_REGEX = re.compile( - r"# Copyright (?:(?P[0-9]{4})(?:-(?P[0-9]{4}))?) Broadcom\." -) +COPYRIGHT_REGEX = re.compile(r"# Copyright (?:(?P[0-9]{4})(?:-(?P[0-9]{4}))?) Broadcom\.") SPDX_REGEX = re.compile(r"# SPDX-License-Identifier:.*") diff --git a/noxfile.py b/noxfile.py index 8a57df93..4d9adcce 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,6 +3,7 @@ """ Nox session definitions. """ + import datetime import os import pathlib diff --git a/pyproject.toml b/pyproject.toml index d845e44f..0977a438 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,44 @@ ensure_newline_before_comments=true [tool.pylint] max-line-length=120 +[tool.ruff] +line-length = 120 +target-version = "py310" + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "D", # pydocstyle + "I", # isort + "UP", # pyupgrade + "TCH", # flake8-type-checking +] +ignore = [ + "D100", # Missing docstring in public module + "D101", # Missing docstring in public class + "D102", # Missing docstring in public method + "D103", # Missing docstring in public function + "D104", # Missing docstring in public package + "D105", # Missing docstring in magic method + "D107", # Missing docstring in __init__ + "D200", # One-line docstring should fit on one line with quotes + "D205", # 1 blank line required between summary line and description + "D212", # Multi-line docstring summary should start at the first line + "D401", # First line should be in imperative mood + "D415", # First line should end with a period, question mark, or exclamation point + "F403", # 'from import *' used + "F405", # '*' may be undefined, or defined from star imports +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] +"noxfile.py" = ["E501"] + +[tool.ruff.lint.pydocstyle] +convention = "pep257" + [tool.mypy] python_version = "3.10" explicit_package_bases = true diff --git a/relenv-pyconfig-fix.md b/relenv-pyconfig-fix.md new file mode 100644 index 00000000..9b9f89fc --- /dev/null +++ b/relenv-pyconfig-fix.md @@ -0,0 +1,193 @@ +# relenv: Windows onedir for Python 3.13+ is missing `pyconfig.h` + +## Repository / branch + +`saltstack/relenv`, current `main` (and all releases `v0.22.7`–`v0.22.8`). + +## Symptom + +Any C extension built against a relenv-produced Windows onedir for Python +3.13.x fails with: + +``` +D:\path\to\onedir\include\Python.h(14): fatal error C1083: + Cannot open include file: 'pyconfig.h': No such file or directory +``` + +Reproduced on the Salt CI run that triggered this report: + + +The failures show up while pip tries to build wheels for `psutil`, +`timelib`, `markupsafe._speedups`, etc. against a Salt onedir that uses +relenv's `3.13.13-amd64-win.tar.xz` / `3.13.13-x86-win.tar.xz`. + +## Direct evidence + +`tar tJf 3.13.13-amd64-win.tar.xz | grep -i pyconfig` → no matches. +`tar tJf 3.12.13-amd64-win.tar.xz | grep -i pyconfig` → `Include/pyconfig.h`. + +So the published 3.13.x Windows tarballs simply do not contain +`Include/pyconfig.h`. The 3.12.x and earlier tarballs do. + +## Root cause + +In `relenv/build/windows.py`, around the `build_python` step, the code +that populates `/Include` is: + +```python +shutil.copytree( + src=str(dirs.source / "Include"), + dst=str(dirs.prefix / "Include"), + dirs_exist_ok=True, +) +if "3.13" not in env["RELENV_PY_MAJOR_VERSION"]: + shutil.copy( + src=str(dirs.source / "PC" / "pyconfig.h"), + dst=str(dirs.prefix / "Include"), + ) +``` + +For Python ≤ 3.12 there is a checked-in `PC/pyconfig.h` in the CPython +source tree. The `shutil.copy` above places it next to `Python.h` in the +onedir — without it, `Python.h` cannot find the configuration macros. + +In Python 3.13 the layout changed: + +* `PC/pyconfig.h` was deleted from the source tree. +* `PC/pyconfig.h.in` is now a template. +* MSBuild generates the real `pyconfig.h` into the build output + directory (`PCbuild\\pyconfig.h`). + +Confirm via the upstream API: + +``` +gh api repos/python/cpython/contents/PC/pyconfig.h?ref=v3.13.13 # 404 +gh api repos/python/cpython/contents/PC?ref=v3.13.13 # only pyconfig.h.in +``` + +Commit `842b42eb` ("Attempt to fix 3.13 windows build", 2024-10-21) added +the `if "3.13" not in ...` guard to stop the failing `shutil.copy(.../PC/pyconfig.h)` +on 3.13 — but never replaced it with logic that copies the *generated* +`pyconfig.h` out of the build directory. The result is a tarball with +`Python.h` but no `pyconfig.h`, which is unusable for compiling C +extensions. + +CPython's own packaging script handles the same fork at +`PC/layout/main.py` (in 3.13.13): + +```python +pc = ns.source / "PC" +if (pc / "pyconfig.h.in").is_file(): + yield "include/pyconfig.h", ns.build / "pyconfig.h" # 3.13+ +else: + yield "include/pyconfig.h", pc / "pyconfig.h" # ≤3.12 +``` + +That's the model relenv should follow. + +## Fix (proposed) + +Replace the existing `if`-block in `relenv/build/windows.py` with one +that picks the source vs. build location based on whether `PC/pyconfig.h.in` +exists. Concretely: + +```python +shutil.copytree( + src=str(dirs.source / "Include"), + dst=str(dirs.prefix / "Include"), + dirs_exist_ok=True, +) + +# Locate pyconfig.h. Python <= 3.12 ships a checked-in PC/pyconfig.h. +# Python 3.13+ replaced that with PC/pyconfig.h.in and MSBuild generates +# the real header into the build output directory. Mirror the logic in +# CPython's PC/layout/main.py. +pc_dir = dirs.source / "PC" +if (pc_dir / "pyconfig.h.in").is_file(): + pyconfig_src = build_dir / "pyconfig.h" +else: + pyconfig_src = pc_dir / "pyconfig.h" + +if not pyconfig_src.is_file(): + raise RuntimeError( + f"Expected pyconfig.h at {pyconfig_src}; CPython build did not " + "produce it. Check that the MSBuild step ran successfully." + ) + +shutil.copy(src=str(pyconfig_src), dst=str(dirs.prefix / "Include")) +``` + +Notes for the implementing agent: + +* `build_dir` is already in scope a few lines below — it is the variable + used to locate `python3.lib` / `python.lib` / `*.pyd` / `*.dll`. + Move the new code below the variable's definition or pass it in. +* The exact path inside the build output may need adjustment per arch + (`PCbuild\amd64\pyconfig.h` vs `PCbuild\win32\pyconfig.h`). On + inspection, MSBuild copies the final `pyconfig.h` to `$(BinaryOutputPath)`, + and other relenv code (e.g. the `*.pyd`/`*.dll` glob and `python3.lib` + copy) already references that same `build_dir`, so the same path is + the right one. Verify by listing `build_dir` for both arches mid-build. +* Do **not** restore the old unconditional copy from `PC/pyconfig.h` — + that path no longer exists on 3.13+ and will raise `FileNotFoundError`. +* Keep the `shutil.copytree(... / "Include", ...)` call as-is; the new + copy still needs to land *after* it so the generated header overwrites + any stale one inadvertently picked up. + +## Verification + +1. Build the affected onedirs locally: + ``` + relenv build --arch amd64 --python 3.13.13 + relenv build --arch x86 --python 3.13.13 + relenv build --arch amd64 --python 3.12.13 # regression check + ``` +2. Confirm `Include/pyconfig.h` exists in each resulting `` and + in the `*.tar.xz` produced by the `relenv-finalize` step: + ``` + tar tJf 3.13.13-amd64-win.tar.xz | grep -i pyconfig + tar tJf 3.13.13-x86-win.tar.xz | grep -i pyconfig + tar tJf 3.12.13-amd64-win.tar.xz | grep -i pyconfig + ``` + All three should print `Include/pyconfig.h`. +3. From the extracted onedir, build any C extension that includes + `Python.h`: + ``` + \Scripts\python.exe -m pip install --no-binary :all: psutil + ``` + This should succeed, which it currently does not on 3.13. + +## Suggested test + +Add a smoke check to relenv's CI that, for each Windows tarball produced, +asserts the presence of `Include/pyconfig.h`. A trivial post-build step: + +```python +import tarfile, sys +with tarfile.open(sys.argv[1]) as tf: + names = tf.getnames() +assert any(n.lower().endswith("include/pyconfig.h") for n in names), ( + f"{sys.argv[1]} is missing Include/pyconfig.h" +) +``` + +Run it against every `*-win.tar.xz` artifact. This would have caught the +regression introduced by `842b42eb` immediately. + +## Release / consumer impact + +* Salt's pyversion101 branch (Python 3.13.13) cannot produce a working + Windows onedir with relenv `0.22.7` or `0.22.8`. Once a relenv release + carrying this fix is cut (call it `0.22.9`), bump + `cicd/shared-gh-workflows-context.yml` `relenv_version` in salt and + re-run the onedir build job to confirm. +* No expected impact on Linux / macOS builds — the affected code path is + Windows-only and the equivalent POSIX builds already ship a generated + `pyconfig.h` correctly. + +## Out of scope + +* Same Salt CI run also shows a macOS arm64 onedir failure (`yaml.h + not found` while building PyYAML's C speedups, and a `cmake_minimum_required` + failure when pyzmq tries to build its bundled libzmq). Those are + separate problems and are **not** addressed by this fix. diff --git a/relenv/__main__.py b/relenv/__main__.py index ef72d5e7..7ecfab3f 100644 --- a/relenv/__main__.py +++ b/relenv/__main__.py @@ -8,11 +8,14 @@ import argparse from argparse import ArgumentParser -from types import ModuleType +from typing import TYPE_CHECKING from . import build, buildenv, check, create, fetch, pyversions, toolchain from .common import __version__ +if TYPE_CHECKING: + from types import ModuleType + def setup_cli() -> ArgumentParser: """ @@ -29,9 +32,7 @@ def setup_cli() -> ArgumentParser: description="Relenv", ) argparser.add_argument("--version", action="version", version=__version__) - subparsers: argparse._SubParsersAction[ - argparse.ArgumentParser - ] = argparser.add_subparsers() + subparsers: argparse._SubParsersAction[argparse.ArgumentParser] = argparser.add_subparsers() modules_to_setup: list[ModuleType] = [ build, diff --git a/relenv/build/__init__.py b/relenv/build/__init__.py index 48a254d3..0528e20b 100644 --- a/relenv/build/__init__.py +++ b/relenv/build/__init__.py @@ -4,17 +4,15 @@ """ Entry points for the ``relenv build`` CLI command. """ + from __future__ import annotations -import argparse import codecs import random import signal import sys -from types import FrameType, ModuleType +from typing import TYPE_CHECKING -from . import darwin, linux, windows -from .common import builds from ..common import build_arch from ..pyversions import ( Version, @@ -22,6 +20,12 @@ python_versions, resolve_python_version, ) +from . import darwin, linux, windows +from .common import builds + +if TYPE_CHECKING: + import argparse + from types import FrameType, ModuleType def platform_module() -> ModuleType: @@ -47,9 +51,7 @@ def setup_parser( :type subparsers: argparse._SubParsersAction """ mod = platform_module() - build_subparser = subparsers.add_parser( - "build", description="Build Relenv Python Environments from source" - ) + build_subparser = subparsers.add_parser("build", description="Build Relenv Python Environments from source") build_subparser.set_defaults(func=main) build_subparser.add_argument( "--arch", @@ -62,17 +64,14 @@ def setup_parser( "--clean", default=False, action="store_true", - help=( - "Clean up before running the build. This option will remove the " - "logs, src, build, and previous tarball." - ), + help=("Clean up before running the build. This option will remove the logs, src, build, and previous tarball."), ) default_version = get_default_python_version() build_subparser.add_argument( "--python", default=default_version, type=str, - help="The python version (e.g., 3.10, 3.13.7) [default: %(default)s]", + help="The python version (e.g., 3.10, 3.14.4) [default: %(default)s]", ) build_subparser.add_argument( "--no-cleanup", @@ -174,9 +173,7 @@ def main(args: argparse.Namespace) -> None: build.set_arch(args.arch) if build.build_arch != build.arch: - print( - "Warning: Cross compilation support is experimental and is not fully tested or working!" - ) + print("Warning: Cross compilation support is experimental and is not fully tested or working!") steps = None if args.steps: steps = [_.strip() for _ in args.steps] diff --git a/relenv/build/common/__init__.py b/relenv/build/common/__init__.py index ad85bd85..9c772f55 100644 --- a/relenv/build/common/__init__.py +++ b/relenv/build/common/__init__.py @@ -6,30 +6,28 @@ This module has been split into focused submodules for better organization. All public APIs are re-exported here for backward compatibility. """ + from __future__ import annotations +from .builder import ( + Dirs, + builds, + get_dependency_version, +) from .builders import ( build_openssl, build_openssl_fips, build_sqlite, ) - from .install import ( - update_ensurepip, - install_runtime, - finalize, create_archive, + finalize, + install_runtime, patch_file, + update_ensurepip, update_sbom_checksums, ) -from .builder import ( - Dirs, - builds, - get_dependency_version, -) - - __all__ = [ # Builder classes and instances "Dirs", diff --git a/relenv/build/common/builder.py b/relenv/build/common/builder.py index c4892310..5565ac72 100644 --- a/relenv/build/common/builder.py +++ b/relenv/build/common/builder.py @@ -3,62 +3,60 @@ """ Builder and Builds classes for managing the build process. """ + from __future__ import annotations -import io import json import logging import multiprocessing import os -import pathlib import shutil import sys +import tempfile import time from typing import ( - Any, - Callable, - Dict, IO, - List, - MutableMapping, - Optional, - Sequence, + TYPE_CHECKING, + Any, TypedDict, - Union, cast, ) -import tempfile from relenv.common import ( DATA_DIR, MODULE_DIR, ConfigurationError, + Version, + WorkDirs, build_arch, extract_archive, get_toolchain, get_triplet, work_dirs, - WorkDirs, ) +from .builders import build_default as _default_build_func from .download import Download from .ui import ( + BuildStats, LineCountHandler, load_build_stats, print_ui, print_ui_expanded, update_build_stats, - BuildStats, ) -from .builders import build_default as _default_build_func + +if TYPE_CHECKING: + import pathlib + from collections.abc import Callable, MutableMapping, Sequence # Type alias for path-like objects -PathLike = Union[str, os.PathLike[str]] +PathLike = str | os.PathLike[str] log = logging.getLogger(__name__) -def _default_populate_env(env: MutableMapping[str, str], dirs: "Dirs") -> None: +def _default_populate_env(env: MutableMapping[str, str], dirs: Dirs) -> None: """Default populate_env implementation (does nothing). This default implementation intentionally does nothing; specific steps may @@ -68,7 +66,7 @@ def _default_populate_env(env: MutableMapping[str, str], dirs: "Dirs") -> None: _ = dirs -def get_dependency_version(name: str, platform: str) -> Optional[Dict[str, str]]: +def get_dependency_version(name: str, platform: str) -> dict[str, str] | None: """ Get dependency version and metadata from python-versions.json. @@ -139,11 +137,11 @@ def __init__(self, dirs: WorkDirs, name: str, arch: str, version: str) -> None: self.downloads = dirs.download self.logs = dirs.logs self.sources = dirs.src - self.tmpbuild = tempfile.mkdtemp(prefix="{}_build".format(name)) - self.source: Optional[pathlib.Path] = None + self.tmpbuild = tempfile.mkdtemp(prefix=f"{name}_build") + self.source: pathlib.Path | None = None @property - def toolchain(self) -> Optional[pathlib.Path]: + def toolchain(self) -> pathlib.Path | None: """Get the toolchain directory path for the current platform.""" if sys.platform == "darwin": return get_toolchain(root=self.root) @@ -155,18 +153,18 @@ def toolchain(self) -> Optional[pathlib.Path]: @property def _triplet(self) -> str: if sys.platform == "darwin": - return "{}-macos".format(self.arch) + return f"{self.arch}-macos" elif sys.platform == "win32": - return "{}-win".format(self.arch) + return f"{self.arch}-win" else: - return "{}-linux-gnu".format(self.arch) + return f"{self.arch}-linux-gnu" @property def prefix(self) -> pathlib.Path: """Get the build prefix directory path.""" return self.build / f"{self.version}-{self._triplet}" - def __getstate__(self) -> Dict[str, Any]: + def __getstate__(self) -> dict[str, Any]: """ Return an object used for pickling. @@ -183,7 +181,7 @@ def __getstate__(self) -> Dict[str, Any]: "tmpbuild": self.tmpbuild, } - def __setstate__(self, state: Dict[str, Any]) -> None: + def __setstate__(self, state: dict[str, Any]) -> None: """ Unwrap the object returned from unpickling. @@ -199,7 +197,7 @@ def __setstate__(self, state: Dict[str, Any]) -> None: self.build = state["build"] self.tmpbuild = state["tmpbuild"] - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Get a dictionary representation of the directories in this collection. @@ -224,8 +222,8 @@ class Recipe(TypedDict): """Typed description of a build recipe entry.""" build_func: Callable[[MutableMapping[str, str], Dirs, IO[str]], None] - wait_on: List[str] - download: Optional[Download] + wait_on: list[str] + download: Download | None class Builder: @@ -248,12 +246,10 @@ class Builder: def __init__( self, - root: Optional[PathLike] = None, - recipies: Optional[Dict[str, Recipe]] = None, - build_default: Optional[ - Callable[[MutableMapping[str, str], Dirs, IO[str]], None] - ] = None, - populate_env: Optional[Callable[[MutableMapping[str, str], Dirs], None]] = None, + root: PathLike | None = None, + recipies: dict[str, Recipe] | None = None, + build_default: Callable[[MutableMapping[str, str], Dirs, IO[str]], None] | None = None, + populate_env: Callable[[MutableMapping[str, str], Dirs], None] | None = None, arch: str = "x86_64", version: str = "", ) -> None: @@ -266,14 +262,14 @@ def __init__( self.downloads = self.dirs.download if recipies is None: - self.recipies: Dict[str, Recipe] = {} + self.recipies: dict[str, Recipe] = {} else: self.recipies = recipies # Use dependency injection with sensible defaults - self.build_default: Callable[ - [MutableMapping[str, str], Dirs, IO[str]], None - ] = (build_default if build_default is not None else _default_build_func) + self.build_default: Callable[[MutableMapping[str, str], Dirs, IO[str]], None] = ( + build_default if build_default is not None else _default_build_func + ) # Use the default populate_env if none provided self.populate_env: Callable[[MutableMapping[str, str], Dirs], None] = ( @@ -283,9 +279,9 @@ def __init__( self.version = version self.set_arch(self.arch) - def copy(self, version: str, checksum: Optional[str]) -> "Builder": + def copy(self, version: str, checksum: str | None) -> Builder: """Create a copy of this Builder with a different version.""" - recipies: Dict[str, Recipe] = {} + recipies: dict[str, Recipe] = {} for name in self.recipies: recipe = self.recipies[name] recipies[name] = { @@ -305,6 +301,10 @@ def copy(self, version: str, checksum: Optional[str]) -> "Builder": if python_download is None: raise ConfigurationError("Python recipe is missing a download entry") python_download.version = version + if checksum is None: + from relenv.pyversions import python_versions + + checksum = python_versions().get(Version(version)) python_download.checksum = checksum return build @@ -316,10 +316,10 @@ def set_arch(self, arch: str) -> None: :type arch: str """ self.arch = arch - self._toolchain: Optional[pathlib.Path] = None + self._toolchain: pathlib.Path | None = None @property - def toolchain(self) -> Optional[pathlib.Path]: + def toolchain(self) -> pathlib.Path | None: """Lazily fetch toolchain only when needed.""" if self._toolchain is None and sys.platform == "linux": from relenv.common import get_toolchain @@ -340,18 +340,18 @@ def prefix(self) -> pathlib.Path: @property def _triplet(self) -> str: if sys.platform == "darwin": - return "{}-macos".format(self.arch) + return f"{self.arch}-macos" elif sys.platform == "win32": - return "{}-win".format(self.arch) + return f"{self.arch}-win" else: - return "{}-linux-gnu".format(self.arch) + return f"{self.arch}-linux-gnu" def add( self, name: str, - build_func: Optional[Callable[..., Any]] = None, - wait_on: Optional[Sequence[str]] = None, - download: Optional[Dict[str, Any]] = None, + build_func: Callable[..., Any] | None = None, + wait_on: Sequence[str] | None = None, + download: dict[str, Any] | None = None, ) -> None: """ Add a step to the build process. @@ -366,12 +366,12 @@ def add( :type download: dict, optional """ if wait_on is None: - wait_on_list: List[str] = [] + wait_on_list: list[str] = [] else: wait_on_list = list(wait_on) if build_func is None: build_func = self.build_default - download_obj: Optional[Download] = None + download_obj: Download | None = None if download is not None: download_obj = Download(name, destination=self.downloads, **download) self.recipies[name] = { @@ -383,12 +383,12 @@ def add( def run( self, name: str, - event: "multiprocessing.synchronize.Event", + event: multiprocessing.synchronize.Event, build_func: Callable[..., Any], - download: Optional[Download], + download: Download | None, show_ui: bool = False, log_level: str = "WARNING", - line_counts: Optional[MutableMapping[str, int]] = None, + line_counts: MutableMapping[str, int] | None = None, ) -> Any: """ Run a build step. @@ -428,13 +428,13 @@ def run( while event.is_set() is False: time.sleep(0.3) - logfp = io.open(os.path.join(dirs.logs, "{}.log".format(name)), "w") + logfp = open(os.path.join(dirs.logs, f"{name}.log"), "w") file_handler = logging.FileHandler(dirs.logs / f"{name}.log") root_log.addHandler(file_handler) root_log.setLevel(logging.NOTSET) # Add line count handler if tracking is enabled - line_count_handler: Optional[LineCountHandler] = None + line_count_handler: LineCountHandler | None = None if line_counts is not None: line_count_handler = LineCountHandler(name, line_counts) root_log.addHandler(line_count_handler) @@ -526,7 +526,7 @@ def clean(self) -> None: def download_files( self, - steps: Optional[Sequence[str]] = None, + steps: Sequence[str] | None = None, force_download: bool = False, show_ui: bool = False, expanded_ui: bool = False, @@ -541,14 +541,14 @@ def download_files( """ step_names = list(steps) if steps is not None else list(self.recipies) - fails: List[str] = [] - processes: Dict[str, multiprocessing.Process] = {} - events: Dict[str, Any] = {} + fails: list[str] = [] + processes: dict[str, multiprocessing.Process] = {} + events: dict[str, Any] = {} # For downloads, we don't track line counts but can still use expanded UI format manager = multiprocessing.Manager() line_counts: MutableMapping[str, int] = manager.dict() - build_stats: Dict[str, BuildStats] = {} + build_stats: dict[str, BuildStats] = {} if show_ui: if not expanded_ui: @@ -574,15 +574,13 @@ def progress_callback(downloaded: int, total: int) -> None: return progress_callback - download_kwargs: Dict[str, Any] = { + download_kwargs: dict[str, Any] = { "force_download": force_download, "show_ui": show_ui, "exit_on_failure": True, } if expanded_ui: - download_kwargs["progress_callback"] = make_progress_callback( - name, line_counts - ) + download_kwargs["progress_callback"] = make_progress_callback(name, line_counts) proc = multiprocessing.Process( name=name, @@ -615,9 +613,7 @@ def progress_callback(downloaded: int, total: int) -> None: fails.append(proc.name) if show_ui: if expanded_ui: - print_ui_expanded( - events, processes, fails, line_counts, build_stats, "download" - ) + print_ui_expanded(events, processes, fails, line_counts, build_stats, "download") else: print_ui(events, processes, fails) sys.stdout.write("\n") @@ -632,7 +628,7 @@ def progress_callback(downloaded: int, total: int) -> None: def build( self, - steps: Optional[Sequence[str]] = None, + steps: Sequence[str] | None = None, cleanup: bool = True, show_ui: bool = False, log_level: str = "WARNING", @@ -648,15 +644,15 @@ def build( :param expanded_ui: Whether to use expanded UI with progress bars :type expanded_ui: bool, optional """ # noqa: D400 - fails: List[str] = [] - events: Dict[str, Any] = {} - waits: Dict[str, List[str]] = {} - processes: Dict[str, multiprocessing.Process] = {} + fails: list[str] = [] + events: dict[str, Any] = {} + waits: dict[str, list[str]] = {} + processes: dict[str, multiprocessing.Process] = {} # Set up shared line counts and load build stats for expanded UI manager = multiprocessing.Manager() line_counts: MutableMapping[str, int] = manager.dict() - build_stats: Dict[str, BuildStats] = {} + build_stats: dict[str, BuildStats] = {} if expanded_ui: build_stats = load_build_stats() @@ -682,7 +678,7 @@ def build( kwargs["line_counts"] = line_counts # Determine needed dependency recipies. - wait_on_seq = cast(List[str], kwargs.pop("wait_on", [])) + wait_on_seq = cast("list[str]", kwargs.pop("wait_on", [])) wait_on_list = list(wait_on_seq) for dependency in wait_on_list[:]: if dependency not in step_names: @@ -692,9 +688,7 @@ def build( if not waits[name]: event.set() - proc = multiprocessing.Process( - name=name, target=self.run, args=(name, event), kwargs=kwargs - ) + proc = multiprocessing.Process(name=name, target=self.run, args=(name, event), kwargs=kwargs) proc.start() processes[name] = proc @@ -706,9 +700,7 @@ def build( if show_ui: # DEBUG: Comment to debug if expanded_ui: - print_ui_expanded( - events, processes, fails, line_counts, build_stats, "build" - ) + print_ui_expanded(events, processes, fails, line_counts, build_stats, "build") else: print_ui(events, processes, fails) if proc.exitcode is None: @@ -735,7 +727,7 @@ def build( for fail in fails: log_file = self.dirs.logs / f"{fail}.log" try: - with io.open(log_file) as fp: + with open(log_file) as fp: fp.seek(0, 2) end = fp.tell() ind = end - 4096 @@ -760,9 +752,7 @@ def build( if show_ui: time.sleep(0.3) if expanded_ui: - print_ui_expanded( - events, processes, fails, line_counts, build_stats, "build" - ) + print_ui_expanded(events, processes, fails, line_counts, build_stats, "build") else: print_ui(events, processes, fails) sys.stdout.write("\n") @@ -771,7 +761,7 @@ def build( log.debug("Performing cleanup.") self.cleanup() - def check_prereqs(self) -> List[str]: + def check_prereqs(self) -> list[str]: """ Check pre-requsists for build. @@ -780,18 +770,16 @@ def check_prereqs(self) -> List[str]: :return: Returns a list of string describing failed checks :rtype: list """ - fail: List[str] = [] + fail: list[str] = [] if sys.platform == "linux": if not self.toolchain or not self.toolchain.exists(): - fail.append( - f"Toolchain for {self.arch} does not exist. Please pip install ppbt." - ) + fail.append(f"Toolchain for {self.arch} does not exist. Please pip install ppbt.") return fail def __call__( self, - steps: Optional[Sequence[str]] = None, - arch: Optional[str] = None, + steps: Sequence[str] | None = None, + arch: str | None = None, clean: bool = True, cleanup: bool = True, force_download: bool = False, @@ -819,7 +807,7 @@ def __call__( log = logging.getLogger(None) log.setLevel(logging.NOTSET) - stream_handler: Optional[logging.Handler] = None + stream_handler: logging.Handler | None = None if not show_ui: stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.getLevelName(log_level)) @@ -885,17 +873,15 @@ class Builds: def __init__(self) -> None: """Initialize an empty collection of builders.""" - self.builds: Dict[str, Builder] = {} + self.builds: dict[str, Builder] = {} def add(self, platform: str, *args: Any, **kwargs: Any) -> Builder: """Add a builder for a specific platform.""" if "builder" in kwargs: build_candidate = kwargs.pop("builder") if args or kwargs: - raise RuntimeError( - "builder keyword can not be used with other kwargs or args" - ) - build = cast(Builder, build_candidate) + raise RuntimeError("builder keyword can not be used with other kwargs or args") + build = cast("Builder", build_candidate) else: build = Builder(*args, **kwargs) self.builds[platform] = build diff --git a/relenv/build/common/builders.py b/relenv/build/common/builders.py index 12e92645..d52f8e70 100644 --- a/relenv/build/common/builders.py +++ b/relenv/build/common/builders.py @@ -3,16 +3,19 @@ """ Build functions for specific dependencies. """ + from __future__ import annotations import pathlib import shutil import sys -from typing import IO, MutableMapping, TYPE_CHECKING +from typing import IO, TYPE_CHECKING from relenv.common import PlatformError, runcmd if TYPE_CHECKING: + from collections.abc import MutableMapping + from .builder import Dirs @@ -29,7 +32,7 @@ def build_default(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> """ cmd = [ "./configure", - "--prefix={}".format(dirs.prefix), + f"--prefix={dirs.prefix}", ] if env["RELENV_HOST"].find("linux") > -1: cmd += [ @@ -41,9 +44,7 @@ def build_default(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) -def build_openssl_fips( - env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str] -) -> None: +def build_openssl_fips(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> None: """Build OpenSSL with FIPS module.""" return build_openssl(env, dirs, logfp, fips=True) @@ -150,7 +151,7 @@ def build_sqlite(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N "--enable-threadsafe", "--disable-readline", "--disable-dependency-tracking", - "--prefix={}".format(dirs.prefix), + f"--prefix={dirs.prefix}", # "--enable-add-ons=nptl,ports", ] if env["RELENV_HOST"].find("linux") > -1: diff --git a/relenv/build/common/download.py b/relenv/build/common/download.py index 481c8d58..f553551f 100644 --- a/relenv/build/common/download.py +++ b/relenv/build/common/download.py @@ -3,6 +3,7 @@ """ Download utility class for fetching build dependencies. """ + from __future__ import annotations import hashlib @@ -11,19 +12,22 @@ import pathlib import subprocess import sys -from typing import Callable, Optional, Tuple, Union +from typing import TYPE_CHECKING from relenv.common import ( - RelenvException, - ConfigurationError, ChecksumValidationError, + ConfigurationError, + RelenvException, download_url, get_download_location, runcmd, ) +if TYPE_CHECKING: + from collections.abc import Callable + # Type alias for path-like objects -PathLike = Union[str, os.PathLike[str]] +PathLike = str | os.PathLike[str] # Environment flag for CI/CD detection CICD = "CI" in os.environ @@ -31,7 +35,7 @@ log = logging.getLogger(__name__) -def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool: +def verify_checksum(file: PathLike, checksum: str | None) -> bool: """ Verify the checksum of a file. @@ -61,9 +65,7 @@ def verify_checksum(file: PathLike, checksum: Optional[str]) -> bool: hash_algo = hashlib.sha1() hash_name = "sha1" else: - raise ChecksumValidationError( - f"Invalid checksum length {len(checksum)}. Expected 40 (SHA-1) or 64 (SHA-256)" - ) + raise ChecksumValidationError(f"Invalid checksum length {len(checksum)}. Expected 40 (SHA-1) or 64 (SHA-256)") with open(file, "rb") as fp: hash_algo.update(fp.read()) @@ -98,11 +100,11 @@ def __init__( self, name: str, url: str, - fallback_url: Optional[str] = None, - signature: Optional[str] = None, + fallback_url: str | None = None, + signature: str | None = None, destination: PathLike = "", version: str = "", - checksum: Optional[str] = None, + checksum: str | None = None, ) -> None: self.name = name self.url_tpl = url @@ -114,7 +116,7 @@ def __init__( self.version = version self.checksum = checksum - def copy(self) -> "Download": + def copy(self) -> Download: """Create a copy of this Download instance.""" return Download( self.name, @@ -132,7 +134,7 @@ def destination(self) -> pathlib.Path: return self._destination @destination.setter - def destination(self, value: Optional[PathLike]) -> None: + def destination(self, value: PathLike | None) -> None: """Set the destination directory path.""" if value: self._destination = pathlib.Path(value) @@ -145,7 +147,7 @@ def url(self) -> str: return self.url_tpl.format(version=self.version) @property - def fallback_url(self) -> Optional[str]: + def fallback_url(self) -> str | None: """Get the formatted fallback URL if configured.""" if self.fallback_url_tpl: return self.fallback_url_tpl.format(version=self.version) @@ -169,9 +171,7 @@ def formatted_url(self) -> str: """Get the formatted URL (alias for url property).""" return self.url_tpl.format(version=self.version) - def fetch_file( - self, progress_callback: Optional[Callable[[int, int], None]] = None - ) -> Tuple[str, bool]: + def fetch_file(self, progress_callback: Callable[[int, int], None] | None = None) -> tuple[str, bool]: """ Download the file. @@ -205,7 +205,7 @@ def fetch_file( ) raise - def fetch_signature(self, version: Optional[str] = None) -> Tuple[str, bool]: + def fetch_signature(self, version: str | None = None) -> tuple[str, bool]: """ Download the file signature. @@ -228,7 +228,7 @@ def valid_hash(self) -> None: pass @staticmethod - def validate_signature(archive: PathLike, signature: Optional[PathLike]) -> bool: + def validate_signature(archive: PathLike, signature: PathLike | None) -> bool: """ True when the archive's signature is valid. @@ -255,7 +255,7 @@ def validate_signature(archive: PathLike, signature: Optional[PathLike]) -> bool return False @staticmethod - def validate_checksum(archive: PathLike, checksum: Optional[str]) -> bool: + def validate_checksum(archive: PathLike, checksum: str | None) -> bool: """ True when when the archive matches the sha1 hash. @@ -278,7 +278,7 @@ def __call__( force_download: bool = False, show_ui: bool = False, exit_on_failure: bool = False, - progress_callback: Optional[Callable[[int, int], None]] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> bool: """ Downloads the url and validates the signature and sha1 sum. @@ -315,9 +315,7 @@ def __call__( if not valid: log.warning("Checksum did not match %s: %s", self.name, self.checksum) if show_ui: - sys.stderr.write( - f"\nChecksum did not match {self.name}: {self.checksum}\n" - ) + sys.stderr.write(f"\nChecksum did not match {self.name}: {self.checksum}\n") sys.stderr.flush() if exit_on_failure and not valid: sys.exit(1) diff --git a/relenv/build/common/install.py b/relenv/build/common/install.py index aa0e40da..5a984c1a 100644 --- a/relenv/build/common/install.py +++ b/relenv/build/common/install.py @@ -3,11 +3,11 @@ """ Installation and finalization functions for the build process. """ + from __future__ import annotations import fnmatch import hashlib -import io import json import logging import os @@ -18,9 +18,9 @@ import shutil import sys import tarfile -from types import ModuleType -from typing import IO, MutableMapping, Optional, Sequence, Union, TYPE_CHECKING +from typing import IO, TYPE_CHECKING +import relenv.relocate from relenv.common import ( LINUX, MODULE_DIR, @@ -30,13 +30,15 @@ format_shebang, runcmd, ) -import relenv.relocate if TYPE_CHECKING: + from collections.abc import MutableMapping, Sequence + from types import ModuleType + from .builder import Dirs # Type alias for path-like objects -PathLike = Union[str, os.PathLike[str]] +PathLike = str | os.PathLike[str] # Relenv PTH file content for bootstrapping RELENV_PTH = ( @@ -77,9 +79,7 @@ def patch_file(path: PathLike, old: str, new: str) -> None: path.write_text(new_content, encoding="utf-8") -def update_sbom_checksums( - source_dir: PathLike, files_to_update: MutableMapping[str, PathLike] -) -> None: +def update_sbom_checksums(source_dir: PathLike, files_to_update: MutableMapping[str, PathLike]) -> None: """ Update checksums in sbom.spdx.json for modified files. @@ -101,7 +101,7 @@ def update_sbom_checksums( return # Read the SBOM JSON - with open(spdx_json, "r") as f: + with open(spdx_json) as f: data = json.load(f) # Compute checksums for each file @@ -179,7 +179,7 @@ def patch_shebang(path: PathLike, old: str, new: str) -> bool: with open(path, "w") as fp: fp.write(new) fp.write(data) - with open(path, "r") as fp: + with open(path) as fp: data = fp.read() log.info("Patched shebang of %s => %r", path, data) return True @@ -251,7 +251,6 @@ def update_ensurepip(directory: pathlib.Path) -> None: update_pip = False update_setuptools = False for file in bundle_dir.glob("*.whl"): - log.debug("Checking whl: %s", str(file)) if file.name.startswith("pip-"): found_version = file.name.split("-")[1] @@ -265,9 +264,7 @@ def update_ensurepip(directory: pathlib.Path) -> None: found_version = file.name.split("-")[1] log.debug("Found version %s", found_version) if Version(found_version) >= Version(setuptools_version): - log.debug( - "Found correct setuptools version or newer: %s", found_version - ) + log.debug("Found correct setuptools version or newer: %s", found_version) else: file.unlink() update_setuptools = True @@ -307,7 +304,7 @@ def install_sysdata( mod: ModuleType, destfile: PathLike, buildroot: PathLike, - toolchain: Optional[PathLike], + toolchain: PathLike | None, ) -> None: """ Create a Relenv Python environment's sysconfigdata. @@ -357,9 +354,7 @@ def ftoolchain(s: str) -> str: sysconfigdata_code = _load_sysconfigdata_template() with open(destfile, "w", encoding="utf8") as f: - f.write( - "# system configuration generated and used by" " the relenv at runtime\n" - ) + f.write("# system configuration generated and used by the relenv at runtime\n") f.write("_build_time_vars = ") pprint.pprint(data, stream=f) f.write(sysconfigdata_code) @@ -388,7 +383,7 @@ def install_runtime(sitepackages: PathLike) -> None: """ site_dir = pathlib.Path(sitepackages) relenv_pth = site_dir / "relenv.pth" - with io.open(str(relenv_pth), "w") as fp: + with open(str(relenv_pth), "w") as fp: fp.write(RELENV_PTH) # Lay down relenv.runtime, we'll pip install the rest later @@ -404,8 +399,8 @@ def install_runtime(sitepackages: PathLike) -> None: ]: src = MODULE_DIR / name dest = relenv / name - with io.open(src, "r") as rfp: - with io.open(dest, "w") as wfp: + with open(src) as rfp: + with open(dest, "w") as wfp: wfp.write(rfp.read()) @@ -435,7 +430,7 @@ def finalize( # Install relenv-sysconfigdata module libdir = pathlib.Path(dirs.prefix) / "lib" - def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: + def find_pythonlib(libdir: pathlib.Path) -> str | None: for _root, dirs, _files in os.walk(libdir): for entry in dirs: if entry.startswith("python"): @@ -488,9 +483,7 @@ def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: # Fix the shebangs in the scripts python layed down. Order matters. shebangs = [ "#!{}".format(bindir / f"python{env['RELENV_PY_MAJOR_VERSION']}"), - "#!{}".format( - bindir / f"python{env['RELENV_PY_MAJOR_VERSION'].split('.', 1)[0]}" - ), + "#!{}".format(bindir / f"python{env['RELENV_PY_MAJOR_VERSION'].split('.', 1)[0]}"), ] newshebang = format_shebang("/python3") for shebang in shebangs: @@ -513,11 +506,7 @@ def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: if toolchain_path is None: raise MissingDependencyError("Toolchain path is required for linux builds") shutil.copy( - pathlib.Path(toolchain_path) - / env["RELENV_HOST"] - / "sysroot" - / "lib" - / "libstdc++.so.6", + pathlib.Path(toolchain_path) / env["RELENV_HOST"] / "sysroot" / "lib" / "libstdc++.so.6", libdir, ) @@ -529,9 +518,9 @@ def find_pythonlib(libdir: pathlib.Path) -> Optional[str]: format_shebang("../../bin/python3"), ) - def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: + def runpip(pkg: str | os.PathLike[str], upgrade: bool = False) -> None: logfp.write(f"\nRUN PIP {pkg} {upgrade}\n") - target: Optional[pathlib.Path] = None + target: pathlib.Path | None = None python_exe = str(dirs.prefix / "bin" / "python3") if sys.platform == LINUX: if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: @@ -547,7 +536,7 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: if upgrade: cmd.append("--upgrade") if target: - cmd.append("--target={}".format(target)) + cmd.append(f"--target={target}") runcmd(cmd, env=env, stderr=logfp, stdout=logfp) runpip("wheel") @@ -570,7 +559,7 @@ def runpip(pkg: Union[str, os.PathLike[str]], upgrade: bool = False) -> None: # Mac specific, factor this out "*.dylib", ] - archive = f"{ dirs.prefix }.tar.xz" + archive = f"{dirs.prefix}.tar.xz" log.info("Archive is %s", archive) with tarfile.open(archive, mode="w:xz") as fp: create_archive(fp, dirs.prefix, globs, logfp) @@ -580,7 +569,7 @@ def create_archive( tarfp: tarfile.TarFile, toarchive: PathLike, globs: Sequence[str], - logfp: Optional[IO[str]] = None, + logfp: IO[str] | None = None, ) -> None: """ Create an archive. diff --git a/relenv/build/common/ui.py b/relenv/build/common/ui.py index e6d7385e..1bf39f1d 100644 --- a/relenv/build/common/ui.py +++ b/relenv/build/common/ui.py @@ -3,20 +3,19 @@ """ UI and build statistics utilities. """ + from __future__ import annotations import logging import os -import pathlib import sys import threading -from typing import Dict, MutableMapping, Optional, Sequence, cast - -import multiprocessing - -from typing import TYPE_CHECKING, TypedDict +from typing import TYPE_CHECKING, TypedDict, cast if TYPE_CHECKING: + import multiprocessing + import pathlib + from collections.abc import MutableMapping, Sequence from multiprocessing.synchronize import Event as SyncEvent else: SyncEvent = None @@ -25,7 +24,6 @@ from .download import CICD - log = logging.getLogger(__name__) @@ -96,7 +94,7 @@ class SpinnerState: def __init__(self) -> None: """Initialize empty spinner state with thread safety.""" - self._state: Dict[str, int] = {} + self._state: dict[str, int] = {} self._lock = threading.Lock() def get(self, name: str) -> int: @@ -120,7 +118,7 @@ def increment(self, name: str) -> None: with self._lock: self._state[name] = self._state.get(name, 0) + 1 - def reset(self, name: Optional[str] = None) -> None: + def reset(self, name: str | None = None) -> None: """Reset spinner state. Args: @@ -146,10 +144,10 @@ class BuildStats(TypedDict): def print_ui( - events: MutableMapping[str, "multiprocessing.synchronize.Event"], + events: MutableMapping[str, multiprocessing.synchronize.Event], processes: MutableMapping[str, multiprocessing.Process], fails: Sequence[str], - flipstat: Optional[Dict[str, tuple[int, float]]] = None, + flipstat: dict[str, tuple[int, float]] | None = None, ) -> None: """ Prints the UI during the relenv building process. @@ -170,19 +168,19 @@ def print_ui( for name in events: if not events[name].is_set(): # Pending: event not yet started - status = " {}{}".format(YELLOW, SYMBOL_PENDING) + status = f" {YELLOW}{SYMBOL_PENDING}" elif name in processes: # Running: show animated spinner frame_idx = _spinner_state.get(name) % len(SPINNER_FRAMES) spinner = SPINNER_FRAMES[frame_idx] _spinner_state.increment(name) - status = " {}{}".format(GREEN, spinner) + status = f" {GREEN}{spinner}" elif name in fails: # Failed: show error symbol - status = " {}{}".format(RED, SYMBOL_FAILED) + status = f" {RED}{SYMBOL_FAILED}" else: # Success: show success symbol - status = " {}{}".format(GREEN, SYMBOL_SUCCESS) + status = f" {GREEN}{SYMBOL_SUCCESS}" uiline.append(status) uiline.append(" " + END) sys.stdout.write("\r") @@ -191,11 +189,11 @@ def print_ui( def print_ui_expanded( - events: MutableMapping[str, "multiprocessing.synchronize.Event"], + events: MutableMapping[str, multiprocessing.synchronize.Event], processes: MutableMapping[str, multiprocessing.Process], fails: Sequence[str], line_counts: MutableMapping[str, int], - build_stats: Dict[str, BuildStats], + build_stats: dict[str, BuildStats], phase: str = "build", ) -> None: """ @@ -277,11 +275,7 @@ def print_ui_expanded( status_text = f"{phase_action} {progress:3d}%" # Progress bar (20 chars wide) filled = int(progress / 5) # 20 segments = 100% / 5 - bar = ( - "█" * filled + "░" * (20 - filled) - if USE_UNICODE - else ("#" * filled + "-" * (20 - filled)) - ) + bar = "█" * filled + "░" * (20 - filled) if USE_UNICODE else ("#" * filled + "-" * (20 - filled)) progress_bar = f" [{bar}]" else: status_text = phase_action @@ -295,11 +289,7 @@ def print_ui_expanded( # Progress bar (20 chars wide) filled = int(progress / 5) # 20 segments = 100% / 5 - bar = ( - "█" * filled + "░" * (20 - filled) - if USE_UNICODE - else ("#" * filled + "-" * (20 - filled)) - ) + bar = "█" * filled + "░" * (20 - filled) if USE_UNICODE else ("#" * filled + "-" * (20 - filled)) progress_bar = f" [{bar}]" else: status_text = phase_action @@ -321,14 +311,12 @@ def print_ui_expanded( # Clear line before writing to prevent leftover text sys.stdout.write("\r\033[K") - sys.stdout.write( - f"{status_symbol} {name_display} {status_display}{progress_bar}\n" - ) + sys.stdout.write(f"{status_symbol} {name_display} {status_display}{progress_bar}\n") sys.stdout.flush() -def load_build_stats() -> Dict[str, BuildStats]: +def load_build_stats() -> dict[str, BuildStats]: """ Load historical build statistics from disk. @@ -341,15 +329,15 @@ def load_build_stats() -> Dict[str, BuildStats]: try: import json - with open(stats_file, "r") as f: + with open(stats_file) as f: data = json.load(f) - return cast(Dict[str, BuildStats], data) - except (json.JSONDecodeError, IOError): + return cast("dict[str, BuildStats]", data) + except (OSError, json.JSONDecodeError): log.warning("Failed to load build stats, starting fresh") return {} -def save_build_stats(stats: Dict[str, BuildStats]) -> None: +def save_build_stats(stats: dict[str, BuildStats]) -> None: """ Save build statistics to disk. @@ -363,7 +351,7 @@ def save_build_stats(stats: Dict[str, BuildStats]) -> None: stats_file.parent.mkdir(parents=True, exist_ok=True) with open(stats_file, "w") as f: json.dump(stats, f, indent=2) - except IOError: + except OSError: log.warning("Failed to save build stats") @@ -380,9 +368,7 @@ def update_build_stats(step_name: str, line_count: int) -> None: """ stats = load_build_stats() if step_name not in stats: - stats[step_name] = BuildStats( - avg_lines=line_count, samples=1, last_lines=line_count - ) + stats[step_name] = BuildStats(avg_lines=line_count, samples=1, last_lines=line_count) else: old_avg = stats[step_name]["avg_lines"] # Exponential moving average: 70% new value, 30% old average diff --git a/relenv/build/darwin.py b/relenv/build/darwin.py index f60aab4a..4016fcdf 100644 --- a/relenv/build/darwin.py +++ b/relenv/build/darwin.py @@ -4,6 +4,7 @@ """ The darwin build process. """ + from __future__ import annotations import glob @@ -14,7 +15,7 @@ import tarfile import time import urllib.request -from typing import IO, MutableMapping +from typing import IO, TYPE_CHECKING from ..common import DARWIN, MACOS_DEVELOPMENT_TARGET, arches, runcmd from .common import ( @@ -27,6 +28,9 @@ update_sbom_checksums, ) +if TYPE_CHECKING: + from collections.abc import MutableMapping + ARCHES = arches[DARWIN] @@ -126,15 +130,13 @@ def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N # Update bundled expat to latest version update_expat(dirs, env) - env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format( - prefix=dirs.prefix, ldflags=env["LDFLAGS"] - ) + env["LDFLAGS"] = "-Wl,-rpath,{prefix}/lib {ldflags}".format(prefix=dirs.prefix, ldflags=env["LDFLAGS"]) runcmd( [ "./configure", "-v", - "--prefix={}".format(dirs.prefix), - "--with-openssl={}".format(dirs.prefix), + f"--prefix={dirs.prefix}", + f"--with-openssl={dirs.prefix}", "--enable-optimizations", "--disable-test-modules", ], @@ -142,12 +144,10 @@ def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N stderr=logfp, stdout=logfp, ) - with io.open("Modules/Setup", "a+") as fp: + with open("Modules/Setup", "a+") as fp: fp.seek(0, io.SEEK_END) - fp.write("*disabled*\n" "_tkinter\n" "nsl\n" "ncurses\n" "nis\n") - runcmd( - ["sed", "s/#zlib/zlib/g", "Modules/Setup"], env=env, stderr=logfp, stdout=logfp - ) + fp.write("*disabled*\n_tkinter\nnsl\nncurses\nnis\n") + runcmd(["sed", "s/#zlib/zlib/g", "Modules/Setup"], env=env, stderr=logfp, stdout=logfp) runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) @@ -229,7 +229,7 @@ def build_python(env: MutableMapping[str, str], dirs: Dirs, logfp: IO[str]) -> N "url": "https://www.python.org/ftp/python/{version}/Python-{version}.tar.xz", "fallback_url": "https://woz.io/relenv/dependencies/Python-{version}.tar.gz", "version": build.version, - "checksum": "d31d548cd2c5ca2ae713bebe346ba15e8406633a", + "checksum": None, }, ) diff --git a/relenv/build/linux.py b/relenv/build/linux.py index 2a9c67b1..032bee0d 100644 --- a/relenv/build/linux.py +++ b/relenv/build/linux.py @@ -4,6 +4,7 @@ """ The linux build process. """ + from __future__ import annotations import glob @@ -15,8 +16,10 @@ import tempfile import time import urllib.request -from typing import IO, MutableMapping +from collections.abc import MutableMapping +from typing import IO +from ..common import LINUX, Version, arches, runcmd from .common import ( Dirs, build_openssl, @@ -27,8 +30,6 @@ get_dependency_version, update_sbom_checksums, ) -from ..common import LINUX, Version, arches, runcmd - ARCHES = arches[LINUX] @@ -298,9 +299,7 @@ def build_libffi(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: stdout=logfp, ) # libffi doens't want to honor libdir, force install to lib instead of lib64 - runcmd( - ["sed", "-i", "s/lib64/lib/g", "Makefile"], env=env, stderr=logfp, stdout=logfp - ) + runcmd(["sed", "-i", "s/lib64/lib/g", "Makefile"], env=env, stderr=logfp, stdout=logfp) runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) @@ -332,6 +331,52 @@ def build_zlib(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: runcmd(["make", "install"], env=env, stderr=logfp, stdout=logfp) +def build_zstd(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: + """ + Build zstd. + + :param env: The environment dictionary + :type env: dict + :param dirs: The working directories + :type dirs: ``relenv.build.common.Dirs`` + :param logfp: A handle for the log file + :type logfp: file + """ + os.chdir(pathlib.Path(dirs.source) / "lib") + # Build static + shared so curl and friends can resolve ZSTD_* via libzstd.so. + # LDFLAGS is left to the lib Makefile because a command-line override would + # clobber its target-specific `LDFLAGS += -shared` and the .so link would + # then pull Scrt1.o and fail on a missing `main`. + runcmd( + [ + "make", + "-j8", + f"PREFIX={dirs.prefix}", + f"CC={env['CC']}", + f"CFLAGS={env['CFLAGS']} -fPIC -pthread", + f"LDFLAGS_DYNLIB={env['LDFLAGS']} -pthread", + "lib", + ], + env=env, + stderr=logfp, + stdout=logfp, + ) + runcmd( + [ + "make", + "install-static", + "install-shared", + "install-pc", + "install-includes", + f"PREFIX={dirs.prefix}", + f"CC={env['CC']}", + ], + env=env, + stderr=logfp, + stdout=logfp, + ) + + def build_krb(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: """ Build kerberos. @@ -348,6 +393,10 @@ def build_krb(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: env["ac_cv_func_regcomp"] = "yes" env["ac_cv_printf_positional"] = "yes" os.chdir(dirs.source / "src") + # krb5 1.22+ and some 1.21 tarballs are missing getdate.c, which normally + # triggers a yacc call when building the kadmin CLI. We skip building the + # CLI by removing it from the Makefile. + runcmd(["sed", "-i", "s/ kadmin / /g", "Makefile.in"], stderr=logfp, stdout=logfp) runcmd( [ "./configure", @@ -486,9 +535,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: "Modules/Setup", ] ) - if Version.parse_string(env["RELENV_PY_MAJOR_VERSION"]) <= Version.parse_string( - "3.10" - ): + if Version.parse_string(env["RELENV_PY_MAJOR_VERSION"]) <= Version.parse_string("3.10"): runcmd( [ "sed", @@ -529,6 +576,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: "--with-builtin-hashlib-hashes=blake2,md5,sha1,sha2,sha3", "--with-readline=readline", "--with-pkg-config=yes", + "--with-zstd=yes", ] if env["RELENV_HOST_ARCH"] != env["RELENV_BUILD_ARCH"]: @@ -544,9 +592,9 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: runcmd(cmd, env=env, stderr=logfp, stdout=logfp) - with io.open("Modules/Setup", "a+") as fp: + with open("Modules/Setup", "a+") as fp: fp.seek(0, io.SEEK_END) - fp.write("*disabled*\n" "_tkinter\n" "nsl\n" "nis\n") + fp.write("*disabled*\n_tkinter\nnsl\nnis\n") for _ in ["LDFLAGS", "CFLAGS", "CPPFLAGS", "CXX", "CC"]: env.pop(_) runcmd(["make", "-j8"], env=env, stderr=logfp, stdout=logfp) @@ -712,9 +760,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: ncurses_checksum = ncurses_info["sha256"] else: ncurses_version = "6.5" - ncurses_url = ( - "https://mirrors.ocf.berkeley.edu/gnu/ncurses/ncurses-{version}.tar.gz" - ) + ncurses_url = "https://mirrors.ocf.berkeley.edu/gnu/ncurses/ncurses-{version}.tar.gz" ncurses_checksum = "cde3024ac3f9ef21eaed6f001476ea8fffcaa381" build.add( @@ -819,9 +865,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: readline_checksum = readline_info["sha256"] else: readline_version = "8.3" - readline_url = ( - "https://mirrors.ocf.berkeley.edu/gnu/readline/readline-{version}.tar.gz" - ) + readline_url = "https://mirrors.ocf.berkeley.edu/gnu/readline/readline-{version}.tar.gz" readline_checksum = "2c05ae9350b695f69d70b47f17f092611de2081f" build.add( @@ -835,6 +879,27 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: }, ) +# Get zstd version from JSON +zstd_info = get_dependency_version("zstd", "linux") +if zstd_info: + zstd_version = zstd_info["version"] + zstd_url = zstd_info["url"] + zstd_checksum = zstd_info["sha256"] +else: + zstd_version = "1.5.7" + zstd_url = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz" + zstd_checksum = "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" + +build.add( + "zstd", + build_func=build_zstd, + download={ + "url": zstd_url, + "version": zstd_version, + "checksum": zstd_checksum, + }, +) + # Get tirpc version from JSON tirpc_info = get_dependency_version("tirpc", "linux") if tirpc_info: @@ -843,9 +908,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: tirpc_checksum = tirpc_info["sha256"] else: tirpc_version = "1.3.4" - tirpc_url = ( - "https://sourceforge.net/projects/libtirpc/files/libtirpc-{version}.tar.bz2" - ) + tirpc_url = "https://sourceforge.net/projects/libtirpc/files/libtirpc-{version}.tar.bz2" tirpc_checksum = "63c800f81f823254d2706637bab551dec176b99b" build.add( @@ -877,12 +940,13 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: "uuid", "krb5", "readline", + "zstd", "tirpc", ], download={ "url": "https://www.python.org/ftp/python/{version}/Python-{version}.tar.xz", "version": build.version, - "checksum": "d31d548cd2c5ca2ae713bebe346ba15e8406633a", + "checksum": None, }, ) diff --git a/relenv/build/windows.py b/relenv/build/windows.py index a7ae6ca3..1c9fb210 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -4,6 +4,7 @@ """ The windows build process. """ + from __future__ import annotations import glob @@ -16,8 +17,17 @@ import sys import tarfile import time -from typing import IO, MutableMapping, Union +from collections.abc import MutableMapping +from typing import IO +from ..common import ( + MODULE_DIR, + WIN32, + arches, + download_url, + extract_archive, + runcmd, +) from .common import ( Dirs, builds, @@ -28,14 +38,6 @@ update_ensurepip, update_sbom_checksums, ) -from ..common import ( - WIN32, - arches, - MODULE_DIR, - download_url, - extract_archive, - runcmd, -) log = logging.getLogger(__name__) @@ -108,13 +110,7 @@ def find_vcvarsall(env: EnvMapping) -> pathlib.Path | None: result = subprocess.run(cmd, capture_output=True, text=True, check=True) vs_path = result.stdout.strip() if vs_path: - candidate = ( - pathlib.Path(vs_path) - / "VC" - / "Auxiliary" - / "Build" - / "vcvarsall.bat" - ) + candidate = pathlib.Path(vs_path) / "VC" / "Auxiliary" / "Build" / "vcvarsall.bat" if candidate.exists(): return candidate except subprocess.CalledProcessError: @@ -177,9 +173,7 @@ def flatten_externals(dirs: Dirs, name: str, version: str) -> None: # Identify what was actually extracted # extract_archive usually extracts into externals_dir # We search for any directory that isn't 'zips' - extracted_dirs = [ - x for x in externals_dir.iterdir() if x.is_dir() and x.name != "zips" - ] + extracted_dirs = [x for x in externals_dir.iterdir() if x.is_dir() and x.name != "zips"] target_dir = externals_dir / f"{name}-{version}" @@ -245,7 +239,7 @@ def update_sqlite(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "sqlite": @@ -296,7 +290,7 @@ def update_xz(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "xz": @@ -375,9 +369,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: ref_loc = f"cpe:2.3:a:openssl:openssl:{version}:*:*:*:*:*:*:*" # noqa: E231 is_binary = "cpython-bin-deps" in url - target_dir = ( - dirs.source / "externals" / f"openssl-{version}-{env['RELENV_HOST_ARCH']}" - ) + target_dir = dirs.source / "externals" / f"openssl-{version}-{env['RELENV_HOST_ARCH']}" update_props( dirs.source, @@ -398,10 +390,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: # but we want openssl--. # We'll find it and move it ourselves. for d in (dirs.source / "externals").iterdir(): - if d.is_dir() and ( - d.name == f"openssl-{version}" - or d.name.startswith(f"openssl-{version}") - ): + if d.is_dir() and (d.name == f"openssl-{version}" or d.name.startswith(f"openssl-{version}")): if d != target_dir: if target_dir.exists(): shutil.rmtree(str(target_dir)) @@ -419,9 +408,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: if not is_binary: # Build from source - log.info( - "Building OpenSSL %s (%s) from source", version, env["RELENV_HOST_ARCH"] - ) + log.info("Building OpenSSL %s (%s) from source", version, env["RELENV_HOST_ARCH"]) perl_dir = update_perl(dirs, env) perl_bin = perl_dir / "perl" / "bin" / "perl.exe" @@ -447,18 +434,12 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: log.warning("Could not find vcvarsall.bat, build may fail") vcvars_cmd = "echo" else: - vcvars_arch = ( - "x64" - if env["RELENV_HOST_ARCH"] == "amd64" - else env["RELENV_HOST_ARCH"] - ) + vcvars_arch = "x64" if env["RELENV_HOST_ARCH"] == "amd64" else env["RELENV_HOST_ARCH"] vcvars_cmd = f'call "{vcvars}" {vcvars_arch}' env_path = os.environ.get("PATH", "") build_env = env.copy() - build_env[ - "PATH" - ] = f"{perl_bin.parent};{nasm_exe[0].parent};{env_path}" # noqa: E231,E702 + build_env["PATH"] = f"{perl_bin.parent};{nasm_exe[0].parent};{env_path}" # noqa: E231,E702 prefix = target_dir / "build" openssldir = prefix / "ssl" @@ -535,9 +516,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: for h in target_dir.glob("**/opensslv.h"): if h.parent.name == "openssl": # Found it, move its parent to include/ - shutil.copytree( - str(h.parent), str(inc_openssl_dir), dirs_exist_ok=True - ) + shutil.copytree(str(h.parent), str(inc_openssl_dir), dirs_exist_ok=True) break # Ensure applink.c is in include/ @@ -562,11 +541,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: update_props( dirs.source, r".*", - ( - f"" - f"$(opensslDir){arch_name}\\" - f"" - ), + (f"$(opensslDir){arch_name}\\"), ) # Patch openssl.props to use correct DLL suffix for OpenSSL 3.x @@ -586,7 +561,7 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "openssl": @@ -621,7 +596,7 @@ def update_bzip2(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "bzip2": @@ -661,9 +636,7 @@ def update_libffi(dirs: Dirs, env: EnvMapping) -> None: lib_name = lib_files[0].name if lib_name != "libffi-7.lib": log.info("Patching libffi library name to %s", lib_name) - patch_file( - dirs.source / "PCbuild" / "libffi.props", r"libffi-7\.lib", lib_name - ) + patch_file(dirs.source / "PCbuild" / "libffi.props", r"libffi-7\.lib", lib_name) patch_file( dirs.source / "PCbuild" / "libffi.props", r"libffi-7\.dll", @@ -674,7 +647,7 @@ def update_libffi(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "libffi": @@ -709,7 +682,7 @@ def update_zlib(dirs: Dirs, env: EnvMapping) -> None: if env["RELENV_PY_MAJOR_VERSION"] in ["3.12", "3.13", "3.14"]: spdx_json = dirs.source / "Misc" / "externals.spdx.json" if spdx_json.exists(): - with open(str(spdx_json), "r") as f: + with open(str(spdx_json)) as f: data = json.load(f) for pkg in data["packages"]: if pkg["name"] == "zlib": @@ -721,6 +694,82 @@ def update_zlib(dirs: Dirs, env: EnvMapping) -> None: json.dump(data, f, indent=2) +def update_zlib_ng(dirs: Dirs, env: EnvMapping) -> None: + """ + Update the zlib-ng library. + + Python 3.14 replaced zlib with zlib-ng for the bundled zlib module on + Windows. The PCbuild project expects sources in + ``externals/zlib-ng-/`` (notably ``zlib.h.in`` and + ``zlib-ng.h.in`` at the top level). + """ + zlib_ng_info = get_dependency_version("zlib-ng", "win32") + if not zlib_ng_info: + return + + version = zlib_ng_info["version"] + url = zlib_ng_info["url"].format(version=version) + sha256 = zlib_ng_info["sha256"] + ref_loc = f"cpe:2.3:a:zlib-ng:zlib-ng:{version}:*:*:*:*:*:*:*" # noqa: E231 + + target_dir = dirs.source / "externals" / f"zlib-ng-{version}" + update_props(dirs.source, r"zlib-ng-\d+(\.\d+)*", f"zlib-ng-{version}") + if not target_dir.exists(): + get_externals_source(externals_dir=dirs.source / "externals", url=url) + flatten_externals(dirs, "zlib-ng", version) + + if env["RELENV_PY_MAJOR_VERSION"] in ["3.14"]: + spdx_json = dirs.source / "Misc" / "externals.spdx.json" + if spdx_json.exists(): + with open(str(spdx_json)) as f: + data = json.load(f) + for pkg in data["packages"]: + if pkg["name"] == "zlib-ng": + pkg["versionInfo"] = version + pkg["downloadLocation"] = url + pkg["checksums"][0]["checksumValue"] = sha256 + pkg["externalRefs"][0]["referenceLocator"] = ref_loc + with open(str(spdx_json), "w") as f: + json.dump(data, f, indent=2) + + +def update_zstd(dirs: Dirs, env: EnvMapping) -> None: + """ + Update the zstd library. + + Python 3.14 added a bundled ``_zstd`` extension whose Windows project + compiles zstd from source in ``externals/zstd-/``. + """ + zstd_info = get_dependency_version("zstd", "win32") + if not zstd_info: + return + + version = zstd_info["version"] + url = zstd_info["url"].format(version=version) + sha256 = zstd_info["sha256"] + ref_loc = f"cpe:2.3:a:facebook:zstandard:{version}:*:*:*:*:*:*:*" # noqa: E231 + + target_dir = dirs.source / "externals" / f"zstd-{version}" + update_props(dirs.source, r"zstd-\d+(\.\d+)*", f"zstd-{version}") + if not target_dir.exists(): + get_externals_source(externals_dir=dirs.source / "externals", url=url) + flatten_externals(dirs, "zstd", version) + + if env["RELENV_PY_MAJOR_VERSION"] in ["3.14"]: + spdx_json = dirs.source / "Misc" / "externals.spdx.json" + if spdx_json.exists(): + with open(str(spdx_json)) as f: + data = json.load(f) + for pkg in data["packages"]: + if pkg["name"] == "zstd": + pkg["versionInfo"] = version + pkg["downloadLocation"] = url + pkg["checksums"][0]["checksumValue"] = sha256 + pkg["externalRefs"][0]["referenceLocator"] = ref_loc + with open(str(spdx_json), "w") as f: + json.dump(data, f, indent=2) + + def update_mpdecimal(dirs: Dirs, env: EnvMapping) -> None: """ Update the MPDECIMAL library. @@ -775,9 +824,7 @@ def update_perl(dirs: Dirs, env: EnvMapping) -> pathlib.Path: return target_dir -def copy_pyconfig_h( - source: pathlib.Path, build_dir: pathlib.Path, dest_dir: pathlib.Path -) -> pathlib.Path: +def copy_pyconfig_h(source: pathlib.Path, build_dir: pathlib.Path, dest_dir: pathlib.Path) -> pathlib.Path: """ Copy ``pyconfig.h`` into the onedir's ``Include`` directory. @@ -819,6 +866,8 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: update_bzip2(dirs=dirs, env=env) update_libffi(dirs=dirs, env=env) update_zlib(dirs=dirs, env=env) + update_zlib_ng(dirs=dirs, env=env) + update_zstd(dirs=dirs, env=env) update_mpdecimal(dirs=dirs, env=env) update_nasm(dirs=dirs, env=env) update_perl(dirs=dirs, env=env) @@ -830,7 +879,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: log.info("Patching regen.targets to skip SBOM generation") patch_file( regen_targets, - r'Command="py -3.13 .*generate_sbom\.py.*"', + r'Command="py -3\.\d+ .*generate_sbom\.py.*"', 'Command="echo skipping sbom"', ) @@ -897,9 +946,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: ) copy_pyconfig_h(dirs.source, build_dir, dirs.prefix / "Include") - shutil.copytree( - src=str(dirs.source / "Lib"), dst=str(dirs.prefix / "Lib"), dirs_exist_ok=True - ) + shutil.copytree(src=str(dirs.source / "Lib"), dst=str(dirs.prefix / "Lib"), dirs_exist_ok=True) os.makedirs(str(dirs.prefix / "Lib" / "site-packages"), exist_ok=True) (dirs.prefix / "libs").mkdir(parents=True, exist_ok=True) @@ -919,7 +966,7 @@ def build_python(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: download={ "url": "https://www.python.org/ftp/python/{version}/Python-{version}.tar.xz", "version": build.version, - "checksum": "d31d548cd2c5ca2ae713bebe346ba15e8406633a", + "checksum": None, }, ) @@ -935,7 +982,7 @@ def finalize(env: EnvMapping, dirs: Dirs, logfp: IO[str]) -> None: python = dirs.prefix / "Scripts" / "python.exe" runcmd([str(python), "-m", "ensurepip"], env=env, stderr=logfp, stdout=logfp) - def runpip(pkg: Union[str, os.PathLike[str]]) -> None: + def runpip(pkg: str | os.PathLike[str]) -> None: env = os.environ.copy() cmd = [str(python), "-m", "pip", "install", str(pkg)] runcmd(cmd, env=env, stderr=logfp, stdout=logfp) diff --git a/relenv/buildenv.py b/relenv/buildenv.py index 12627656..b069062d 100644 --- a/relenv/buildenv.py +++ b/relenv/buildenv.py @@ -3,14 +3,13 @@ """ Helper for building libraries to install into a relenv environment. """ + from __future__ import annotations -import argparse import json import logging -import os import sys -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast from .common import ( MACOS_DEVELOPMENT_TARGET, @@ -20,6 +19,10 @@ get_triplet, ) +if TYPE_CHECKING: + import argparse + import os + log = logging.getLogger() @@ -32,9 +35,7 @@ def setup_parser( :param subparsers: The subparsers object returned from ``add_subparsers`` :type subparsers: argparse._SubParsersAction """ - subparser = subparsers.add_parser( - "buildenv", description="Relenv build environment" - ) + subparser = subparsers.add_parser("buildenv", description="Relenv build environment") subparser.set_defaults(func=main) subparser.add_argument( "--json", @@ -60,7 +61,7 @@ def buildenv( if not relenv_path: if not is_relenv(): raise RelenvEnvironmentError("Not in a relenv environment") - relenv_path = cast(str | os.PathLike[str], cast(Any, sys).RELENV) + relenv_path = cast("str | os.PathLike[str]", cast("Any", sys).RELENV) if sys.platform != "linux": raise PlatformError("buildenv is only supported on Linux") @@ -78,12 +79,7 @@ def buildenv( "RELENV_PATH": f"{relenv_path}", "CC": f"{toolchain}/bin/{triplet}-gcc", "CXX": f"{toolchain}/bin/{triplet}-g++", - "CFLAGS": ( - f"--sysroot={sysroot} " - f"-fPIC " - f"-I{relenv_path}/include " - f"-I{sysroot}/usr/include" - ), + "CFLAGS": (f"--sysroot={sysroot} -fPIC -I{relenv_path}/include -I{sysroot}/usr/include"), "CXXFLAGS": ( f"--sysroot={sysroot} " f"-fPIC " @@ -92,23 +88,9 @@ def buildenv( f"-L{relenv_path}/lib -L{sysroot}/lib " f"-Wl,-rpath,{relenv_path}/lib" ), - "CPPFLAGS": ( - f"--sysroot={sysroot} " - f"-fPIC " - f"-I{relenv_path}/include " - f"-I{sysroot}/usr/include" - ), - "CMAKE_CFLAGS": ( - f"--sysroot={sysroot} " - f"-fPIC " - f"-I{relenv_path}/include " - f"-I{sysroot}/usr/include" - ), - "LDFLAGS": ( - f"--sysroot={sysroot} " - f"-L{relenv_path}/lib -L{sysroot}/lib " - f"-Wl,-rpath,{relenv_path}/lib" - ), + "CPPFLAGS": (f"--sysroot={sysroot} -fPIC -I{relenv_path}/include -I{sysroot}/usr/include"), + "CMAKE_CFLAGS": (f"--sysroot={sysroot} -fPIC -I{relenv_path}/include -I{sysroot}/usr/include"), + "LDFLAGS": (f"--sysroot={sysroot} -L{relenv_path}/lib -L{sysroot}/lib -Wl,-rpath,{relenv_path}/lib"), "CRATE_CC_NO_DEFAULTS": "1", "OPENSSL_DIR": f"{relenv_path}", "OPENSSL_INCLUDE_DIR": f"{relenv_path}/include", diff --git a/relenv/check.py b/relenv/check.py index c308fbdc..97be9b81 100644 --- a/relenv/check.py +++ b/relenv/check.py @@ -3,15 +3,19 @@ """ Check the integrety of a relenv environment. """ + from __future__ import annotations -import argparse import logging import pathlib import sys +from typing import TYPE_CHECKING from relenv import relocate +if TYPE_CHECKING: + import argparse + log: logging.Logger = logging.getLogger(__name__) diff --git a/relenv/common.py b/relenv/common.py index f3669361..4925a22d 100644 --- a/relenv/common.py +++ b/relenv/common.py @@ -3,6 +3,7 @@ """ Common classes and values used around relenv. """ + from __future__ import annotations import http.client @@ -20,18 +21,10 @@ import textwrap import threading import time -from typing import ( - IO, - Any, - BinaryIO, - Callable, - Iterable, - Literal, - Mapping, - Optional, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, BinaryIO, Literal, cast + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping # relenv package version __version__ = "0.22.9" @@ -51,7 +44,7 @@ # 8 GiB archives are not unusual; stick to metadata to fingerprint them. -def _archive_metadata(path: pathlib.Path) -> dict[str, Union[str, int]]: +def _archive_metadata(path: pathlib.Path) -> dict[str, str | int]: stat = path.stat() return { "archive": str(path.resolve()), @@ -60,7 +53,7 @@ def _archive_metadata(path: pathlib.Path) -> dict[str, Union[str, int]]: } -def _toolchain_cache_root() -> Optional[pathlib.Path]: +def _toolchain_cache_root() -> pathlib.Path | None: override = os.environ.get(TOOLCHAIN_CACHE_ENV) if override: if override.strip().lower() == "none": @@ -81,7 +74,7 @@ def _toolchain_manifest_path(toolchain_path: pathlib.Path) -> pathlib.Path: return toolchain_path / _TOOLCHAIN_MANIFEST -def _load_toolchain_manifest(path: pathlib.Path) -> Optional[Mapping[str, Any]]: +def _load_toolchain_manifest(path: pathlib.Path) -> Mapping[str, Any] | None: if not path.exists(): return None try: @@ -102,18 +95,14 @@ def _manifest_matches(manifest: Mapping[str, Any], metadata: Mapping[str, Any]) ) -def _write_toolchain_manifest( - toolchain_path: pathlib.Path, metadata: Mapping[str, Any] -) -> None: +def _write_toolchain_manifest(toolchain_path: pathlib.Path, metadata: Mapping[str, Any]) -> None: manifest_path = _toolchain_manifest_path(toolchain_path) try: with manifest_path.open("w", encoding="utf-8") as handle: json.dump(metadata, handle, indent=2, sort_keys=True) handle.write("\n") except OSError as exc: # pragma: no cover - permissions edge cases - log.warning( - "Unable to persist toolchain manifest at %s: %s", manifest_path, exc - ) + log.warning("Unable to persist toolchain manifest at %s: %s", manifest_path, exc) def toolchain_root_dir() -> pathlib.Path: @@ -334,7 +323,7 @@ def build_arch() -> str: def work_root( - root: Optional[Union[str, os.PathLike[str]]] = None, + root: str | os.PathLike[str] | None = None, ) -> pathlib.Path: """ Get the root directory that all other relenv working directories should be based on. @@ -352,9 +341,7 @@ def work_root( return base -def work_dir( - name: str, root: Optional[Union[str, os.PathLike[str]]] = None -) -> pathlib.Path: +def work_dir(name: str, root: str | os.PathLike[str] | None = None) -> pathlib.Path: """ Get the absolute path to the relenv working directory of the given name. @@ -368,7 +355,7 @@ def work_dir( """ root = work_root(root) if root == MODULE_DIR: - base = root / "_{}".format(name) + base = root / f"_{name}" else: base = root / name return base @@ -382,7 +369,7 @@ class WorkDirs: :type root: str """ - def __init__(self: "WorkDirs", root: Union[str, os.PathLike[str]]) -> None: + def __init__(self: WorkDirs, root: str | os.PathLike[str]) -> None: self.root: pathlib.Path = pathlib.Path(root) self.data: pathlib.Path = DATA_DIR self.toolchain_config: pathlib.Path = work_dir("toolchain", self.root) @@ -392,7 +379,7 @@ def __init__(self: "WorkDirs", root: Union[str, os.PathLike[str]]) -> None: self.logs: pathlib.Path = work_dir("logs", DATA_DIR) self.download: pathlib.Path = work_dir("download", DATA_DIR) - def __getstate__(self: "WorkDirs") -> dict[str, pathlib.Path]: + def __getstate__(self: WorkDirs) -> dict[str, pathlib.Path]: """ Return an object used for pickling. @@ -408,7 +395,7 @@ def __getstate__(self: "WorkDirs") -> dict[str, pathlib.Path]: "download": self.download, } - def __setstate__(self: "WorkDirs", state: Mapping[str, pathlib.Path]) -> None: + def __setstate__(self: WorkDirs, state: Mapping[str, pathlib.Path]) -> None: """ Unwrap the object returned from unpickling. @@ -425,7 +412,7 @@ def __setstate__(self: "WorkDirs", state: Mapping[str, pathlib.Path]) -> None: def work_dirs( - root: Optional[Union[str, os.PathLike[str]]] = None, + root: str | os.PathLike[str] | None = None, ) -> WorkDirs: """ Returns a WorkDirs instance based on the given root. @@ -440,9 +427,9 @@ def work_dirs( def get_toolchain( - arch: Optional[str] = None, - root: Optional[Union[str, os.PathLike[str]]] = None, -) -> Optional[pathlib.Path]: + arch: str | None = None, + root: str | os.PathLike[str] | None = None, +) -> pathlib.Path | None: """ Get a the toolchain directory, specific to the arch if supplied. @@ -469,7 +456,7 @@ def get_toolchain( toolchain_root = toolchain_root_dir() triplet = get_triplet(machine=arch) toolchain_path = toolchain_root / triplet - metadata: Optional[Mapping[str, Any]] = None + metadata: Mapping[str, Any] | None = None if toolchain_path.exists(): metadata = _load_toolchain_manifest(_toolchain_manifest_path(toolchain_path)) @@ -489,11 +476,7 @@ def get_toolchain( archive_path = pathlib.Path(archive_attr) archive_meta = _archive_metadata(archive_path) - if ( - toolchain_path.exists() - and metadata - and _manifest_matches(metadata, archive_meta) - ): + if toolchain_path.exists() and metadata and _manifest_matches(metadata, archive_meta): return toolchain_path if toolchain_path.exists(): @@ -501,14 +484,12 @@ def get_toolchain( extract(str(toolchain_root), str(archive_path)) if not toolchain_path.exists(): - raise RelenvException( - f"Toolchain archive {archive_path} did not produce {toolchain_path}" - ) + raise RelenvException(f"Toolchain archive {archive_path} did not produce {toolchain_path}") _write_toolchain_manifest(toolchain_path, archive_meta) return toolchain_path -def get_triplet(machine: Optional[str] = None, plat: Optional[str] = None) -> str: +def get_triplet(machine: str | None = None, plat: str | None = None) -> str: """ Get the target triplet for the specified machine and platform. @@ -568,7 +549,7 @@ def list_archived_builds() -> list[tuple[str, str, str]]: return builds -def archived_build(triplet: Optional[str] = None) -> pathlib.Path: +def archived_build(triplet: str | None = None) -> pathlib.Path: """ Finds a the location of an archived build. @@ -585,9 +566,7 @@ def archived_build(triplet: Optional[str] = None) -> pathlib.Path: return dirs.build / archive -def extract_archive( - to_dir: Union[str, os.PathLike[str]], archive: Union[str, os.PathLike[str]] -) -> None: +def extract_archive(to_dir: str | os.PathLike[str], archive: str | os.PathLike[str]) -> None: """ Extract an archive to a specific location. @@ -627,7 +606,7 @@ def extract_archive( tar.extractall(str(to_path)) -def get_download_location(url: str, dest: Union[str, os.PathLike[str]]) -> str: +def get_download_location(url: str, dest: str | os.PathLike[str]) -> str: """ Get the full path to where the url will be downloaded to. @@ -642,7 +621,7 @@ def get_download_location(url: str, dest: Union[str, os.PathLike[str]]) -> str: return os.path.join(os.fspath(dest), os.path.basename(url)) -def check_url(url: str, timestamp: Optional[float] = None, timeout: float = 30) -> bool: +def check_url(url: str, timestamp: float | None = None, timeout: float = 30) -> bool: """ Check that the url returns a 200. """ @@ -654,9 +633,7 @@ def check_url(url: str, timestamp: Optional[float] = None, timeout: float = 30) req = urllib.request.Request(url) if timestamp: - headers["If-Modified-Since"] = time.strftime( - "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp) - ) + headers["If-Modified-Since"] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(timestamp)) for k, v in headers.items(): req.add_header(k, v) @@ -678,7 +655,7 @@ def fetch_url( fp: BinaryIO, backoff: int = 3, timeout: float = 30, - progress_callback: Optional[Callable[[int, int], None]] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> None: """ Fetch the contents of a url. @@ -788,11 +765,11 @@ def fetch_url_content(url: str, backoff: int = 3, timeout: float = 30) -> str: def download_url( url: str, - dest: Union[str, os.PathLike[str]], + dest: str | os.PathLike[str], verbose: bool = True, backoff: int = 3, timeout: float = 60, - progress_callback: Optional[Callable[[int, int], None]] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> str: """ Download the url to the provided destination. @@ -854,7 +831,6 @@ def runcmd(*args: Any, **kwargs: Any) -> subprocess.Popen[str]: if "universal_newlines" not in kwargs: kwargs["universal_newlines"] = True if sys.platform != "win32": - p = subprocess.Popen(*args, **kwargs) stdout_stream = p.stdout stderr_stream = p.stderr @@ -869,7 +845,7 @@ def runcmd(*args: Any, **kwargs: Any) -> subprocess.Popen[str]: while ok: for key, val1 in sel.select(): del val1 # unused - stream = cast(IO[str], key.fileobj) + stream = cast("IO[str]", key.fileobj) line = stream.readline() if not line: ok = False @@ -885,7 +861,7 @@ def runcmd(*args: Any, **kwargs: Any) -> subprocess.Popen[str]: def enqueue_stream( stream: IO[str], - item_queue: "queue.Queue[tuple[int | str, str]]", + item_queue: queue.Queue[tuple[int | str, str]], kind: int, ) -> None: last_line = "" @@ -899,7 +875,7 @@ def enqueue_stream( def enqueue_process( process: subprocess.Popen[str], - item_queue: "queue.Queue[tuple[int | str, str]]", + item_queue: queue.Queue[tuple[int | str, str]], ) -> None: process.wait() item_queue.put(("x", "")) @@ -910,7 +886,7 @@ def enqueue_process( if stdout_stream is None or stderr_stream is None: p.wait() raise RelenvException("Process pipes are unavailable") - q: "queue.Queue[tuple[int | str, str]]" = queue.Queue() + q: queue.Queue[tuple[int | str, str]] = queue.Queue() to = threading.Thread(target=enqueue_stream, args=(stdout_stream, q, 1)) te = threading.Thread(target=enqueue_stream, args=(stderr_stream, q, 2)) tp = threading.Thread(target=enqueue_process, args=(p, q)) @@ -939,9 +915,9 @@ def enqueue_process( def relative_interpreter( - root_dir: Union[str, os.PathLike[str]], - scripts_dir: Union[str, os.PathLike[str]], - interpreter: Union[str, os.PathLike[str]], + root_dir: str | os.PathLike[str], + scripts_dir: str | os.PathLike[str], + interpreter: str | os.PathLike[str], ) -> pathlib.Path: """ Return a relativized path to the given scripts_dir and interpreter. @@ -960,7 +936,7 @@ def relative_interpreter( return relscripts / relinterp -def makepath(*paths: Union[str, os.PathLike[str]]) -> tuple[str, str]: +def makepath(*paths: str | os.PathLike[str]) -> tuple[str, str]: """ Make a normalized path name from paths. """ @@ -972,7 +948,7 @@ def makepath(*paths: Union[str, os.PathLike[str]]) -> tuple[str, str]: return dir, os.path.normcase(dir) -def addpackage(sitedir: str, name: Union[str, os.PathLike[str]]) -> list[str] | None: +def addpackage(sitedir: str, name: str | os.PathLike[str]) -> list[str] | None: """ Add editable package to path. """ @@ -987,9 +963,7 @@ def addpackage(sitedir: str, name: Union[str, os.PathLike[str]]) -> list[str] | return None file_attr_hidden = getattr(stat, "FILE_ATTRIBUTE_HIDDEN", 0) uf_hidden = getattr(stat, "UF_HIDDEN", 0) - if (getattr(st, "st_flags", 0) & uf_hidden) or ( - getattr(st, "st_file_attributes", 0) & file_attr_hidden - ): + if (getattr(st, "st_flags", 0) & uf_hidden) or (getattr(st, "st_file_attributes", 0) & file_attr_hidden): # print(f"Skipping hidden .pth file: {fullname!r}") return None # print(f"Processing .pth file: {fullname!r}") @@ -1015,7 +989,7 @@ def addpackage(sitedir: str, name: Union[str, os.PathLike[str]]) -> list[str] | paths.append(dir) except Exception: print( - "Error processing line {:d} of {}:\n".format(n + 1, fullname), + f"Error processing line {n + 1:d} of {fullname}:\n", file=sys.stderr, ) import traceback @@ -1069,11 +1043,11 @@ class Version: def __init__(self, data: str) -> None: major, minor, micro = self.parse_string(data) self.major: int = major - self.minor: Optional[int] = minor - self.micro: Optional[int] = micro + self.minor: int | None = minor + self.micro: int | None = micro self._data: str = data - def __str__(self: "Version") -> str: + def __str__(self: Version) -> str: """ Version as string. """ @@ -1085,7 +1059,7 @@ def __str__(self: "Version") -> str: # XXX What if minor was None but micro was an int. return result - def __hash__(self: "Version") -> int: + def __hash__(self: Version) -> int: """ Hash of the version. @@ -1094,7 +1068,7 @@ def __hash__(self: "Version") -> int: return hash((self.major, self.minor, self.micro)) @staticmethod - def parse_string(data: str) -> tuple[int, Optional[int], Optional[int]]: + def parse_string(data: str) -> tuple[int, int | None, int | None]: """ Parse a version string into major, minor, and micro integers. """ @@ -1108,7 +1082,7 @@ def parse_string(data: str) -> tuple[int, Optional[int], Optional[int]]: else: raise RuntimeError("Too many parts to parse") - def __eq__(self: "Version", other: object) -> bool: + def __eq__(self: Version, other: object) -> bool: """ Equality comparisons. """ @@ -1122,7 +1096,7 @@ def __eq__(self: "Version", other: object) -> bool: micro = 0 if other.micro is None else other.micro return mymajor == major and myminor == minor and mymicro == micro - def __lt__(self: "Version", other: object) -> bool: + def __lt__(self: Version, other: object) -> bool: """ Less than comparrison. """ @@ -1143,7 +1117,7 @@ def __lt__(self: "Version", other: object) -> bool: return True return False - def __le__(self: "Version", other: object) -> bool: + def __le__(self: Version, other: object) -> bool: """ Less than or equal to comparrison. """ @@ -1161,7 +1135,7 @@ def __le__(self: "Version", other: object) -> bool: return True return False - def __gt__(self: "Version", other: object) -> bool: + def __gt__(self: Version, other: object) -> bool: """ Greater than comparrison. """ @@ -1169,7 +1143,7 @@ def __gt__(self: "Version", other: object) -> bool: return NotImplemented return not self.__le__(other) - def __ge__(self: "Version", other: object) -> bool: + def __ge__(self: Version, other: object) -> bool: """ Greater than or equal to comparrison. """ diff --git a/relenv/create.py b/relenv/create.py index 73f4b894..bcc2ce2e 100644 --- a/relenv/create.py +++ b/relenv/create.py @@ -6,14 +6,13 @@ from __future__ import annotations -import argparse import contextlib import os import pathlib import shutil import sys import tarfile -from collections.abc import Iterator +from typing import TYPE_CHECKING from .common import ( RelenvException, @@ -29,6 +28,10 @@ resolve_python_version, ) +if TYPE_CHECKING: + import argparse + from collections.abc import Iterator + @contextlib.contextmanager def chdir(path: str | os.PathLike[str]) -> Iterator[None]: @@ -82,7 +85,7 @@ def setup_parser( "--python", default=default_version, type=str, - help="The python version (e.g., 3.10, 3.13.7) [default: %(default)s]", + help="The python version (e.g., 3.10, 3.14.4) [default: %(default)s]", ) @@ -122,17 +125,17 @@ def create( if plat == "linux": if arch in arches[plat]: - triplet = "{}-{}-gnu".format(arch, plat) + triplet = f"{arch}-{plat}-gnu" else: raise CreateException("Unknown arch") elif plat == "darwin": if arch in arches[plat]: - triplet = "{}-macos".format(arch) + triplet = f"{arch}-macos" else: raise CreateException("Unknown arch") elif plat == "win32": if arch in arches[plat]: - triplet = "{}-win".format(arch) + triplet = f"{arch}-win" else: raise CreateException("Unknown arch") else: @@ -142,8 +145,7 @@ def create( tar = archived_build(f"{version}-{triplet}") if not tar.exists(): raise CreateException( - f"Error, build archive for {arch} doesn't exist: {tar}\n" - "You might try relenv fetch to resolve this." + f"Error, build archive for {arch} doesn't exist: {tar}\nYou might try relenv fetch to resolve this." ) with tarfile.open(tar, "r:xz") as fp: for f in fp: @@ -255,9 +257,7 @@ def main(args: argparse.Namespace) -> None: """ name = args.name if args.arch != build_arch(): - print( - "Warning: Cross compilation support is experimental and is not fully tested or working!" - ) + print("Warning: Cross compilation support is experimental and is not fully tested or working!") try: create_version = resolve_python_version(args.python) diff --git a/relenv/fetch.py b/relenv/fetch.py index fa677b09..873ce280 100644 --- a/relenv/fetch.py +++ b/relenv/fetch.py @@ -7,10 +7,9 @@ from __future__ import annotations -import argparse import os import sys -from collections.abc import Sequence +from typing import TYPE_CHECKING from .build import platform_module from .common import ( @@ -25,6 +24,10 @@ ) from .pyversions import get_default_python_version, resolve_python_version +if TYPE_CHECKING: + import argparse + from collections.abc import Sequence + def setup_parser( subparsers: argparse._SubParsersAction[argparse.ArgumentParser], @@ -50,7 +53,7 @@ def setup_parser( "--python", default=default_version, type=str, - help="The python version (e.g., 3.10, 3.13.7) [default: %(default)s]", + help="The python version (e.g., 3.10, 3.14.4) [default: %(default)s]", ) @@ -70,9 +73,7 @@ def fetch( if check_url(url, timeout=5): break else: - print( - f"Unable to find file on any hosts: github.com {' '.join(x.split('/')[0] for x in check_hosts)}" - ) + print(f"Unable to find file on any hosts: github.com {' '.join(x.split('/')[0] for x in check_hosts)}") sys.exit(1) builddir = work_dir("build", DATA_DIR) os.makedirs(builddir, exist_ok=True) diff --git a/relenv/manifest.py b/relenv/manifest.py index ca4dff65..e681580f 100644 --- a/relenv/manifest.py +++ b/relenv/manifest.py @@ -4,6 +4,7 @@ """ Relenv manifest. """ + from __future__ import annotations import hashlib @@ -16,11 +17,7 @@ def manifest(root: str | os.PathLike[str] | None = None) -> None: """ List all the file in a relenv and their hashes. """ - base = ( - pathlib.Path(root) - if root is not None - else pathlib.Path(getattr(sys, "RELENV", os.getcwd())) - ) + base = pathlib.Path(root) if root is not None else pathlib.Path(getattr(sys, "RELENV", os.getcwd())) for dirpath, _dirs, files in os.walk(base): directory = pathlib.Path(dirpath) for file in files: diff --git a/relenv/python-versions.json b/relenv/python-versions.json index 11a26630..7ceb98b9 100644 --- a/relenv/python-versions.json +++ b/relenv/python-versions.json @@ -189,7 +189,12 @@ "3.12.13": "ad3e9c333d91bee73f1d5f4a6fe6e88f2e74d911", "3.11.15": "e434ba0457a632f86e73239174bb1737cb57c09c", "3.10.20": "33b99a3309d5a0323b71a4764543f61ff1fcf8f3", - "3.13.13": "be80bbd34ab6627c464a2a2d965d8b8fa5aa2388" + "3.13.13": "be80bbd34ab6627c464a2a2d965d8b8fa5aa2388", + "3.14.4": "567ffd2b4116db4cbcd3fb7adb304ec32687f99f", + "3.14.3": "83eed62ba54742382542474db798717e6ee6b3f2", + "3.14.2": "b21c499c9e0250c1bfabc29a08c160018d2f6f57", + "3.14.1": "da8bd5ae7a346b80db64bac2dc2c9d9da3ca6eac", + "3.14.0": "8a1ae36a2c4212637401af93c8a7856d126156e3" }, "dependencies": { "perl": { @@ -341,6 +346,15 @@ ] } }, + "zlib-ng": { + "2.2.4": { + "url": "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/{version}.tar.gz", + "sha256": "a73343c3093e5cdc50d9377997c3815b878fd110bf6511c2c7759f2afb90f5a3", + "platforms": [ + "win32" + ] + } + }, "ncurses": { "6.5": { "url": "https://mirrors.ocf.berkeley.edu/gnu/ncurses/ncurses-{version}.tar.gz", @@ -391,6 +405,17 @@ ] } }, + "zstd": { + "1.5.7": { + "url": "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", + "sha256": "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3", + "platforms": [ + "linux", + "darwin", + "win32" + ] + } + }, "krb5": { "1.22.2": { "url": "https://kerberos.org/dist/krb5/1.22/krb5-{version}.tar.gz", diff --git a/relenv/pyversions.py b/relenv/pyversions.py index c8b243e8..41e0d21f 100644 --- a/relenv/pyversions.py +++ b/relenv/pyversions.py @@ -13,7 +13,6 @@ from __future__ import annotations -import argparse import hashlib import json import logging @@ -23,10 +22,13 @@ import subprocess as _subprocess import sys as _sys import time -from typing import Any +from typing import TYPE_CHECKING, Any from relenv.common import Version, check_url, download_url, fetch_url_content +if TYPE_CHECKING: + import argparse + log = logging.getLogger(__name__) os = _os @@ -70,9 +72,7 @@ def _release_urls(version: Version, gzip: bool = False) -> tuple[str, str | None def _receive_key(keyid: str, server: str) -> bool: - proc = subprocess.run( - ["gpg", "--keyserver", server, "--recv-keys", keyid], capture_output=True - ) + proc = subprocess.run(["gpg", "--keyserver", server, "--recv-keys", keyid], capture_output=True) if proc.returncode == 0: return True return False @@ -120,7 +120,7 @@ def verify_signature( PRINT = True CHECK = True -VERSION = None # '3.13.2' +VERSION = None # '3.14.0' UPDATE = False PINNED_VERSIONS = { @@ -139,7 +139,6 @@ def digest(file: str | os.PathLike[str]) -> str: def _main() -> None: - pyversions: dict[str, Any] = {"versions": []} vfile = pathlib.Path(".pyversions") @@ -259,9 +258,7 @@ def detect_openssl_versions() -> list[str]: matches = [v for v in matches if v == pin or v.startswith(f"{pin}.")] # Deduplicate and sort - versions = sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + versions = sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) return versions @@ -287,9 +284,7 @@ def detect_sqlite_versions() -> list[tuple[str, str]]: subpatch = int(sqlite_ver[5:7]) version = f"{major}.{minor}.{patch}.{subpatch}" versions.append((version, sqlite_ver)) - return sorted( - versions, key=lambda x: [int(n) for n in x[0].split(".")], reverse=True - ) + return sorted(versions, key=lambda x: [int(n) for n in x[0].split(".")], reverse=True) def detect_xz_versions() -> list[str]: @@ -302,9 +297,7 @@ def detect_xz_versions() -> list[str]: pattern = r"xz-(\d+\.\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) # Deduplicate and sort - versions = sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + versions = sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) return versions @@ -314,9 +307,7 @@ def detect_libffi_versions() -> list[str]: content = fetch_url_content(url) pattern = r'v(\d+\.\d+\.\d+)"' matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_zlib_versions() -> list[str]: @@ -325,9 +316,7 @@ def detect_zlib_versions() -> list[str]: content = fetch_url_content(url) pattern = r"zlib-(\d+\.\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_bzip2_versions() -> list[str]: @@ -336,9 +325,7 @@ def detect_bzip2_versions() -> list[str]: content = fetch_url_content(url) pattern = r"bzip2-(\d+\.\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_ncurses_versions() -> list[str]: @@ -347,9 +334,7 @@ def detect_ncurses_versions() -> list[str]: content = fetch_url_content(url) pattern = r"ncurses-(\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_readline_versions() -> list[str]: @@ -358,9 +343,7 @@ def detect_readline_versions() -> list[str]: content = fetch_url_content(url) pattern = r"readline-(\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_gdbm_versions() -> list[str]: @@ -369,9 +352,7 @@ def detect_gdbm_versions() -> list[str]: content = fetch_url_content(url) pattern = r"gdbm-(\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_libxcrypt_versions() -> list[str]: @@ -380,9 +361,7 @@ def detect_libxcrypt_versions() -> list[str]: content = fetch_url_content(url) pattern = r'v(\d+\.\d+\.\d+)"' matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_krb5_versions() -> list[str]: @@ -392,9 +371,7 @@ def detect_krb5_versions() -> list[str]: # krb5 versions are like 1.22/ pattern = r"(\d+\.\d+)/" matches = re.findall(pattern, content) - majors = sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + majors = sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) if not majors: return [] @@ -405,9 +382,7 @@ def detect_krb5_versions() -> list[str]: pattern = r"krb5-(\d+\.\d+(\.\d+)?)\.tar\.gz" matches = re.findall(pattern, content) versions = [m[0] for m in matches] - return sorted( - set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_uuid_versions() -> list[str]: @@ -416,9 +391,7 @@ def detect_uuid_versions() -> list[str]: content = fetch_url_content(url) pattern = r"libuuid-(\d+\.\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_tirpc_versions() -> list[str]: @@ -427,9 +400,7 @@ def detect_tirpc_versions() -> list[str]: content = fetch_url_content(url) pattern = r"(\d+\.\d+\.\d+)/" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_expat_versions() -> list[str]: @@ -441,9 +412,7 @@ def detect_expat_versions() -> list[str]: matches = re.findall(pattern, content) # Convert R_2_7_3 to 2.7.3 versions = [f"{m[0]}.{m[1]}.{m[2]}" for m in matches] - return sorted( - set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_cpython_bin_deps_versions(name: str) -> list[str]: @@ -454,9 +423,7 @@ def detect_cpython_bin_deps_versions(name: str) -> list[str]: pattern = rf"{name}-(\d+\.\d+(\.\d+)*)\"" matches = re.findall(pattern, content) versions = [m[0] for m in matches] - return sorted( - set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_perl_versions() -> list[str]: @@ -475,9 +442,7 @@ def detect_perl_versions() -> list[str]: patch = int(m[3:4]) subpatch = int(m[4:]) versions.append(f"{major}.{minor}.{patch}.{subpatch}") - return sorted( - set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_mpdecimal_versions() -> list[str]: @@ -486,9 +451,7 @@ def detect_mpdecimal_versions() -> list[str]: content = fetch_url_content(url) pattern = r"mpdecimal-(\d+\.\d+\.\d+)\.tar\.gz" matches = re.findall(pattern, content) - return sorted( - set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(matches), key=lambda v: [int(x) for x in v.split(".")], reverse=True) def detect_nasm_versions() -> list[str]: @@ -498,14 +461,10 @@ def detect_nasm_versions() -> list[str]: pattern = r'href="(\d+\.\d+(\.\d+)?)/"' matches = re.findall(pattern, content) versions = [m[0] for m in matches] - return sorted( - set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True - ) + return sorted(set(versions), key=lambda v: [int(x) for x in v.split(".")], reverse=True) -def update_dependency_versions( - path: pathlib.Path, deps_to_update: list[str] | None = None -) -> None: +def update_dependency_versions(path: pathlib.Path, deps_to_update: list[str] | None = None) -> None: """ Update dependency versions in python-versions.json. @@ -605,8 +564,7 @@ def update_dependency_versions( checksum = sha256_digest(download_path) print(f"SHA-256: {checksum}") url_template = ( - "https://github.com/openssl/openssl/releases/download/" - "openssl-{version}/openssl-{version}.tar.gz" + "https://github.com/openssl/openssl/releases/download/openssl-{version}/openssl-{version}.tar.gz" ) dependencies["openssl"][latest] = { "url": url_template, @@ -624,9 +582,7 @@ def update_dependency_versions( sqlite_versions = detect_sqlite_versions() if sqlite_versions: latest_version, latest_sqliteversion = sqlite_versions[0] - print( - f"Latest SQLite: {latest_version} (sqlite version {latest_sqliteversion})" - ) + print(f"Latest SQLite: {latest_version} (sqlite version {latest_sqliteversion})") if "sqlite" not in dependencies: dependencies["sqlite"] = {} if latest_version not in dependencies["sqlite"]: @@ -861,9 +817,7 @@ def update_dependency_versions( dependencies["krb5"] = {} if latest not in dependencies["krb5"]: major_minor = ".".join(latest.split(".")[:2]) - url = ( - f"https://kerberos.org/dist/krb5/{major_minor}/krb5-{latest}.tar.gz" - ) + url = f"https://kerberos.org/dist/krb5/{major_minor}/krb5-{latest}.tar.gz" print(f"Downloading {url}...") try: download_path = download_url(url, cwd) @@ -1067,7 +1021,7 @@ def create_pyversions(path: pathlib.Path) -> None: dependencies = {} for version in versions: - if version >= Version("3.14"): + if version >= Version("3.15"): continue if str(version) in pydata: @@ -1082,8 +1036,12 @@ def create_pyversions(path: pathlib.Path) -> None: else: url = ARCHIVE.format(version=url_version, ext="tgz") download_path = download_url(url, cwd) - sig_path = download_url(f"{url}.asc", cwd) - verified = verify_signature(download_path, sig_path) + if version < Version("3.14"): + sig_path = download_url(f"{url}.asc", cwd) + verified = verify_signature(download_path, sig_path) + else: + print(f"Skipping signature verification for {version}") + verified = True if verified: print(f"Version {version} has digest {digest(download_path)}") pydata[str(version)] = digest(download_path) @@ -1127,11 +1085,7 @@ def python_versions( raise RuntimeError("No versions file found") data = json.loads(readfrom.read_text()) # Handle both old format (flat dict) and new format (nested with "python" key) - pyversions = ( - data.get("python", data) - if isinstance(data, dict) and "python" in data - else data - ) + pyversions = data.get("python", data) if isinstance(data, dict) and "python" in data else data versions = [Version(_) for _ in pyversions] if minor: mv = Version(minor) @@ -1217,7 +1171,7 @@ def setup_parser( ) subparser.add_argument( "--version", - default="3.13", + default="3.14", type=str, help="The python version [default: %(default)s]", ) @@ -1318,21 +1272,13 @@ def main(args: argparse.Namespace) -> None: # Compare versions if current_version == latest_version: - print( - f"{ok_symbol} {dep_name:12} {current_version:15} " f"(up-to-date)" - ) + print(f"{ok_symbol} {dep_name:12} {current_version:15} (up-to-date)") up_to_date.append(dep_name) elif current_version: - print( - f"{update_symbol} {dep_name:12} {current_version:15} " - f"{arrow} {latest_version} (update available)" - ) + print(f"{update_symbol} {dep_name:12} {current_version:15} {arrow} {latest_version} (update available)") updates_available.append((dep_name, current_version, latest_version)) else: - print( - f"{new_symbol} {dep_name:12} {'(not tracked)':15} " - f"{arrow} {latest_version}" - ) + print(f"{new_symbol} {dep_name:12} {'(not tracked)':15} {arrow} {latest_version}") updates_available.append((dep_name, None, latest_version)) # Summary diff --git a/relenv/relocate.py b/relenv/relocate.py index 2ceddea4..f1c1c05c 100755 --- a/relenv/relocate.py +++ b/relenv/relocate.py @@ -12,7 +12,6 @@ import shutil as _shutil import subprocess as _subprocess import sys as _sys -from typing import Optional log = logging.getLogger(__name__) @@ -73,10 +72,10 @@ LC_RPATH = "LC_RPATH" # Cache for readelf binary path -_READELF_BINARY: Optional[str] = None +_READELF_BINARY: str | None = None # Cache for patchelf binary path -_PATCHELF_BINARY: Optional[str] = None +_PATCHELF_BINARY: str | None = None def _get_readelf_binary() -> str: @@ -192,11 +191,10 @@ def parse_otool_l(stdout: str) -> dict[str, list[str]]: :rtype: dict """ in_cmd = False - cmd: Optional[str] = None - name: Optional[str] = None + cmd: str | None = None + name: str | None = None data: dict[str, list[str]] = {} for line in [x.strip() for x in stdout.split("\n")]: - if not line: continue @@ -252,9 +250,7 @@ def parse_macho(path: str | os.PathLike[str]) -> dict[str, list[str]] | None: :return: The parsed relevant RPATH content, or None if it isn't an object file :rtype: dict or None """ - proc = subprocess.run( - ["otool", "-l", path], stderr=subprocess.PIPE, stdout=subprocess.PIPE - ) + proc = subprocess.run(["otool", "-l", path], stderr=subprocess.PIPE, stdout=subprocess.PIPE) stdout = proc.stdout.decode() if stdout.find("is not an object file") != -1: return None @@ -272,9 +268,7 @@ def parse_rpath(path: str | os.PathLike[str]) -> list[str]: :rtype: list """ readelf = _get_readelf_binary() - proc = subprocess.run( - [readelf, "-d", path], stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) + proc = subprocess.run([readelf, "-d", path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) return parse_readelf_d(proc.stdout.decode()) @@ -318,18 +312,14 @@ def handle_macho( shutil.copymode(x, y) log.info("Copied %s to %s", x, y) log.info("Use %s to %s", y, path_str) - z = pathlib.Path("@loader_path") / os.path.relpath( - y, path_obj.resolve().parent - ) + z = pathlib.Path("@loader_path") / os.path.relpath(y, path_obj.resolve().parent) cmd = ["install_name_tool", "-change", x, str(z), path_str] subprocess.run(cmd) log.info("Changed %s to %s in %s", x, z, path_str) return obj -def is_in_dir( - filepath: str | os.PathLike[str], directory: str | os.PathLike[str] -) -> bool: +def is_in_dir(filepath: str | os.PathLike[str], directory: str | os.PathLike[str]) -> bool: """ Determines whether a file is contained within a directory. diff --git a/relenv/runtime.py b/relenv/runtime.py index 04c938cc..a756b5dd 100644 --- a/relenv/runtime.py +++ b/relenv/runtime.py @@ -10,6 +10,7 @@ gcc. This ensures when using pip any c dependencies are compiled against the proper glibc version. """ + from __future__ import annotations import contextlib @@ -25,32 +26,25 @@ import sys as _sys import textwrap import warnings as _warnings -from importlib.machinery import ModuleSpec -from types import ModuleType -from typing import ( - Any, - Callable, - Dict, - Iterable, - Iterator, - Optional, - Sequence, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, cast + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Iterator, Sequence + from importlib.machinery import ModuleSpec + from types import ModuleType # The tests monkeypatch these module-level imports (e.g., json.loads) inside # relenv.runtime itself; keeping them as Any both preserves test isolation—no # need to patch the global stdlib modules—and avoids mypy attr-defined noise # while still exercising the real runtime wiring. -json = cast(Any, _json) -importlib = cast(Any, _importlib) -site = cast(Any, _site) -subprocess = cast(Any, _subprocess) -sys = cast(Any, _sys) -ctypes = cast(Any, _ctypes) -shutil = cast(Any, _shutil) -warnings = cast(Any, _warnings) +json = cast("Any", _json) +importlib = cast("Any", _importlib) +site = cast("Any", _site) +subprocess = cast("Any", _subprocess) +sys = cast("Any", _sys) +ctypes = cast("Any", _ctypes) +shutil = cast("Any", _shutil) +warnings = cast("Any", _warnings) __all__ = [ "sys", @@ -63,8 +57,8 @@ "warnings", ] -PathType = Union[str, os.PathLike[str]] -ConfigVars = Dict[str, str] +PathType = str | os.PathLike[str] +ConfigVars = dict[str, str] # relenv.pth has a __file__ which is set to the path to site.py of the python # interpreter being used. We're using that to determine the proper @@ -92,18 +86,16 @@ def path_import(name: str, path: PathType) -> ModuleType: return module -_COMMON: Optional[ModuleType] = None -_RELOCATE: Optional[ModuleType] = None -_BUILDENV: Optional[ModuleType] = None +_COMMON: ModuleType | None = None +_RELOCATE: ModuleType | None = None +_BUILDENV: ModuleType | None = None def common() -> ModuleType: """Return the cached ``relenv.common`` module.""" global _COMMON if _COMMON is None: - _COMMON = path_import( - "relenv.common", str(pathlib.Path(__file__).parent / "common.py") - ) + _COMMON = path_import("relenv.common", str(pathlib.Path(__file__).parent / "common.py")) return _COMMON @@ -111,9 +103,7 @@ def relocate() -> ModuleType: """Return the cached ``relenv.relocate`` module.""" global _RELOCATE if _RELOCATE is None: - _RELOCATE = path_import( - "relenv.relocate", str(pathlib.Path(__file__).parent / "relocate.py") - ) + _RELOCATE = path_import("relenv.relocate", str(pathlib.Path(__file__).parent / "relocate.py")) return _RELOCATE @@ -121,9 +111,7 @@ def buildenv() -> ModuleType: """Return the cached ``relenv.buildenv`` module.""" global _BUILDENV if _BUILDENV is None: - _BUILDENV = path_import( - "relenv.buildenv", str(pathlib.Path(__file__).parent / "buildenv.py") - ) + _BUILDENV = path_import("relenv.buildenv", str(pathlib.Path(__file__).parent / "buildenv.py")) return _BUILDENV @@ -172,9 +160,7 @@ def relenv_root() -> pathlib.Path: return MODULE_DIR.parent.parent.parent.parent -def _build_shebang( - func: Callable[..., bytes], *args: Any, **kwargs: Any -) -> Callable[..., bytes]: +def _build_shebang(func: Callable[..., bytes], *args: Any, **kwargs: Any) -> Callable[..., bytes]: """ Build a shebang to point to the proper location. @@ -188,20 +174,16 @@ def wrapped(self: Any, *args: Any, **kwargs: Any) -> bytes: if TARGET.TARGET: scripts = pathlib.Path(_ensure_target_path()).absolute() / "bin" try: - interpreter = common().relative_interpreter( - sys.RELENV, scripts, pathlib.Path(sys.executable).resolve() - ) + interpreter = common().relative_interpreter(sys.RELENV, scripts, pathlib.Path(sys.executable).resolve()) except ValueError: debug(f"Relenv Value Error - _build_shebang {self.target_dir}") original_result: bytes = func(self, *args, **kwargs) return original_result debug(f"Relenv - _build_shebang {scripts} {interpreter}") if sys.platform == "win32": - return ( - str(pathlib.Path("#!") / interpreter).encode() + b"\r\n" - ) + return str(pathlib.Path("#!") / interpreter).encode() + b"\r\n" rel_path = str(pathlib.PurePosixPath("/") / interpreter) - formatted = cast(str, common().format_shebang(rel_path)) + formatted = cast("str", common().format_shebang(rel_path)) return formatted.encode() return wrapped @@ -244,7 +226,7 @@ def wrapped(name: str) -> Any: "LDSHARED": "gcc -shared", } -_SYSTEM_CONFIG_VARS: Optional[ConfigVars] = None +_SYSTEM_CONFIG_VARS: ConfigVars | None = None def system_sysconfig() -> ConfigVars: @@ -278,9 +260,7 @@ def system_sysconfig() -> ConfigVars: return _SYSTEM_CONFIG_VARS -def get_config_vars_wrapper( - func: Callable[..., ConfigVars], mod: ModuleType -) -> Callable[..., ConfigVars]: +def get_config_vars_wrapper(func: Callable[..., ConfigVars], mod: ModuleType) -> Callable[..., ConfigVars]: """ Return a wrapper to resolve paths relative to the relenv root. """ @@ -312,19 +292,17 @@ def wrapped(*args: Any) -> ConfigVars: return wrapped -def get_paths_wrapper( - func: Callable[..., Dict[str, str]], default_scheme: str -) -> Callable[..., Dict[str, str]]: +def get_paths_wrapper(func: Callable[..., dict[str, str]], default_scheme: str) -> Callable[..., dict[str, str]]: """ Return a wrapper to resolve paths relative to the relenv root. """ @functools.wraps(func) def wrapped( - scheme: Optional[str] = default_scheme, - vars: Optional[Dict[str, str]] = None, + scheme: str | None = default_scheme, + vars: dict[str, str] | None = None, expand: bool = True, - ) -> Dict[str, str]: + ) -> dict[str, str]: paths = func(scheme=scheme, vars=vars, expand=expand) if "RELENV_PIP_DIR" in os.environ: paths["scripts"] = str(relenv_root()) @@ -395,19 +373,13 @@ def wrapper( continue if relocate().is_elf(file): debug(f"Relenv - Found elf {file}") - relocate().handle_elf( - plat / file, rootdir / "lib", True, rootdir - ) + relocate().handle_elf(plat / file, rootdir / "lib", True, rootdir) elif relocate().is_macho(file): otool_bin = shutil.which("otool") if otool_bin: - relocate().handle_macho( - str(plat / file), str(rootdir), True - ) + relocate().handle_macho(str(plat / file), str(rootdir), True) else: - debug( - "The otool command is not available, please run `xcode-select --install`" - ) + debug("The otool command is not available, please run `xcode-select --install`") return wrapper @@ -438,7 +410,6 @@ def wrapper( unpacked_source_directory: Any, req_description: Any, ) -> Any: - pkginfo = pathlib.Path(setup_py_path).parent / "PKG-INFO" with open(pkginfo) as fp: pkg_info = fp.read() @@ -471,12 +442,7 @@ def wrapper( ) egginfo = None if prefix: - sitepack = ( - pathlib.Path(prefix) - / "lib" - / f"python{get_major_version()}" - / "site-packages" - ) + sitepack = pathlib.Path(prefix) / "lib" / f"python{get_major_version()}" / "site-packages" for path in sorted(sitepack.glob("*.egg-info")): if path.name.startswith(f"{name}-{version}"): egginfo = path @@ -499,9 +465,7 @@ def wrapper( continue if relocate().is_elf(file): debug(f"Relenv - Found elf {file}") - relocate().handle_elf( - plat / file, rootdir / "lib", True, rootdir - ) + relocate().handle_elf(plat / file, rootdir / "lib", True, rootdir) return wrapper @@ -523,7 +487,7 @@ def __init__( self.matcher = matcher self.loading = _loading - def matches(self: "Wrapper", module: str) -> bool: + def matches(self: Wrapper, module: str) -> bool: """ Check if wrapper metches module being imported. """ @@ -531,7 +495,7 @@ def matches(self: "Wrapper", module: str) -> bool: return module.startswith(self.module) return self.module == module - def __call__(self: "Wrapper", module_name: str) -> ModuleType: + def __call__(self: Wrapper, module_name: str) -> ModuleType: """ Preform the wrapper operation. """ @@ -545,22 +509,22 @@ class RelenvImporter: def __init__( self, - wrappers: Optional[Iterable[Wrapper]] = None, - _loads: Optional[Dict[str, ModuleType]] = None, + wrappers: Iterable[Wrapper] | None = None, + _loads: dict[str, ModuleType] | None = None, ) -> None: if wrappers is None: wrappers = [] self.wrappers: set[Wrapper] = set(wrappers) if _loads is None: _loads = {} - self._loads: Dict[str, ModuleType] = _loads + self._loads: dict[str, ModuleType] = _loads def find_spec( - self: "RelenvImporter", + self: RelenvImporter, module_name: str, - package_path: Optional[Sequence[str]] = None, + package_path: Sequence[str] | None = None, target: Any = None, - ) -> Optional[ModuleSpec]: + ) -> ModuleSpec | None: """ Find modules being imported. """ @@ -569,14 +533,14 @@ def find_spec( debug(f"RelenvImporter - match {module_name} {package_path} {target}") wrapper.loading = True spec = importlib.util.spec_from_loader(module_name, self) - return cast(Optional[ModuleSpec], spec) + return cast("ModuleSpec | None", spec) return None def find_module( - self: "RelenvImporter", + self: RelenvImporter, module_name: str, - package_path: Optional[Sequence[str]] = None, - ) -> Optional["RelenvImporter"]: + package_path: Sequence[str] | None = None, + ) -> RelenvImporter | None: """ Find modules being imported. """ @@ -587,11 +551,11 @@ def find_module( return self return None - def load_module(self: "RelenvImporter", name: str) -> ModuleType: + def load_module(self: RelenvImporter, name: str) -> ModuleType: """ Load an imported module. """ - mod: Optional[ModuleType] = None + mod: ModuleType | None = None for wrapper in self.wrappers: if wrapper.matches(name): debug(f"RelenvImporter - load_module {name}") @@ -603,13 +567,13 @@ def load_module(self: "RelenvImporter", name: str) -> ModuleType: sys.modules[name] = mod return mod - def create_module(self: "RelenvImporter", spec: ModuleSpec) -> Optional[ModuleType]: + def create_module(self: RelenvImporter, spec: ModuleSpec) -> ModuleType | None: """ Create the module via a spec. """ return self.load_module(spec.name) - def exec_module(self: "RelenvImporter", module: ModuleType) -> None: + def exec_module(self: RelenvImporter, module: ModuleType) -> None: """ Exec module noop. """ @@ -621,7 +585,7 @@ def wrap_sysconfig(name: str) -> ModuleType: Sysconfig wrapper. """ module: ModuleType = importlib.import_module("sysconfig") - mod = cast(Any, module) + mod = cast("Any", module) mod.get_config_var = get_config_var_wrapper(mod.get_config_var) mod.get_config_vars = get_config_vars_wrapper(mod.get_config_vars, mod) mod._PIP_USE_SYSCONFIG = True @@ -640,7 +604,7 @@ def wrap_pip_distlib_scripts(name: str) -> ModuleType: pip.distlib.scripts wrapper. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) mod.ScriptMaker._build_shebang = _build_shebang(mod.ScriptMaker._build_shebang) return module @@ -650,10 +614,8 @@ def wrap_distutils_command(name: str) -> ModuleType: distutils.command wrapper. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) - mod.build_ext.finalize_options = finalize_options_wrapper( - mod.build_ext.finalize_options - ) + mod = cast("Any", module) + mod.build_ext.finalize_options = finalize_options_wrapper(mod.build_ext.finalize_options) return module @@ -662,7 +624,7 @@ def wrap_pip_install_wheel(name: str) -> ModuleType: pip._internal.operations.install.wheel wrapper. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) mod.install_wheel = install_wheel_wrapper(mod.install_wheel) return module @@ -672,7 +634,7 @@ def wrap_pip_install_legacy(name: str) -> ModuleType: pip._internal.operations.install.legacy wrapper. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) mod.install = install_legacy_wrapper(mod.install) return module @@ -685,10 +647,7 @@ def set_env_if_not_set(name: str, value: str) -> None: user. """ if name in os.environ and os.environ[name] != value: - print( - f"Warning: {name} environment not set to relenv's root!\n" - f"expected: {value}\ncurrent: {os.environ[name]}" - ) + print(f"Warning: {name} environment not set to relenv's root!\nexpected: {value}\ncurrent: {os.environ[name]}") else: debug(f"Relenv set {name}") os.environ[name] = value @@ -699,7 +658,7 @@ def wrap_pip_build_wheel(name: str) -> ModuleType: pip._internal.operations.build wrapper. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) def wrap(func: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(func) @@ -734,7 +693,7 @@ class TARGET: """ TARGET: bool = False - PATH: Optional[str] = None + PATH: str | None = None IGNORE: bool = False INSTALL: bool = False @@ -753,7 +712,7 @@ def wrap_cmd_install(name: str) -> ModuleType: Wrap pip install command to store target argument state. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) def wrap_run(func: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(func) @@ -771,9 +730,7 @@ def wrapper(self: Any, options: Any, args: Sequence[str]) -> Any: def wrap_handle_target(func: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(func) - def wrapper( - self: Any, target_dir: str, target_temp_dir: str, upgrade: bool - ) -> int: + def wrapper(self: Any, target_dir: str, target_temp_dir: str, upgrade: bool) -> int: from pip._internal.cli.status_codes import SUCCESS return SUCCESS @@ -781,9 +738,7 @@ def wrapper( return wrapper if hasattr(mod.InstallCommand, "_handle_target_dir"): - mod.InstallCommand._handle_target_dir = wrap_handle_target( - mod.InstallCommand._handle_target_dir - ) + mod.InstallCommand._handle_target_dir = wrap_handle_target(mod.InstallCommand._handle_target_dir) return module @@ -792,17 +747,17 @@ def wrap_locations(name: str) -> ModuleType: Wrap pip locations to fix locations when installing with target. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) def make_scheme_wrapper(func: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(func) def wrapper( dist_name: str, user: bool = False, - home: Optional[PathType] = None, - root: Optional[PathType] = None, + home: PathType | None = None, + root: PathType | None = None, isolated: bool = False, - prefix: Optional[PathType] = None, + prefix: PathType | None = None, ) -> Any: scheme = func(dist_name, user, home, root, isolated, prefix) if TARGET.TARGET and TARGET.INSTALL: @@ -834,7 +789,7 @@ def wrap_req_command(name: str) -> ModuleType: Honor ignore installed option from pip cli. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) def make_package_finder_wrapper(func: Callable[..., Any]) -> Callable[..., Any]: @functools.wraps(func) @@ -862,7 +817,7 @@ def wrap_req_install(name: str) -> ModuleType: Honor ignore installed option from pip cli. """ module: ModuleType = importlib.import_module(name) - mod = cast(Any, module) + mod = cast("Any", module) original = mod.InstallRequirement.install argcount = original.__code__.co_argcount @@ -872,9 +827,9 @@ def wrap_req_install(name: str) -> ModuleType: @functools.wraps(original) def install_wrapper_pep517( self: Any, - root: Optional[PathType] = None, - home: Optional[PathType] = None, - prefix: Optional[PathType] = None, + root: PathType | None = None, + home: PathType | None = None, + prefix: PathType | None = None, warn_script_location: bool = True, use_user_site: bool = False, pycompile: bool = True, @@ -903,9 +858,9 @@ def install_wrapper_pep517( def install_wrapper_pep517_opts( self: Any, global_options: Any = None, - root: Optional[PathType] = None, - home: Optional[PathType] = None, - prefix: Optional[PathType] = None, + root: PathType | None = None, + home: PathType | None = None, + prefix: PathType | None = None, warn_script_location: bool = True, use_user_site: bool = False, pycompile: bool = True, @@ -936,9 +891,9 @@ def install_wrapper_legacy( self: Any, install_options: Any, global_options: Any = None, - root: Optional[PathType] = None, - home: Optional[PathType] = None, - prefix: Optional[PathType] = None, + root: PathType | None = None, + home: PathType | None = None, + prefix: PathType | None = None, warn_script_location: bool = True, use_user_site: bool = False, pycompile: bool = True, @@ -969,9 +924,9 @@ def install_wrapper_legacy( def install_wrapper_generic( self: Any, global_options: Any = None, - root: Optional[PathType] = None, - home: Optional[PathType] = None, - prefix: Optional[PathType] = None, + root: PathType | None = None, + home: PathType | None = None, + prefix: PathType | None = None, warn_script_location: bool = True, use_user_site: bool = False, pycompile: bool = True, @@ -1177,9 +1132,7 @@ def set_openssl_modules_dir(path: str) -> None: cryptopath = str(sys.RELENV / "lib" / "libcrypto.so") libcrypto = ctypes.CDLL(cryptopath) POSSL_LIB_CTX = ctypes.c_void_p - OSSL_PROVIDER_set_default_search_path = ( - libcrypto.OSSL_PROVIDER_set_default_search_path - ) + OSSL_PROVIDER_set_default_search_path = libcrypto.OSSL_PROVIDER_set_default_search_path OSSL_PROVIDER_set_default_search_path.argtypes = (POSSL_LIB_CTX, ctypes.c_char_p) OSSL_PROVIDER_set_default_search_path.restype = ctypes.c_int OSSL_PROVIDER_set_default_search_path(None, path.encode()) @@ -1245,10 +1198,7 @@ def wrapper() -> None: # relenv environment. This can't be done when pip is using build_env to # install packages. This code seems potentially brittle and there may # be reasonable arguments against doing it at all. - if ( - sitecustomize_module is None - or "pip-build-env" not in sitecustomize_module.__file__ - ): + if sitecustomize_module is None or "pip-build-env" not in sitecustomize_module.__file__: _orig = sys.path[:] # Replace sys.path sys.path[:] = common().sanitize_sys_path(sys.path) diff --git a/relenv/toolchain.py b/relenv/toolchain.py index bbe69a78..559196b8 100644 --- a/relenv/toolchain.py +++ b/relenv/toolchain.py @@ -6,8 +6,11 @@ from __future__ import annotations -import argparse import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import argparse def setup_parser( @@ -27,9 +30,7 @@ def main(*args: object, **kwargs: object) -> None: """ Notify users of toolchain command deprecation. """ - sys.stderr.write( - "The relenv toolchain command has been deprecated. Please pip install relenv[toolchain].\n" - ) + sys.stderr.write("The relenv toolchain command has been deprecated. Please pip install relenv[toolchain].\n") sys.stderr.flush() sys.exit(1) diff --git a/setup.cfg b/setup.cfg index 03402ab3..a32990d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,63 +28,3 @@ toolchain = ppbt [sdist] owner = root group = root - -[flake8] -max-line-length = 120 -exclude = - # No need to traverse our git directory - .git, - # Nox virtualenvs are also not important - .nox, - # There's no value in checking cache directories - __pycache__, - # Package build stuff - build, - dist, - # The conf file is mostly autogenerated, ignore it - docs/conf.py, - # Also ignore setup.py, it's mostly a shim - setup.py, - # Ignore our custom pre-commit hooks - .pre-commit-hooks - -ignore = - # D104 Missing docstring in public package - D104, - # D107 Missing docstring in __init__ - D107, - # D200 One-line docstring should fit on one line with quotes - D200, - # D401 First line should be in imperative mood; try rephrasing - D401, - # F403 'from import *' used; unable to detect undefined names - F403, - # F405 '*' may be undefined, or defined from star imports: * - F405 - # line break before binary operator, black does this with pathlib.Path objects - W503, - # TYP001 guard import by TYPE_CHECKING (not needed for py3.10+) - TYP001 - -per-file-ignores = - # F401 imported but unused - __init__.py: F401 - # D100 Missing docstring in public module - # D103 Missing docstring in public function - noxfile.py: D100,D102,D103,D107,D212,E501 - # D100 Missing docstring in public module - docs/source/conf.py: D100 - # D102 Missing docstring in public method - # D103 Missing docstring in public function - # D107 Missing docstring in __init__ - relenv/build/common.py: D102,D103,D107 - # D100 Missing docstring in public module - # D101 Missing docstring in public class - # D102 Missing docstring in public method - # D103 Missing docstring in public function - # D105 Missing docstring in magic method - # D107 Missing docstring in __init__ - # D205 1 blank line required between summary line and description - # D415 First line should end with a period, question mark, or exclamation poin - # W503 line break before binary operator (black causes this) - tests/*.py: D100,D101,D102,D103,D105,D107,D205,D415,W503 diff --git a/tests/_pytest_typing.py b/tests/_pytest_typing.py index 9ff2b470..6d5708b5 100644 --- a/tests/_pytest_typing.py +++ b/tests/_pytest_typing.py @@ -3,9 +3,11 @@ """ Typed helper wrappers for common pytest decorators so mypy understands them. """ + from __future__ import annotations -from typing import Any, Callable, Iterable, Sequence, TypeVar, cast +from collections.abc import Callable, Iterable, Sequence +from typing import Any, TypeVar, cast import pytest @@ -14,12 +16,12 @@ def fixture(*args: Any, **kwargs: Any) -> Callable[[F], F] | F: if args and callable(args[0]) and not kwargs: - func = cast(F, args[0]) - return cast(F, pytest.fixture()(func)) + func = cast("F", args[0]) + return cast("F", pytest.fixture()(func)) def decorator(func: F) -> F: wrapped = pytest.fixture(*args, **kwargs)(func) - return cast(F, wrapped) + return cast("F", wrapped) return decorator @@ -27,7 +29,7 @@ def decorator(func: F) -> F: def mark_skipif(*args: Any, **kwargs: Any) -> Callable[[F], F]: def decorator(func: F) -> F: wrapped = pytest.mark.skipif(*args, **kwargs)(func) - return cast(F, wrapped) + return cast("F", wrapped) return decorator @@ -40,6 +42,6 @@ def parametrize( ) -> Callable[[F], F]: def decorator(func: F) -> F: wrapped = pytest.mark.parametrize(argnames, argvalues, *args, **kwargs)(func) - return cast(F, wrapped) + return cast("F", wrapped) return decorator diff --git a/tests/conftest.py b/tests/conftest.py index 98ddd323..55bc08e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,8 +6,8 @@ import platform import shutil import sys +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, Optional import pytest from _pytest.config import Config @@ -22,7 +22,7 @@ log = logging.getLogger(__name__) -def get_build_version() -> Optional[str]: +def get_build_version() -> str | None: if "RELENV_PY_VERSION" in os.environ: return os.environ["RELENV_PY_VERSION"] builds = list(list_archived_builds()) @@ -33,9 +33,7 @@ def get_build_version() -> Optional[str]: versions.append(version) if versions: version = versions[0] - log.warning( - "Environment RELENV_PY_VERSION not set, detected version %s", version - ) + log.warning("Environment RELENV_PY_VERSION not set, detected version %s", version) return version return None diff --git a/tests/test_build.py b/tests/test_build.py index a055efa5..d3085eb2 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -74,9 +74,7 @@ def mock_get_toolchain(arch=None, root=None): # Access toolchain property - should call get_toolchain once toolchain = builder.toolchain - assert ( - call_count["count"] == 1 - ), "get_toolchain should be called when property is accessed" + assert call_count["count"] == 1, "get_toolchain should be called when property is accessed" assert toolchain == pathlib.Path("/fake/toolchain/aarch64") # Access again - should use cached value, not call again @@ -90,9 +88,7 @@ def mock_get_toolchain(arch=None, root=None): # Access after arch change - should call get_toolchain again toolchain3 = builder.toolchain - assert ( - call_count["count"] == 2 - ), "get_toolchain should be called again after arch change" + assert call_count["count"] == 2, "get_toolchain should be called again after arch change" assert toolchain3 == pathlib.Path("/fake/toolchain/x86_64") @@ -100,9 +96,7 @@ def test_verify_checksum(fake_download: pathlib.Path, fake_download_md5: str) -> assert verify_checksum(fake_download, fake_download_md5) is True -def test_verify_checksum_sha256( - fake_download: pathlib.Path, fake_download_sha256: str -) -> None: +def test_verify_checksum_sha256(fake_download: pathlib.Path, fake_download_sha256: str) -> None: """Test SHA-256 checksum validation.""" assert verify_checksum(fake_download, fake_download_sha256) is True @@ -183,9 +177,7 @@ def test_get_dependency_version_wrong_platform() -> None: # Build stats tests -def test_build_stats_save_load( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_build_stats_save_load(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test saving and loading build statistics.""" monkeypatch.setattr("relenv.build.common.ui.DATA_DIR", tmp_path) @@ -205,18 +197,14 @@ def test_build_stats_save_load( assert loaded["openssl"]["samples"] == 2 -def test_build_stats_load_nonexistent( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_build_stats_load_nonexistent(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test loading stats when file doesn't exist returns empty dict.""" monkeypatch.setattr("relenv.build.common.ui.DATA_DIR", tmp_path) loaded = load_build_stats() assert loaded == {} -def test_build_stats_update_new_step( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_build_stats_update_new_step(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test updating stats for a new build step.""" monkeypatch.setattr("relenv.build.common.ui.DATA_DIR", tmp_path) @@ -230,9 +218,7 @@ def test_build_stats_update_new_step( assert stats["python"]["last_lines"] == 100 -def test_build_stats_update_existing_step( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_build_stats_update_existing_step(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """Test updating stats for an existing step uses exponential moving average.""" monkeypatch.setattr("relenv.build.common.ui.DATA_DIR", tmp_path) @@ -493,9 +479,7 @@ def test_copy_pyconfig_h_legacy(tmp_path: pathlib.Path) -> None: """Python <= 3.12: copies PC/pyconfig.h (no pyconfig.h.in template).""" from relenv.build.windows import copy_pyconfig_h - source, build_dir, dest_dir = _make_layout( - tmp_path, has_in=False, pc_content="/* checked in 3.12 */\n" - ) + source, build_dir, dest_dir = _make_layout(tmp_path, has_in=False, pc_content="/* checked in 3.12 */\n") # Ensure the build_dir variant would NOT be picked up if it happened to # exist as well -- the legacy path takes precedence when there's no .in. (build_dir / "pyconfig.h").write_text("/* should be ignored */\n") @@ -511,9 +495,7 @@ def test_copy_pyconfig_h_generated(tmp_path: pathlib.Path) -> None: """Python 3.13+: copies the generated pyconfig.h from build_dir.""" from relenv.build.windows import copy_pyconfig_h - source, build_dir, dest_dir = _make_layout( - tmp_path, has_in=True, build_content="/* generated 3.13 */\n" - ) + source, build_dir, dest_dir = _make_layout(tmp_path, has_in=True, build_content="/* generated 3.13 */\n") # A stale PC/pyconfig.h must NOT win when pyconfig.h.in is present. (source / "PC" / "pyconfig.h").write_text("/* stale legacy */\n") diff --git a/tests/test_common.py b/tests/test_common.py index 7d76b83e..92362dac 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -11,7 +11,7 @@ import sys import tarfile from types import ModuleType -from typing import BinaryIO, Callable, Literal, Optional +from typing import TYPE_CHECKING, BinaryIO, Literal from unittest.mock import patch import pytest @@ -42,10 +42,11 @@ ) from tests._pytest_typing import mark_skipif, parametrize +if TYPE_CHECKING: + from collections.abc import Callable -def _mock_ppbt_module( - monkeypatch: pytest.MonkeyPatch, triplet: str, archive_path: pathlib.Path -) -> None: + +def _mock_ppbt_module(monkeypatch: pytest.MonkeyPatch, triplet: str, archive_path: pathlib.Path) -> None: """ Provide a lightweight ppbt.common stub so get_toolchain() skips the real extraction. """ @@ -161,9 +162,7 @@ def test_get_toolchain(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) triplet = "aarch64-linux-gnu" monkeypatch.setattr(relenv.common, "DATA_DIR", data_dir, raising=False) monkeypatch.setattr(sys, "platform", "linux") - monkeypatch.setattr( - relenv.common, "get_triplet", lambda machine=None, plat=None: triplet - ) + monkeypatch.setattr(relenv.common, "get_triplet", lambda machine=None, plat=None: triplet) monkeypatch.setenv("RELENV_TOOLCHAIN_CACHE", str(data_dir / "toolchain")) archive_path = tmp_path / "dummy-toolchain.tar.xz" archive_path.write_bytes(b"") @@ -177,26 +176,25 @@ def test_get_toolchain_linux_existing(tmp_path: pathlib.Path) -> None: triplet = "x86_64-linux-gnu" toolchain_path = data_dir / "toolchain" / triplet toolchain_path.mkdir(parents=True) - with patch("relenv.common.DATA_DIR", data_dir), patch( - "sys.platform", "linux" - ), patch("relenv.common.get_triplet", return_value=triplet), patch.dict( - os.environ, - {"RELENV_TOOLCHAIN_CACHE": str(data_dir / "toolchain")}, + with ( + patch("relenv.common.DATA_DIR", data_dir), + patch("sys.platform", "linux"), + patch("relenv.common.get_triplet", return_value=triplet), + patch.dict( + os.environ, + {"RELENV_TOOLCHAIN_CACHE": str(data_dir / "toolchain")}, + ), ): ret = get_toolchain() assert ret == toolchain_path -def test_get_toolchain_no_arch( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_get_toolchain_no_arch(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: data_dir = tmp_path / "data" triplet = "x86_64-linux-gnu" monkeypatch.setattr(relenv.common, "DATA_DIR", data_dir, raising=False) monkeypatch.setattr(sys, "platform", "linux") - monkeypatch.setattr( - relenv.common, "get_triplet", lambda machine=None, plat=None: triplet - ) + monkeypatch.setattr(relenv.common, "get_triplet", lambda machine=None, plat=None: triplet) monkeypatch.setenv("RELENV_TOOLCHAIN_CACHE", str(data_dir / "toolchain")) archive_path = tmp_path / "dummy-toolchain.tar.xz" archive_path.write_bytes(b"") @@ -248,7 +246,7 @@ def fake_fetch( fp: BinaryIO, backoff: int, timeout: float, - progress_callback: Optional[Callable[[int, int], None]] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> None: fp.write(data) @@ -268,13 +266,15 @@ def fake_fetch( fp: BinaryIO, backoff: int, timeout: float, - progress_callback: Optional[Callable[[int, int], None]] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> None: raise RelenvException("fail") - with patch("relenv.common.get_download_location", return_value=str(created)), patch( - "relenv.common.fetch_url", side_effect=fake_fetch - ), patch("relenv.common.log") as log_mock: + with ( + patch("relenv.common.get_download_location", return_value=str(created)), + patch("relenv.common.fetch_url", side_effect=fake_fetch), + patch("relenv.common.log") as log_mock, + ): with pytest.raises(RelenvException): download_url("https://example.com/a.txt", dest) log_mock.error.assert_called() @@ -317,21 +317,21 @@ def test_format_shebang_newline() -> None: def test_relative_interpreter_default_location() -> None: - assert relative_interpreter( - "/tmp/relenv", "/tmp/relenv/bin", "/tmp/relenv/bin/python3" - ) == pathlib.Path("..", "bin", "python3") + assert relative_interpreter("/tmp/relenv", "/tmp/relenv/bin", "/tmp/relenv/bin/python3") == pathlib.Path( + "..", "bin", "python3" + ) def test_relative_interpreter_pip_dir_location() -> None: - assert relative_interpreter( - "/tmp/relenv", "/tmp/relenv", "/tmp/relenv/bin/python3" - ) == pathlib.Path("bin", "python3") + assert relative_interpreter("/tmp/relenv", "/tmp/relenv", "/tmp/relenv/bin/python3") == pathlib.Path( + "bin", "python3" + ) def test_relative_interpreter_alternate_location() -> None: - assert relative_interpreter( - "/tmp/relenv", "/tmp/relenv/bar/bin", "/tmp/relenv/bin/python3" - ) == pathlib.Path("..", "..", "bin", "python3") + assert relative_interpreter("/tmp/relenv", "/tmp/relenv/bar/bin", "/tmp/relenv/bin/python3") == pathlib.Path( + "..", "..", "bin", "python3" + ) def test_relative_interpreter_interpreter_not_relative_to_root() -> None: @@ -364,9 +364,11 @@ def test_sanitize_sys_path() -> None: f"{path_prefix}bar{separator}2", f"{path_prefix}lib{separator}3", ] - with patch.object(sys, "prefix", f"{path_prefix}foo"), patch.object( - sys, "base_prefix", f"{path_prefix}bar" - ), patch.dict(os.environ, PYTHONPATH=os.pathsep.join(python_path_entries)): + with ( + patch.object(sys, "prefix", f"{path_prefix}foo"), + patch.object(sys, "base_prefix", f"{path_prefix}bar"), + patch.dict(os.environ, PYTHONPATH=os.pathsep.join(python_path_entries)), + ): new_sys_path = sanitize_sys_path(sys_path) assert new_sys_path != sys_path assert new_sys_path == expected @@ -393,6 +395,7 @@ def test_version_comparisons() -> None: assert Version("3.10.1") > Version("3.10.0") assert Version("3.11") >= Version("3.11.0") assert Version("3.12.2") <= Version("3.12.2") + assert Version("3.14") >= Version("3.14.0") def test_version_parse_string_too_many_parts() -> None: @@ -447,10 +450,11 @@ def test_sanitize_sys_path_with_editable_paths(tmp_path: pathlib.Path) -> None: editable_file = known_path / "__editable__.demo.pth" editable_file.touch() extra_path = str(known_path / "extra") - with patch.object(sys, "prefix", str(base)), patch.object( - sys, "base_prefix", str(base) - ), patch.dict(os.environ, {}, clear=True), patch( - "relenv.common.addpackage", return_value=[extra_path] + with ( + patch.object(sys, "prefix", str(base)), + patch.object(sys, "base_prefix", str(base)), + patch.dict(os.environ, {}, clear=True), + patch("relenv.common.addpackage", return_value=[extra_path]), ): sanitized = sanitize_sys_path([str(known_path)]) assert extra_path in sanitized @@ -464,9 +468,7 @@ def test_makepath_oserror() -> None: assert case == os.path.normcase(expected) -def test_toolchain_respects_relenv_data( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_toolchain_respects_relenv_data(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """ Test that RELENV_DATA environment variable controls toolchain location. @@ -482,9 +484,7 @@ def test_toolchain_respects_relenv_data( # Patch sys.platform to simulate Linux monkeypatch.setattr(sys, "platform", "linux") - monkeypatch.setattr( - relenv.common, "get_triplet", lambda machine=None, plat=None: triplet - ) + monkeypatch.setattr(relenv.common, "get_triplet", lambda machine=None, plat=None: triplet) # Set RELENV_DATA environment variable monkeypatch.setenv("RELENV_DATA", str(data_dir)) @@ -499,9 +499,7 @@ def test_toolchain_respects_relenv_data( assert result == data_dir / "toolchain" -def test_toolchain_uses_cache_without_relenv_data( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_toolchain_uses_cache_without_relenv_data(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: """ Test that toolchain uses cache directory when RELENV_DATA is not set. @@ -528,9 +526,7 @@ def test_toolchain_uses_cache_without_relenv_data( def test_copyright_headers() -> None: """Verify all Python source files have the correct copyright header.""" - expected_header = ( - "# Copyright 2022-2026 Broadcom.\n" "# SPDX-License-Identifier: Apache-2.0\n" - ) + expected_header = "# Copyright 2022-2026 Broadcom.\n# SPDX-License-Identifier: Apache-2.0\n" # Find all Python files in relenv/ and tests/ root = MODULE_DIR.parent @@ -542,16 +538,12 @@ def test_copyright_headers() -> None: # Skip generated and cache files python_files = [ - f - for f in python_files - if "__pycache__" not in f.parts - and ".nox" not in f.parts - and "build" not in f.parts + f for f in python_files if "__pycache__" not in f.parts and ".nox" not in f.parts and "build" not in f.parts ] failures = [] for py_file in python_files: - with open(py_file, "r", encoding="utf-8") as f: + with open(py_file, encoding="utf-8") as f: content = f.read() if not content.startswith(expected_header): diff --git a/tests/test_create.py b/tests/test_create.py index 62ac3089..2ebb9997 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -69,6 +69,7 @@ def test_create_with_minor_version(tmp_path: pathlib.Path) -> None: Version("3.12.6"): "def456", Version("3.12.7"): "ghi789", Version("3.13.1"): "zzz999", + Version("3.14.1"): "yyy888", } def mock_python_versions(minor: str | None = None) -> dict[Version, str]: @@ -77,11 +78,7 @@ def mock_python_versions(minor: str | None = None) -> dict[Version, str]: return all_versions # Filter versions matching the minor version mv = Version(minor) - return { - v: h - for v, h in all_versions.items() - if v.major == mv.major and v.minor == mv.minor - } + return {v: h for v, h in all_versions.items() if v.major == mv.major and v.minor == mv.minor} # Create a fake archive to_be_archived = tmp_path / "to_be_archived" @@ -121,6 +118,7 @@ def test_create_with_full_version(tmp_path: pathlib.Path) -> None: Version("3.12.6"): "def456", Version("3.12.7"): "ghi789", Version("3.13.1"): "zzz999", + Version("3.14.1"): "yyy888", } def mock_python_versions(minor: str | None = None) -> dict[Version, str]: @@ -129,11 +127,7 @@ def mock_python_versions(minor: str | None = None) -> dict[Version, str]: return all_versions # Filter versions matching the minor version mv = Version(minor) - return { - v: h - for v, h in all_versions.items() - if v.major == mv.major and v.minor == mv.minor - } + return {v: h for v, h in all_versions.items() if v.major == mv.major and v.minor == mv.minor} # Create a fake archive to_be_archived = tmp_path / "to_be_archived" @@ -173,6 +167,7 @@ def test_create_with_unknown_minor_version(tmp_path: pathlib.Path) -> None: Version("3.12.6"): "def456", Version("3.12.7"): "ghi789", Version("3.13.1"): "zzz999", + Version("3.14.1"): "yyy888", } # Use appropriate architecture for the platform @@ -185,11 +180,7 @@ def mock_python_versions(minor: str | None = None) -> dict[Version, str]: return all_versions # Filter versions matching the minor version mv = Version(minor) - return { - v: h - for v, h in all_versions.items() - if v.major == mv.major and v.minor == mv.minor - } + return {v: h for v, h in all_versions.items() if v.major == mv.major and v.minor == mv.minor} with patch("relenv.create.python_versions", side_effect=mock_python_versions): with patch("relenv.create.build_arch", return_value=test_arch): diff --git a/tests/test_downloads.py b/tests/test_downloads.py index f50101fe..2d7ea0b3 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -12,16 +12,12 @@ def test_download_url() -> None: - download = Download( - "test", "https://test.com/{version}/test-{version}.tar.xz", version="1.0.0" - ) + download = Download("test", "https://test.com/{version}/test-{version}.tar.xz", version="1.0.0") assert download.url == "https://test.com/1.0.0/test-1.0.0.tar.xz" def test_download_url_change_version() -> None: - download = Download( - "test", "https://test.com/{version}/test-{version}.tar.xz", version="1.0.0" - ) + download = Download("test", "https://test.com/{version}/test-{version}.tar.xz", version="1.0.0") download.version = "1.2.2" assert download.url == "https://test.com/1.2.2/test-1.2.2.tar.xz" @@ -76,9 +72,7 @@ def test_validate_md5sum(tmp_path: pathlib.Path) -> None: def test_validate_md5sum_failed(tmp_path: pathlib.Path) -> None: fake_md5 = "fakemd5" - with patch( - "relenv.build.common.download.verify_checksum", side_effect=RelenvException - ) as run_mock: + with patch("relenv.build.common.download.verify_checksum", side_effect=RelenvException) as run_mock: assert Download.validate_checksum(str(tmp_path), fake_md5) is False run_mock.assert_called_with(str(tmp_path), fake_md5) @@ -96,9 +90,7 @@ def test_validate_signature(tmp_path: pathlib.Path) -> None: def test_validate_signature_failed(tmp_path: pathlib.Path) -> None: sig = "fakesig" - with patch( - "relenv.build.common.download.runcmd", side_effect=RelenvException - ) as run_mock: + with patch("relenv.build.common.download.runcmd", side_effect=RelenvException) as run_mock: assert Download.validate_signature(str(tmp_path), sig) is False run_mock.assert_called_with( ["gpg", "--verify", sig, str(tmp_path)], diff --git a/tests/test_fips_photon.py b/tests/test_fips_photon.py index 4d82f257..502c0c05 100644 --- a/tests/test_fips_photon.py +++ b/tests/test_fips_photon.py @@ -25,9 +25,7 @@ def check_test_environment() -> bool: pytestmark = [ pytest.mark.skipif(not get_build_version(), reason="Build archive does not exist"), - pytest.mark.skipif( - not check_test_environment(), reason="Not running on photon 4 with fips enabled" - ), + pytest.mark.skipif(not check_test_environment(), reason="Not running on photon 4 with fips enabled"), ] diff --git a/tests/test_module_imports.py b/tests/test_module_imports.py index b9fcc2f7..730420f2 100644 --- a/tests/test_module_imports.py +++ b/tests/test_module_imports.py @@ -5,7 +5,8 @@ import importlib import pathlib -from typing import TYPE_CHECKING, Any, Callable, List, Sequence, TypeVar, cast +from collections.abc import Callable, Sequence +from typing import TYPE_CHECKING, Any, TypeVar, cast import pytest @@ -18,12 +19,12 @@ def typed_parametrize(*args: Any, **kwargs: Any) -> Callable[[F], F]: """Type-aware wrapper around pytest.mark.parametrize.""" decorator = pytest.mark.parametrize(*args, **kwargs) - return cast(Callable[[F], F], decorator) + return cast("Callable[[F], F]", decorator) -def _top_level_modules() -> Sequence["ParameterSet"]: +def _top_level_modules() -> Sequence[ParameterSet]: relenv_dir = pathlib.Path(__file__).resolve().parents[1] / "relenv" - params: List["ParameterSet"] = [] + params: list[ParameterSet] = [] for path in sorted(relenv_dir.iterdir()): if not path.is_file() or path.suffix != ".py": continue diff --git a/tests/test_pyversions_runtime.py b/tests/test_pyversions_runtime.py index dd69c373..e6c8afe9 100644 --- a/tests/test_pyversions_runtime.py +++ b/tests/test_pyversions_runtime.py @@ -4,17 +4,20 @@ from __future__ import annotations import hashlib -import pathlib import subprocess -from typing import Any, Dict, Sequence +from typing import TYPE_CHECKING, Any import pytest from relenv import pyversions +if TYPE_CHECKING: + import pathlib + from collections.abc import Sequence + def test_python_versions_returns_versions() -> None: - versions: Dict[pyversions.Version, str] = pyversions.python_versions() + versions: dict[pyversions.Version, str] = pyversions.python_versions() assert versions, "python_versions() should return known versions" first_version = next(iter(versions)) assert isinstance(first_version, pyversions.Version) @@ -64,14 +67,10 @@ def test_get_keyid_parses_second_line() -> None: assert pyversions._get_keyid(proc) == "CB1234" -def test_verify_signature_success( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: - called: Dict[str, list[str]] = {} +def test_verify_signature_success(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: + called: dict[str, list[str]] = {} - def fake_run( - cmd: Sequence[str], **kwargs: Any - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess[bytes]: called.setdefault("cmd", []).extend(cmd) return subprocess.CompletedProcess(cmd, 0, stdout=b"", stderr=b"") @@ -85,9 +84,7 @@ def test_verify_signature_failure_with_missing_key( ) -> None: responses: list[str] = [] - def fake_run( - cmd: Sequence[str], **kwargs: Any - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: Sequence[str], **kwargs: Any) -> subprocess.CompletedProcess[bytes]: if len(responses) == 0: responses.append("first") stderr = b"gpg: error\n[GNUPG:] INV_SGNR 0 ABCDEF12\nNo public key\n" diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 17917630..1d597969 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -231,14 +231,12 @@ def test_handle_elf(tmp_path: pathlib.Path) -> None: libcrypt = tmp_path / "libcrypt.so.2" libcrypt.touch() - ldd_ret = """ + ldd_ret = f""" linux-vdso.so.1 => linux-vdso.so.1 (0x0123456789) libcrypt.so.2 => {libcrypt} (0x0123456789) libm.so.6 => /usr/lib/libm.so.6 (0x0123456789) libc.so.6 => /usr/lib/libc.so.6 (0x0123456789) - """.format( - libcrypt=libcrypt - ).encode() + """.encode() with proj: with patch("subprocess.run", return_value=MagicMock(stdout=ldd_ret)): @@ -259,15 +257,13 @@ def test_handle_elf_rpath_only(tmp_path: pathlib.Path) -> None: fake = tmp_path / "fake.so.2" fake.touch() - ldd_ret = """ + ldd_ret = f""" linux-vdso.so.1 => linux-vdso.so.1 (0x0123456789) libcrypt.so.2 => {libcrypt} (0x0123456789) fake.so.2 => {fake} (0x0123456789) libm.so.6 => /usr/lib/libm.so.6 (0x0123456789) libc.so.6 => /usr/lib/libc.so.6 (0x0123456789) - """.format( - libcrypt=libcrypt, fake=fake - ).encode() + """.encode() with proj: libcrypt.touch() @@ -314,19 +310,17 @@ def test_handle_elf_removes_rpath_when_no_relenv_libs(tmp_path: pathlib.Path) -> module = proj.add_simple_elf("array.so", "lib", "python3.10", "lib-dynload") # ldd output showing only system libraries - ldd_ret = """ + ldd_ret = b""" linux-vdso.so.1 => linux-vdso.so.1 (0x0123456789) libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x0123456789) libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x0123456789) - """.encode() + """ with proj: with patch("subprocess.run", return_value=MagicMock(stdout=ldd_ret)): with patch("relenv.relocate.remove_rpath") as remove_rpath_mock: with patch("relenv.relocate.patch_rpath") as patch_rpath_mock: - handle_elf( - str(module), str(proj.libs_dir), True, str(proj.root_dir) - ) + handle_elf(str(module), str(proj.libs_dir), True, str(proj.root_dir)) # Should remove RPATH, not patch it assert remove_rpath_mock.call_count == 1 assert patch_rpath_mock.call_count == 0 @@ -341,22 +335,18 @@ def test_handle_elf_sets_rpath_when_relenv_libs_present(tmp_path: pathlib.Path) libssl.touch() # ldd output showing relenv-built library - ldd_ret = """ + ldd_ret = f""" linux-vdso.so.1 => linux-vdso.so.1 (0x0123456789) libssl.so.3 => {libssl} (0x0123456789) libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x0123456789) libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x0123456789) - """.format( - libssl=libssl - ).encode() + """.encode() with proj: with patch("subprocess.run", return_value=MagicMock(stdout=ldd_ret)): with patch("relenv.relocate.remove_rpath") as remove_rpath_mock: with patch("relenv.relocate.patch_rpath") as patch_rpath_mock: - handle_elf( - str(module), str(proj.libs_dir), True, str(proj.root_dir) - ) + handle_elf(str(module), str(proj.libs_dir), True, str(proj.root_dir)) # Should patch RPATH, not remove it assert patch_rpath_mock.call_count == 1 assert remove_rpath_mock.call_count == 0 diff --git a/tests/test_relocate_module.py b/tests/test_relocate_module.py index 54d16739..56a5dbac 100644 --- a/tests/test_relocate_module.py +++ b/tests/test_relocate_module.py @@ -7,12 +7,13 @@ import pathlib import shutil import subprocess -from typing import Dict, List, Tuple - -import pytest +from typing import TYPE_CHECKING from relenv import relocate +if TYPE_CHECKING: + import pytest + def test_is_elf_on_text_file(tmp_path: pathlib.Path) -> None: sample = tmp_path / "sample.txt" @@ -51,9 +52,7 @@ def test_parse_otool_output_extracts_rpaths() -> None: assert parsed[relocate.LC_RPATH] == ["@loader_path/../lib"] -def test_patch_rpath_adds_new_entry( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_patch_rpath_adds_new_entry(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: binary = tmp_path / "prog" binary.write_text("dummy") @@ -63,11 +62,9 @@ def test_patch_rpath_adds_new_entry( lambda path: ["$ORIGIN/lib", "/abs/lib"], ) - recorded: Dict[str, List[str]] = {} + recorded: dict[str, list[str]] = {} - def fake_run( - cmd: List[str], **kwargs: object - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: recorded.setdefault("cmd", []).extend(cmd) return subprocess.CompletedProcess(cmd, 0, stdout=b"", stderr=b"") @@ -78,9 +75,7 @@ def fake_run( assert pathlib.Path(recorded["cmd"][-1]) == binary -def test_patch_rpath_skips_when_present( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_patch_rpath_skips_when_present(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: binary = tmp_path / "prog" binary.write_text("dummy") @@ -95,9 +90,7 @@ def fail_run(*_args: object, **_kwargs: object) -> None: assert result == "$ORIGIN/lib" -def test_handle_elf_sets_rpath( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_handle_elf_sets_rpath(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: bin_dir = tmp_path / "bin" lib_dir = tmp_path / "lib" bin_dir.mkdir() @@ -108,19 +101,15 @@ def test_handle_elf_sets_rpath( resident = lib_dir / "libfoo.so" resident.write_text("library") - def fake_run( - cmd: List[str], **kwargs: object - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: if cmd[0] == "ldd": stdout = f"libfoo.so => {resident} (0x00007)\nlibc.so.6 => /lib/libc.so.6 (0x00007)\n" - return subprocess.CompletedProcess( - cmd, 0, stdout=stdout.encode(), stderr=b"" - ) + return subprocess.CompletedProcess(cmd, 0, stdout=stdout.encode(), stderr=b"") raise AssertionError(f"Unexpected command {cmd}") monkeypatch.setattr(relocate.subprocess, "run", fake_run) - captured: Dict[str, str] = {} + captured: dict[str, str] = {} def fake_patch_rpath(path: str, relpath: str) -> str: captured["path"] = path @@ -140,17 +129,13 @@ def fake_patch_rpath(path: str, relpath: str) -> str: assert captured["relpath"] == expected_rpath -def test_patch_rpath_failure( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_patch_rpath_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: binary = tmp_path / "prog" binary.write_text("dummy") monkeypatch.setattr(relocate, "parse_rpath", lambda path: []) - def fake_run( - cmd: List[str], **kwargs: object - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: return subprocess.CompletedProcess(cmd, 1, stdout=b"", stderr=b"err") monkeypatch.setattr(relocate.subprocess, "run", fake_run) @@ -158,23 +143,17 @@ def fake_run( assert relocate.patch_rpath(binary, "$ORIGIN/lib") is False -def test_parse_macho_non_object( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_parse_macho_non_object(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: output = "foo: is not an object file\n" monkeypatch.setattr( relocate.subprocess, "run", - lambda cmd, **kwargs: subprocess.CompletedProcess( - cmd, 0, stdout=output.encode(), stderr=b"" - ), + lambda cmd, **kwargs: subprocess.CompletedProcess(cmd, 0, stdout=output.encode(), stderr=b""), ) assert relocate.parse_macho(tmp_path / "lib.dylib") is None -def test_handle_macho_copies_when_needed( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_handle_macho_copies_when_needed(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: binary = tmp_path / "bin" / "prog" binary.parent.mkdir() binary.write_text("exe") @@ -192,20 +171,14 @@ def test_handle_macho_copies_when_needed( monkeypatch.setattr(os.path, "exists", lambda path: path == str(source_lib)) - copied: Dict[str, Tuple[str, str]] = {} + copied: dict[str, tuple[str, str]] = {} - monkeypatch.setattr( - shutil, "copy", lambda src, dst: copied.setdefault("copy", (src, dst)) - ) - monkeypatch.setattr( - shutil, "copymode", lambda src, dst: copied.setdefault("copymode", (src, dst)) - ) + monkeypatch.setattr(shutil, "copy", lambda src, dst: copied.setdefault("copy", (src, dst))) + monkeypatch.setattr(shutil, "copymode", lambda src, dst: copied.setdefault("copymode", (src, dst))) - recorded: Dict[str, List[str]] = {} + recorded: dict[str, list[str]] = {} - def fake_run( - cmd: List[str], **kwargs: object - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: recorded.setdefault("cmd", []).extend(cmd) return subprocess.CompletedProcess(cmd, 0, stdout=b"", stderr=b"") @@ -218,9 +191,7 @@ def fake_run( assert recorded["cmd"][0] == "install_name_tool" -def test_handle_macho_rpath_only( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_handle_macho_rpath_only(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: binary = tmp_path / "bin" / "prog" binary.parent.mkdir() binary.write_text("exe") @@ -245,9 +216,7 @@ def test_handle_macho_rpath_only( monkeypatch.setattr(shutil, "copy", lambda *_args, **_kw: (_args, _kw)) monkeypatch.setattr(shutil, "copymode", lambda *_args, **_kw: (_args, _kw)) - def fake_run( - cmd: List[str], **kwargs: object - ) -> subprocess.CompletedProcess[bytes]: + def fake_run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: if cmd[0] == "install_name_tool": raise AssertionError("install_name_tool should not run in rpath_only mode") return subprocess.CompletedProcess(cmd, 0, stdout=b"", stderr=b"") diff --git a/tests/test_relocate_tools.py b/tests/test_relocate_tools.py index 41c70ee1..d4abd55b 100644 --- a/tests/test_relocate_tools.py +++ b/tests/test_relocate_tools.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import pathlib -from typing import Iterator +from collections.abc import Iterator from unittest.mock import patch import pytest diff --git a/tests/test_runtime.py b/tests/test_runtime.py index 71acdd44..f3800162 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -9,7 +9,6 @@ import pathlib import sys from types import ModuleType, SimpleNamespace -from typing import Optional import pytest @@ -22,12 +21,8 @@ def _raise(exc: Exception): raise exc -def test_path_import_failure( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: - monkeypatch.setattr( - importlib.util, "spec_from_file_location", lambda *args, **kwargs: None - ) +def test_path_import_failure(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: + monkeypatch.setattr(importlib.util, "spec_from_file_location", lambda *args, **kwargs: None) with pytest.raises(ImportError): relenv.runtime.path_import("demo", tmp_path / "demo.py") @@ -40,9 +35,7 @@ def test_path_import_success(tmp_path: pathlib.Path) -> None: assert sys.modules["temp_mod"] is mod -def test_debug_print( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: +def test_debug_print(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: monkeypatch.setenv("RELENV_DEBUG", "1") relenv.runtime.debug("hello") out = capsys.readouterr().out @@ -94,9 +87,7 @@ def mywrapper(name: str) -> ModuleType: assert pip._internal.locations.__test_case__ is True -def test_set_env_if_not_set( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: +def test_set_env_if_not_set(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: env_name = "RELENV_TEST_ENV" monkeypatch.delenv(env_name, raising=False) relenv.runtime.set_env_if_not_set(env_name, "value") @@ -127,9 +118,7 @@ def fake_exists(path: pathlib.Path) -> bool: monkeypatch.setattr(pathlib.Path, "exists", fake_exists) expected = {"AR": "ar"} completed = SimpleNamespace(stdout=json.dumps(expected).encode(), returncode=0) - monkeypatch.setattr( - relenv.runtime.subprocess, "run", lambda *args, **kwargs: completed - ) + monkeypatch.setattr(relenv.runtime.subprocess, "run", lambda *args, **kwargs: completed) result = relenv.runtime.system_sysconfig() assert result["AR"] == "ar" @@ -152,9 +141,7 @@ def test_system_sysconfig_fallback(monkeypatch: pytest.MonkeyPatch) -> None: assert result == relenv.runtime.CONFIG_VARS_DEFAULTS -def test_install_cargo_config_creates_file( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_cargo_config_creates_file(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "platform", "linux") data_dir = tmp_path / "data" data_dir.mkdir() @@ -183,15 +170,11 @@ def __init__(self, data: pathlib.Path) -> None: def test_build_shebang_value_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr( relenv.runtime, "common", - lambda: SimpleNamespace( - relative_interpreter=lambda *args, **kwargs: _raise(ValueError("boom")) - ), + lambda: SimpleNamespace(relative_interpreter=lambda *args, **kwargs: _raise(ValueError("boom"))), ) called = {"count": 0} @@ -208,15 +191,11 @@ def original(self: object, *args: object, **kwargs: object) -> bytes: # type: i def test_build_shebang_windows(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "platform", "win32", raising=False) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr( relenv.runtime, "common", - lambda: SimpleNamespace( - relative_interpreter=lambda *args: pathlib.Path("python.exe") - ), + lambda: SimpleNamespace(relative_interpreter=lambda *args: pathlib.Path("python.exe")), ) def original(self: object) -> bytes: # type: ignore[override] @@ -234,9 +213,7 @@ def test_get_config_var_wrapper_bindir(monkeypatch: pytest.MonkeyPatch) -> None: assert result == pathlib.Path("/root/Scripts") -def test_get_config_var_wrapper_other( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: +def test_get_config_var_wrapper_other(monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: monkeypatch.setattr(relenv.runtime, "relenv_root", lambda: pathlib.Path("/root")) result = relenv.runtime.get_config_var_wrapper(lambda name: "value")("OTHER") assert result == "value" @@ -244,9 +221,7 @@ def test_get_config_var_wrapper_other( def test_system_sysconfig_json_error(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime, "_SYSTEM_CONFIG_VARS", None, raising=False) - monkeypatch.setattr( - pathlib.Path, "exists", lambda self: str(self) == "/usr/bin/python3" - ) + monkeypatch.setattr(pathlib.Path, "exists", lambda self: str(self) == "/usr/bin/python3") monkeypatch.setattr( relenv.runtime.subprocess, "run", @@ -262,9 +237,7 @@ def fake_loads(_data: bytes) -> dict: def test_get_paths_wrapper_updates_scripts(monkeypatch: pytest.MonkeyPatch) -> None: - def original_get_paths( - *, scheme: Optional[str], vars: Optional[dict[str, str]], expand: bool - ) -> dict[str, str]: + def original_get_paths(*, scheme: str | None, vars: dict[str, str] | None, expand: bool) -> dict[str, str]: return {"scripts": "/original/scripts"} wrapped = relenv.runtime.get_paths_wrapper(original_get_paths, "default") @@ -371,9 +344,7 @@ def original(self: Dummy, *args: object, **kwargs: object) -> None: monkeypatch.delenv("RELENV_BUILDENV", raising=False) -def test_install_wheel_wrapper_processes_record( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_wheel_wrapper_processes_record(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("RELENV_BUILDENV", "1") plat_dir = tmp_path / "plat" info_dir = plat_dir / "demo.dist-info" @@ -438,9 +409,7 @@ def original_install(*_args: object, **_kwargs: object) -> str: assert handled and handled[0][0].name == "libdemo.so" -def test_install_wheel_wrapper_missing_file( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_wheel_wrapper_missing_file(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("RELENV_BUILDENV", "1") plat_dir = tmp_path / "plat" info_dir = plat_dir / "demo.dist-info" @@ -476,9 +445,7 @@ def test_install_wheel_wrapper_missing_file( ) -def test_install_wheel_wrapper_macho_with_otool( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_wheel_wrapper_macho_with_otool(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("RELENV_BUILDENV", "1") plat_dir = tmp_path / "plat" info_dir = plat_dir / "demo.dist-info" @@ -520,9 +487,7 @@ def test_install_wheel_wrapper_macho_with_otool( ) -def test_install_wheel_wrapper_macho_without_otool( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_wheel_wrapper_macho_without_otool(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("RELENV_BUILDENV", "1") plat_dir = tmp_path / "plat" info_dir = plat_dir / "demo.dist-info" @@ -543,9 +508,7 @@ def test_install_wheel_wrapper_macho_without_otool( lambda: SimpleNamespace( is_elf=lambda path: False, is_macho=lambda path: True, - handle_macho=lambda *args, **kwargs: _raise( - AssertionError("unexpected macho") - ), + handle_macho=lambda *args, **kwargs: _raise(AssertionError("unexpected macho")), ), ) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda cmd: None) @@ -569,9 +532,7 @@ def test_install_wheel_wrapper_macho_without_otool( assert any("otool command is not available" in msg for msg in messages) -def test_install_wheel_wrapper_skips_without_buildenv( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_wheel_wrapper_skips_without_buildenv(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("RELENV_BUILDENV", raising=False) plat_dir = tmp_path / "plat" info_dir = plat_dir / "demo.dist-info" @@ -637,25 +598,15 @@ def original_install(*_args: object, **_kwargs: object) -> str: assert not handled -def test_install_legacy_wrapper_prefix( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_install_legacy_wrapper_prefix(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: pkg_dir = tmp_path / "pkg" pkg_dir.mkdir() (pkg_dir / "PKG-INFO").write_text("Version: 1.0\nName: demo\n") - sitepack = ( - tmp_path - / "prefix" - / "lib" - / f"python{relenv.runtime.get_major_version()}" - / "site-packages" - ) + sitepack = tmp_path / "prefix" / "lib" / f"python{relenv.runtime.get_major_version()}" / "site-packages" egg_dir = sitepack / "demo-1.0.egg-info" egg_dir.mkdir(parents=True) (egg_dir / "installed-files.txt").write_text("missing.so\n") - scheme = SimpleNamespace( - purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure") - ) + scheme = SimpleNamespace(purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure")) module = ModuleType("pip._internal.operations.install.legacy.prefix") module.install = lambda *args, **kwargs: None # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, module.__name__, module) @@ -684,9 +635,7 @@ def test_install_legacy_wrapper_prefix( ) -def test_install_legacy_wrapper_no_egginfo( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_install_legacy_wrapper_no_egginfo(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: pkg_dir = tmp_path / "pkg" pkg_dir.mkdir() (pkg_dir / "PKG-INFO").write_text("Name: demo\nVersion: 1.0\n") @@ -714,18 +663,14 @@ def test_install_legacy_wrapper_no_egginfo( ) -def test_install_legacy_wrapper_file_missing( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_install_legacy_wrapper_file_missing(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: pkg_dir = tmp_path / "pkg" pkg_dir.mkdir() (pkg_dir / "PKG-INFO").write_text("Name: demo\nVersion: 1.0\n") egg_dir = tmp_path / "pure" / "demo-1.0.egg-info" egg_dir.mkdir(parents=True) (egg_dir / "installed-files.txt").write_text("missing.so\n") - scheme = SimpleNamespace( - purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure") - ) + scheme = SimpleNamespace(purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure")) module = ModuleType("pip._internal.operations.install.legacy.missing") module.install = lambda *args, **kwargs: None # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, module.__name__, module) @@ -754,9 +699,7 @@ def test_install_legacy_wrapper_file_missing( ) -def test_install_legacy_wrapper_handles_elf( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_install_legacy_wrapper_handles_elf(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: pkg_dir = tmp_path / "pkg" pkg_dir.mkdir() (pkg_dir / "PKG-INFO").write_text("Name: demo\nVersion: 1.0\n") @@ -766,9 +709,7 @@ def test_install_legacy_wrapper_handles_elf( binary.parent.mkdir(parents=True, exist_ok=True) binary.write_bytes(b"") (egg_dir / "installed-files.txt").write_text(f"{binary}\n") - scheme = SimpleNamespace( - purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure") - ) + scheme = SimpleNamespace(purelib=str(tmp_path / "pure"), platlib=str(tmp_path / "pure")) module = ModuleType("pip._internal.operations.install.legacy.elf") module.install = lambda *args, **kwargs: None # type: ignore[attr-defined] monkeypatch.setitem(sys.modules, module.__name__, module) @@ -841,15 +782,11 @@ def _build_shebang(self, target: str) -> bytes: module.ScriptMaker = ScriptMaker monkeypatch.setitem(sys.modules, module.__name__, module) wrapped = relenv.runtime.wrap_pip_distlib_scripts(module.__name__) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr( relenv.runtime, "common", - lambda: SimpleNamespace( - relative_interpreter=lambda *args, **kwargs: _raise(ValueError("boom")) - ), + lambda: SimpleNamespace(relative_interpreter=lambda *args, **kwargs: _raise(ValueError("boom"))), ) result = wrapped.ScriptMaker()._build_shebang("target") assert result == b"orig" @@ -874,9 +811,7 @@ def finalize_options(self) -> None: monkeypatch.delenv("RELENV_BUILDENV", raising=False) -def test_wrap_pip_build_wheel_sets_env( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_wrap_pip_build_wheel_sets_env(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: relenv.runtime.TARGET.TARGET = False monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) toolchain = tmp_path / "toolchain" / "trip" @@ -905,9 +840,7 @@ def build_wheel_pep517(self, *args: object, **kwargs: object) -> str: # type: i monkeypatch.setitem(sys.modules, module_name, dummy) monkeypatch.setattr(relenv.runtime.importlib, "import_module", lambda name: dummy) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) wrapped = relenv.runtime.wrap_pip_build_wheel(module_name) result = wrapped.build_wheel_pep517("backend", {}, {}) assert result == "built" @@ -958,9 +891,7 @@ def run(self, options: SimpleNamespace, args: list[str]) -> str: options.ran = True return "ran" - def _handle_target_dir( - self, target_dir: str, target_temp_dir: str, upgrade: bool - ) -> str: + def _handle_target_dir(self, target_dir: str, target_temp_dir: str, upgrade: bool) -> str: return "handled" fake_module.InstallCommand = FakeInstallCommand @@ -980,9 +911,7 @@ def fake_import_module(name: str) -> ModuleType: monkeypatch.setattr(relenv.runtime.importlib, "import_module", fake_import_module) wrapped = relenv.runtime.wrap_cmd_install(fake_module.__name__) - options = SimpleNamespace( - use_user_site=False, target_dir="/tmp/target", ignore_installed=True - ) + options = SimpleNamespace(use_user_site=False, target_dir="/tmp/target", ignore_installed=True) command = wrapped.InstallCommand() result = command.run(options, []) @@ -1015,9 +944,7 @@ def run(self, options: SimpleNamespace, args: list[str]) -> str: ) wrapped = relenv.runtime.wrap_cmd_install(fake_module.__name__) - options = SimpleNamespace( - use_user_site=True, target_dir=None, ignore_installed=False - ) + options = SimpleNamespace(use_user_site=True, target_dir=None, ignore_installed=False) result = wrapped.InstallCommand().run(options, []) assert result == "ran" assert relenv.runtime.TARGET.TARGET is False @@ -1100,9 +1027,7 @@ def __init__(self) -> None: fake_module.get_scheme = lambda *args, **kwargs: OriginalScheme() monkeypatch.setitem(sys.modules, fake_module.__name__, fake_module) - monkeypatch.setattr( - relenv.runtime.importlib, "import_module", lambda name: fake_module - ) + monkeypatch.setattr(relenv.runtime.importlib, "import_module", lambda name: fake_module) wrapped = relenv.runtime.wrap_locations(fake_module.__name__) scheme = wrapped.get_scheme("dist") @@ -1161,9 +1086,7 @@ def _build_package_finder( fake_module.RequirementCommand = RequirementCommand monkeypatch.setitem(sys.modules, fake_module.__name__, fake_module) - monkeypatch.setattr( - relenv.runtime.importlib, "import_module", lambda name: fake_module - ) + monkeypatch.setattr(relenv.runtime.importlib, "import_module", lambda name: fake_module) wrapped = relenv.runtime.wrap_req_command(fake_module.__name__) options = SimpleNamespace(ignore_installed=False) @@ -1283,9 +1206,7 @@ def test_wrapsitecustomize_sanitizes_sys_path(monkeypatch: pytest.MonkeyPatch) - "common", lambda: SimpleNamespace(sanitize_sys_path=lambda _: sanitized), ) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) def original() -> None: pass @@ -1328,9 +1249,7 @@ def test_install_cargo_config_non_linux(monkeypatch: pytest.MonkeyPatch) -> None relenv.runtime.install_cargo_config() -def test_install_cargo_config_alt_triplet( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_install_cargo_config_alt_triplet(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) data_dir = tmp_path / "data" data_dir.mkdir() @@ -1354,9 +1273,7 @@ def test_setup_openssl_windows(monkeypatch: pytest.MonkeyPatch) -> None: relenv.runtime.setup_openssl() -def test_setup_openssl_without_binary( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_setup_openssl_without_binary(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "RELENV", tmp_path, raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux") monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: None) @@ -1380,9 +1297,7 @@ def fail_provider(name: str) -> int: assert providers == ["default", "legacy"] -def test_setup_openssl_with_system_binary( - tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_setup_openssl_with_system_binary(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "RELENV", tmp_path, raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux") monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") @@ -1400,9 +1315,7 @@ def test_setup_openssl_with_system_binary( def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: if args[:2] == ["/usr/bin/openssl", "version"]: if "-m" in args: - return SimpleNamespace( - returncode=0, stdout='MODULESDIR: "/usr/lib/ssl"' - ) + return SimpleNamespace(returncode=0, stdout='MODULESDIR: "/usr/lib/ssl"') if "-d" in args: return SimpleNamespace(returncode=0, stdout='OPENSSLDIR: "/etc/ssl"') return SimpleNamespace(returncode=1, stdout="", stderr="error") @@ -1413,8 +1326,7 @@ def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: monkeypatch.setattr( pathlib.Path, "exists", - lambda self: str(self) - in (str(certs_dir), str(tmp_path / "lib" / "libcrypto.so")), + lambda self: str(self) in (str(certs_dir), str(tmp_path / "lib" / "libcrypto.so")), ) monkeypatch.delenv("OPENSSL_MODULES", raising=False) @@ -1475,22 +1387,12 @@ def test_install_cargo_config_toolchain_missing( def test_bootstrap(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[str] = [] - monkeypatch.setattr( - relenv.runtime, "relenv_root", lambda: pathlib.Path("/relbootstrap") - ) + monkeypatch.setattr(relenv.runtime, "relenv_root", lambda: pathlib.Path("/relbootstrap")) monkeypatch.setattr(relenv.runtime, "setup_openssl", lambda: calls.append("ssl")) - monkeypatch.setattr( - relenv.runtime.site, "execsitecustomize", lambda: None, raising=False - ) - monkeypatch.setattr( - relenv.runtime, "setup_crossroot", lambda: calls.append("cross") - ) - monkeypatch.setattr( - relenv.runtime, "install_cargo_config", lambda: calls.append("cargo") - ) - monkeypatch.setattr( - relenv.runtime.warnings, "filterwarnings", lambda *args, **kwargs: None - ) + monkeypatch.setattr(relenv.runtime.site, "execsitecustomize", lambda: None, raising=False) + monkeypatch.setattr(relenv.runtime, "setup_crossroot", lambda: calls.append("cross")) + monkeypatch.setattr(relenv.runtime, "install_cargo_config", lambda: calls.append("cargo")) + monkeypatch.setattr(relenv.runtime.warnings, "filterwarnings", lambda *args, **kwargs: None) original_meta = list(relenv.runtime.sys.meta_path) relenv.runtime.bootstrap() assert relenv.runtime.sys.RELENV == pathlib.Path("/relbootstrap") @@ -1555,9 +1457,7 @@ def test_buildenv_cached(monkeypatch: pytest.MonkeyPatch) -> None: def test_build_shebang_target(monkeypatch: pytest.MonkeyPatch) -> None: relenv.runtime.TARGET.TARGET = True relenv.runtime.TARGET.PATH = "/target" - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr( relenv.runtime, "common", @@ -1570,9 +1470,7 @@ def test_build_shebang_target(monkeypatch: pytest.MonkeyPatch) -> None: def original(self: object) -> bytes: # type: ignore[override] return b"" - result = relenv.runtime._build_shebang(original)( - SimpleNamespace(target_dir="/tmp/scripts") - ) + result = relenv.runtime._build_shebang(original)(SimpleNamespace(target_dir="/tmp/scripts")) shebang = result.decode().strip() assert shebang.startswith("#!") path_part = shebang[2:] @@ -1585,9 +1483,7 @@ def original(self: object) -> bytes: # type: ignore[override] def test_build_shebang_linux(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) class StubCommon: @staticmethod @@ -1603,9 +1499,7 @@ def format_shebang(path: pathlib.Path) -> str: def original(self: object) -> bytes: # type: ignore[override] return b"" - result = relenv.runtime._build_shebang(original)( - SimpleNamespace(target_dir="/tmp/dir") - ) + result = relenv.runtime._build_shebang(original)(SimpleNamespace(target_dir="/tmp/dir")) shebang = result.decode().strip() assert shebang.startswith("#!") path_part = shebang[2:] @@ -1615,9 +1509,7 @@ def original(self: object) -> bytes: # type: ignore[override] def test_setup_openssl_version_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") monkeypatch.setattr(relenv.runtime, "set_openssl_modules_dir", lambda path: None) @@ -1631,9 +1523,7 @@ def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: def test_setup_openssl_parse_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") monkeypatch.setattr(relenv.runtime, "load_openssl_provider", lambda name: 1) @@ -1652,9 +1542,7 @@ def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: def test_setup_openssl_cert_dir_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") monkeypatch.setattr(relenv.runtime, "load_openssl_provider", lambda name: 1) @@ -1662,9 +1550,7 @@ def test_setup_openssl_cert_dir_error(monkeypatch: pytest.MonkeyPatch) -> None: def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: if "-m" in args: - return SimpleNamespace( - returncode=0, stdout='MODULESDIR: "/usr/lib"', stderr="" - ) + return SimpleNamespace(returncode=0, stdout='MODULESDIR: "/usr/lib"', stderr="") return SimpleNamespace(returncode=1, stdout="", stderr="error") monkeypatch.setattr(relenv.runtime.subprocess, "run", fake_run) @@ -1673,9 +1559,7 @@ def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: def test_setup_openssl_cert_dir_parse_error(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") monkeypatch.setattr(relenv.runtime, "load_openssl_provider", lambda name: 1) @@ -1686,21 +1570,15 @@ def test_setup_openssl_cert_dir_parse_error(monkeypatch: pytest.MonkeyPatch) -> def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: if "-m" in args: - return SimpleNamespace( - returncode=0, stdout='MODULESDIR: "/usr/lib"', stderr="" - ) + return SimpleNamespace(returncode=0, stdout='MODULESDIR: "/usr/lib"', stderr="") return SimpleNamespace(returncode=0, stdout="invalid", stderr="") monkeypatch.setattr(relenv.runtime.subprocess, "run", fake_run) relenv.runtime.setup_openssl() -def test_setup_openssl_cert_file( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) +def test_setup_openssl_cert_file(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") cert_dir = tmp_path / "etc" / "ssl" @@ -1710,12 +1588,8 @@ def test_setup_openssl_cert_file( def fake_run(args: list[str], **kwargs: object) -> SimpleNamespace: if "-m" in args: - return SimpleNamespace( - returncode=0, stdout='MODULESDIR: "{}"'.format(cert_dir), stderr="" - ) - return SimpleNamespace( - returncode=0, stdout='OPENSSLDIR: "{}"'.format(cert_dir), stderr="" - ) + return SimpleNamespace(returncode=0, stdout=f'MODULESDIR: "{cert_dir}"', stderr="") + return SimpleNamespace(returncode=0, stdout=f'OPENSSLDIR: "{cert_dir}"', stderr="") monkeypatch.setattr(relenv.runtime.subprocess, "run", fake_run) monkeypatch.setenv("OPENSSL_MODULES", "") @@ -1734,14 +1608,10 @@ def test_set_openssl_modules_dir(monkeypatch: pytest.MonkeyPatch) -> None: class FakeLib: def __init__(self) -> None: - self.OSSL_PROVIDER_set_default_search_path = ( - lambda ctx, path: called.update({"path": path}) or 1 - ) + self.OSSL_PROVIDER_set_default_search_path = lambda ctx, path: called.update({"path": path}) or 1 monkeypatch.setattr(relenv.runtime.ctypes, "CDLL", lambda path: FakeLib()) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "darwin", raising=False) relenv.runtime.set_openssl_modules_dir("/mods") assert called["path"] == b"/mods" @@ -1753,16 +1623,12 @@ def __init__(self) -> None: self.OSSL_PROVIDER_load = lambda ctx, name: 123 monkeypatch.setattr(relenv.runtime.ctypes, "CDLL", lambda path: FakeLib()) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "darwin", raising=False) assert relenv.runtime.load_openssl_provider("default") == 123 -def test_setup_crossroot( - monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path -) -> None: +def test_setup_crossroot(monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path) -> None: monkeypatch.setenv("RELENV_CROSS", str(tmp_path)) original_path = sys.path[:] try: @@ -1775,15 +1641,11 @@ def test_setup_crossroot( def test_setup_openssl_provider_failure(monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) monkeypatch.setattr(relenv.runtime.shutil, "which", lambda _: "/usr/bin/openssl") order: list[str] = [] - monkeypatch.setattr( - relenv.runtime, "set_openssl_modules_dir", lambda path: order.append(path) - ) + monkeypatch.setattr(relenv.runtime, "set_openssl_modules_dir", lambda path: order.append(path)) providers: list[str] = [] monkeypatch.setattr( relenv.runtime, @@ -1795,9 +1657,7 @@ def test_setup_openssl_provider_failure(monkeypatch: pytest.MonkeyPatch) -> None "run", lambda args, **kwargs: SimpleNamespace( returncode=0, - stdout='MODULESDIR: "/usr/lib"' - if "-m" in args - else 'OPENSSLDIR: "/etc/ssl"', + stdout='MODULESDIR: "/usr/lib"' if "-m" in args else 'OPENSSLDIR: "/etc/ssl"', stderr="", ), ) @@ -1827,8 +1687,8 @@ def __init__(self) -> None: def fake_import( name: str, - globals: Optional[dict] = None, - locals: Optional[dict] = None, + globals: dict | None = None, + locals: dict | None = None, fromlist=(), level: int = 0, ): @@ -1869,14 +1729,10 @@ def test_set_openssl_modules_dir_linux(monkeypatch: pytest.MonkeyPatch) -> None: class FakeLib: def __init__(self) -> None: - self.OSSL_PROVIDER_set_default_search_path = ( - lambda ctx, path: called.update({"path": path}) or 1 - ) + self.OSSL_PROVIDER_set_default_search_path = lambda ctx, path: called.update({"path": path}) or 1 monkeypatch.setattr(relenv.runtime.ctypes, "CDLL", lambda path: FakeLib()) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) relenv.runtime.set_openssl_modules_dir("/mods") assert called["path"] == b"/mods" @@ -1888,9 +1744,7 @@ def __init__(self) -> None: self.OSSL_PROVIDER_load = lambda ctx, name: 456 monkeypatch.setattr(relenv.runtime.ctypes, "CDLL", lambda path: FakeLib()) - monkeypatch.setattr( - relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False - ) + monkeypatch.setattr(relenv.runtime.sys, "RELENV", pathlib.Path("/rel"), raising=False) monkeypatch.setattr(relenv.runtime.sys, "platform", "linux", raising=False) assert relenv.runtime.load_openssl_provider("default") == 456 @@ -1928,12 +1782,8 @@ def fake_wrap_sysconfig(name: str) -> ModuleType: monkeypatch.setattr(relenv.runtime, "setup_openssl", lambda: None) monkeypatch.setattr(relenv.runtime, "setup_crossroot", lambda: None) monkeypatch.setattr(relenv.runtime, "install_cargo_config", lambda: None) - monkeypatch.setattr( - relenv.runtime.site, "execsitecustomize", lambda: None, raising=False - ) - monkeypatch.setattr( - relenv.runtime, "wrapsitecustomize", lambda func: func, raising=False - ) + monkeypatch.setattr(relenv.runtime.site, "execsitecustomize", lambda: None, raising=False) + monkeypatch.setattr(relenv.runtime, "wrapsitecustomize", lambda func: func, raising=False) # Mock importer fake_importer = SimpleNamespace() @@ -2011,12 +1861,8 @@ def fake_wrap_sysconfig(name: str) -> ModuleType: monkeypatch.setattr(relenv.runtime, "setup_openssl", lambda: None) monkeypatch.setattr(relenv.runtime, "setup_crossroot", lambda: None) monkeypatch.setattr(relenv.runtime, "install_cargo_config", lambda: None) - monkeypatch.setattr( - relenv.runtime.site, "execsitecustomize", lambda: None, raising=False - ) - monkeypatch.setattr( - relenv.runtime, "wrapsitecustomize", lambda func: func, raising=False - ) + monkeypatch.setattr(relenv.runtime.site, "execsitecustomize", lambda: None, raising=False) + monkeypatch.setattr(relenv.runtime, "wrapsitecustomize", lambda func: func, raising=False) fake_importer = SimpleNamespace() monkeypatch.setattr(relenv.runtime, "importer", fake_importer, raising=False) diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 0f611e0e..39e2b803 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -4,6 +4,7 @@ """ Verify relenv builds. """ + import json import os import pathlib @@ -62,9 +63,7 @@ def _install_ppbt(pexec): ] ) assert p.returncode == 0, "Failed to install ppbt" - p = subprocess.run( - [str(pexec), "-c", "from relenv import common; assert common.get_toolchain()"] - ) + p = subprocess.run([str(pexec), "-c", "from relenv import common; assert common.get_toolchain()"]) assert p.returncode == 0, "Failed to extract toolchain" @@ -185,14 +184,15 @@ def test_pip_install_salt_git(pipexec, build, build_dir, pyexec, build_version): or "3.11" in build_version or "3.12" in build_version or "3.13" in build_version + or "3.14" in build_version ): - pytest.xfail( - "Salt git install fails on Windows (setup.py tries to install missing man pages)" - ) + pytest.xfail("Salt git install fails on Windows (setup.py tries to install missing man pages)") if sys.platform == "darwin" and "3.12" in build_version: pytest.xfail("Salt does not work with 3.12 on macos yet") - if sys.platform == "darwin" and "3.13" in build_version: - pytest.xfail("Salt does not work with 3.13 on macos yet") + if sys.platform == "darwin" and ("3.13" in build_version or "3.14" in build_version): + pytest.xfail("Salt does not work with 3.13+ on macos yet") + if sys.platform == "darwin" and "3.14" in build_version: + pytest.xfail("Salt does not work with 3.14 on macos yet") _install_ppbt(pyexec) @@ -229,13 +229,10 @@ def test_pip_install_salt_git(pipexec, build, build_dir, pyexec, build_version): @pytest.mark.skip_on_darwin @pytest.mark.skip_on_windows @pytest.mark.skipif( - get_build_version() - and packaging.version.parse(get_build_version()) - >= packaging.version.parse("3.11.7"), + get_build_version() and packaging.version.parse(get_build_version()) >= packaging.version.parse("3.11.7"), reason="3.11.7 and greater will not work with 3005.x", ) def test_pip_install_salt(pipexec, build, tmp_path, pyexec): - _install_ppbt(pyexec) packages = [ @@ -282,22 +279,18 @@ def test_symlinked_scripts(pipexec, pyexec, tmp_path, build): # Make sure symlinks work with our custom shebang in the scripts p = subprocess.run([str(script), "--version"]) - assert ( - p.returncode == 0 - ), f"Could not run script for {name}, likely not pinning to the correct python" + assert p.returncode == 0, f"Could not run script for {name}, likely not pinning to the correct python" @pytest.mark.parametrize("salt_branch", ["3006.x", "3007.x", "master"]) -def test_pip_install_salt_w_static_requirements( - pipexec, pyexec, build, tmp_path, salt_branch, build_version -): +def test_pip_install_salt_w_static_requirements(pipexec, pyexec, build, tmp_path, salt_branch, build_version): if salt_branch in ["3007.x", "master"]: pytest.xfail("Known failure") if sys.platform == "darwin" and salt_branch in ["3006.x"]: pytest.xfail("Known failure") - for py_version in ("3.11", "3.12", "3.13"): + for py_version in ("3.11", "3.12", "3.13", "3.14"): if build_version.startswith(py_version): pytest.xfail(f"{py_version} builds fail.") @@ -348,11 +341,8 @@ def test_pip_install_salt_w_static_requirements( @pytest.mark.parametrize("salt_branch", ["3006.x", "master"]) -def test_pip_install_salt_w_package_requirements( - pipexec, pyexec, tmp_path, salt_branch, build_version -): - - for py_version in ("3.11", "3.12", "3.13"): +def test_pip_install_salt_w_package_requirements(pipexec, pyexec, tmp_path, salt_branch, build_version): + for py_version in ("3.11", "3.12", "3.13", "3.14"): if build_version.startswith(py_version): pytest.xfail(f"{py_version} builds fail.") @@ -449,13 +439,15 @@ def test_pip_install_pyzmq( build, tmp_path: pathlib.Path, ) -> None: - if pyzmq_version == "23.2.0" and "3.12" in build_version: pytest.xfail(f"{pyzmq_version} does not install on 3.12") if pyzmq_version == "23.2.0" and "3.13" in build_version: pytest.xfail(f"{pyzmq_version} does not install on 3.13") + if pyzmq_version == "23.2.0" and "3.14" in build_version: + pytest.xfail(f"{pyzmq_version} does not install on 3.14") + if pyzmq_version == "23.2.0" and sys.platform == "darwin": pytest.xfail("pyzmq 23.2.0 fails on macos arm64") @@ -465,6 +457,9 @@ def test_pip_install_pyzmq( if pyzmq_version == "25.1.2" and "3.13" in build_version: pytest.xfail(f"{pyzmq_version} does not install on 3.13") + if pyzmq_version == "25.1.2" and "3.14" in build_version: + pytest.xfail(f"{pyzmq_version} does not install on 3.14") + if pyzmq_version == "25.1.2" and sys.platform == "win32": pytest.xfail("pyzmq 25.1.2 fails on windows") @@ -491,9 +486,7 @@ def test_pip_install_pyzmq( if pyzmq_version == "26.2.0" and sys.platform == "darwin": pytest.xfail(f"{pyzmq_version} does not install on m1 mac") if pyzmq_version == "26.2.0" and sys.platform == "darwin": - env[ - "CFLAGS" - ] = f"{env.get('CFLAGS', '')} -DCMAKE_OSX_ARCHITECTURES='arm64' -DZMQ_HAVE_CURVE=0" + env["CFLAGS"] = f"{env.get('CFLAGS', '')} -DCMAKE_OSX_ARCHITECTURES='arm64' -DZMQ_HAVE_CURVE=0" env = os.environ.copy() if sys.platform == "linux": p = subprocess.run( @@ -509,9 +502,7 @@ def test_pip_install_pyzmq( try: buildenv = json.loads(p.stdout) except json.JSONDecodeError: - assert ( - False - ), f"Failed to decode json: {p.stdout.decode()} {p.stderr.decode()}" + assert False, f"Failed to decode json: {p.stdout.decode()} {p.stderr.decode()}" for k in buildenv: env[k] = buildenv[k] @@ -620,9 +611,7 @@ def test_pip_install_pyzmq( include_flag = f"-I{fake_bsd_root}" for key in ("CFLAGS", "CXXFLAGS", "CPPFLAGS"): env[key] = " ".join(filter(None, [env.get(key, ""), include_flag])).strip() - env["CPATH"] = ":".join( - filter(None, [str(fake_bsd_root), env.get("CPATH", "")]) - ) + env["CPATH"] = ":".join(filter(None, [str(fake_bsd_root), env.get("CPATH", "")])) for key in ("C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH"): env[key] = ":".join(filter(None, [str(fake_bsd_root), env.get(key, "")])) cc_value = env.get("CC") @@ -660,13 +649,9 @@ def test_pip_install_pyzmq( ) assert archive_result.returncode == 0, "Failed to archive libbsd shim" lib_dir_flag = f"-L{fake_bsd_root}" - env["LDFLAGS"] = " ".join( - filter(None, [lib_dir_flag, env.get("LDFLAGS", "")]) - ).strip() + env["LDFLAGS"] = " ".join(filter(None, [lib_dir_flag, env.get("LDFLAGS", "")])).strip() env["LIBS"] = " ".join(filter(None, ["-lbsd", env.get("LIBS", "")])).strip() - env["LIBRARY_PATH"] = ":".join( - filter(None, [str(fake_bsd_root), env.get("LIBRARY_PATH", "")]) - ) + env["LIBRARY_PATH"] = ":".join(filter(None, [str(fake_bsd_root), env.get("LIBRARY_PATH", "")])) env["ac_cv_func_strlcpy"] = "yes" env["ac_cv_func_strlcat"] = "yes" env["ac_cv_have_decl_strlcpy"] = "yes" @@ -740,19 +725,20 @@ def test_pip_install_and_import_libcloud(pipexec, pyexec): def test_pip_install_salt_pip_dir(pipexec, pyexec, build, build_version, arch): - - if "3.12" in build_version: - pytest.xfail("Don't try to install on 3.12 yet") + if "3.12" in build_version or "3.13" in build_version or "3.14" in build_version: + pytest.xfail("Don't try to install on 3.12+ yet") if build_version.startswith("3.11") and sys.platform == "darwin": - pytest.xfail("Known failure on py 3.11 macos") if sys.platform == "win32" and arch == "amd64": pytest.xfail("Known failure on windows amd64") - if sys.platform == "darwin" and "3.13" in build_version: - pytest.xfail("Salt does not work with 3.13 on macos yet") + if sys.platform == "darwin" and ("3.13" in build_version or "3.14" in build_version): + pytest.xfail("Salt does not work with 3.13+ on macos yet") + + if sys.platform == "darwin" and "3.14" in build_version: + pytest.xfail("Salt does not work with 3.14 on macos yet") _install_ppbt(pyexec) env = os.environ.copy() @@ -790,15 +776,13 @@ def test_nox_virtualenvs(pipexec, pyexec, build, tmp_path): session = "fake_session" nox_contents = textwrap.dedent( - """ + f""" import nox @nox.session() - def {}(session): + def {session}(session): session.install("nox") - """.format( - session - ) + """ ) noxfile = tmp_path / "tmp_noxfile.py" noxfile.write_text(nox_contents) @@ -917,7 +901,7 @@ def ssl_version(pyexec, tmp_path): def test_pip_install_m2crypto_relenv_ssl( m2crypto_version, pipexec, pyexec, build, build_version, minor_version, ssl_version ): - if m2crypto_version == "0.38.0" and minor_version in ["3.12", "3.13"]: + if m2crypto_version == "0.38.0" and minor_version in ["3.12", "3.13", "3.14"]: pytest.xfail("Fails due to no distutils") if ssl_version >= (3, 5) and m2crypto_version in ["0.38.0", "0.44.0"]: @@ -978,8 +962,7 @@ def test_pip_install_m2crypto_relenv_ssl( p = subprocess.run( [str(pyexec), "-c", "import M2Crypto"], env=env, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + capture_output=True, check=False, ) assert p.returncode == 0, (p.stdout, p.stderr) @@ -988,24 +971,18 @@ def test_pip_install_m2crypto_relenv_ssl( @pytest.mark.skip_on_windows def test_shebangs(pipexec, build, minor_version): def validate_shebang(path): - with open(path, "r") as fp: + with open(path) as fp: return fp.read(9) == "#!/bin/sh" path = build / "bin" / "pip3" assert path.exists() assert validate_shebang(path) - if "3.13" not in minor_version: + if minor_version not in ["3.13", "3.14"]: path = build / "lib" / f"python{minor_version}" / "cgi.py" assert path.exists() assert validate_shebang(path) if sys.platform == "linux": - path = ( - build - / "lib" - / f"python{minor_version}" - / f"config-{minor_version}-{get_triplet()}" - / "python-config.py" - ) + path = build / "lib" / f"python{minor_version}" / f"config-{minor_version}-{get_triplet()}" / "python-config.py" assert path.exists() assert validate_shebang(path) @@ -1025,9 +1002,7 @@ def test_moving_pip_installed_c_extentions(pipexec, pyexec, build, minor_version assert p.returncode == 0, "Failed to pip install cffi" b2 = build.parent / "test2" build.rename(b2) - libname = ( - f"_cffi_backend.cpython-{minor_version.replace('.', '')}-x86_64-linux-gnu.so" - ) + libname = f"_cffi_backend.cpython-{minor_version.replace('.', '')}-x86_64-linux-gnu.so" p = subprocess.run( ["ldd", b2 / "lib" / f"python{minor_version}" / "site-packages" / libname], stdout=subprocess.PIPE, @@ -1043,9 +1018,7 @@ def test_moving_pip_installed_c_extentions(pipexec, pyexec, build, minor_version @pytest.mark.skip_unless_on_linux @pytest.mark.parametrize("cryptography_version", ["40.0.1", "39.0.2"]) -def test_cryptography_rpath( - pyexec, pipexec, build, minor_version, cryptography_version -): +def test_cryptography_rpath(pyexec, pipexec, build, minor_version, cryptography_version): _install_ppbt(pyexec) def find_library(path, search): @@ -1069,22 +1042,12 @@ def find_library(path, search): env=env, ) assert p.returncode != 1, "Failed to pip install cryptography" - bindings = ( - build - / "lib" - / f"python{minor_version}" - / "site-packages" - / "cryptography" - / "hazmat" - / "bindings" - ) + bindings = build / "lib" / f"python{minor_version}" / "site-packages" / "cryptography" / "hazmat" / "bindings" if cryptography_version == "39.0.2": osslinked = find_library(bindings, "_openssl") else: osslinked = "_rust.abi3.so" - p = subprocess.run( - ["ldd", bindings / osslinked], stdout=subprocess.PIPE, check=True - ) + p = subprocess.run(["ldd", bindings / osslinked], stdout=subprocess.PIPE, check=True) found = 0 for line in p.stdout.splitlines(): line = line.decode() @@ -1127,7 +1090,7 @@ def test_cryptography_rpath_darwin(pipexec, build, minor_version, cryptography_v env["RELENV_BUILDENV"] = "yes" env["OPENSSL_DIR"] = f"{build}" - if minor_version == "3.13": + if minor_version in ["3.13", "3.14"]: env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" p = subprocess.run( @@ -1195,9 +1158,7 @@ def test_install_pycurl(pipexec, pyexec, build): env["RELENV_BUILDENV"] = "yes" # Install pycurl - subprocess.run( - [str(pipexec), "install", "pycurl", "--no-cache-dir"], env=env, check=True - ) + subprocess.run([str(pipexec), "install", "pycurl", "--no-cache-dir"], env=env, check=True) # Move the relenv environment, if something goes wrong this will break the linker. b2 = build.parent / "test2" @@ -1300,11 +1261,7 @@ def test_install_libgit2(pipexec, build, minor_version, build_dir, versions): print(versions) with open("buildscript.sh", "w") as fp: - fp.write( - buildscript.format( - libssh2=versions["libssh2"], libgit2=versions["libgit2"], build=build - ) - ) + fp.write(buildscript.format(libssh2=versions["libssh2"], libgit2=versions["libgit2"], build=build)) subprocess.run(["/usr/bin/bash", "buildscript.sh"], check=True) @@ -1410,12 +1367,7 @@ def test_install_with_target_shebang(pipexec, build, minor_version): if stripped.startswith('"exec"'): exec_line = stripped break - assert ( - exec_line - == '"exec" "$(dirname "$(readlink -f "$0")")/../../bin/python{}" "$0" "$@"'.format( - minor_version - ) - ) + assert exec_line == f'"exec" "$(dirname "$(readlink -f "$0")")/../../bin/python{minor_version}" "$0" "$@"' @pytest.mark.skip_unless_on_linux @@ -1478,7 +1430,7 @@ def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version env = os.environ.copy() env["RELENV_DEBUG"] = "yes" extras = build / "extras" - if "3.13" not in build_version: + if build_version[:4] not in ["3.13", "3.14"]: subprocess.run( [str(pipexec), "install", "cffi==1.14.6"], check=True, @@ -1505,7 +1457,10 @@ def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version def test_install_with_target_no_ignore_installed(pipexec, pyexec, build, build_version): - if build_version.startswith("3.13"): + if build_version.startswith("3.14"): + cffi = "cffi==2.0.0" + pygit2 = "pygit2==1.19.2" + elif build_version.startswith("3.13"): cffi = "cffi==1.17.1" pygit2 = "pygit2==1.16.0" elif build_version.startswith("3.12"): @@ -1538,7 +1493,10 @@ def test_install_with_target_no_ignore_installed(pipexec, pyexec, build, build_v def test_install_with_target_ignore_installed(pipexec, pyexec, build, build_version): - if build_version.startswith("3.13"): + if build_version.startswith("3.14"): + cffi = "cffi==2.0.0" + pygit2 = "pygit2==1.19.2" + elif build_version.startswith("3.13"): cffi = "cffi==1.17.1" pygit2 = "pygit2==1.16.0" elif build_version.startswith("3.12"): @@ -1652,7 +1610,7 @@ def test_legacy_hashlib(pipexec, pyexec, build): stdout=subprocess.PIPE, env=env, ) - with open(env["OPENSSL_CONF"], "r") as fp: + with open(env["OPENSSL_CONF"]) as fp: print(fp.read()) assert b"md4" in proc.stdout @@ -1783,8 +1741,8 @@ def test_install_with_target_namespaces(pipexec, build, minor_version, build_ver @pytest.mark.skip_unless_on_linux def test_debugpy(pipexec, build, arch, minor_version): - if "3.13" in minor_version: - pytest.xfail("Failes on python 3.13.0") + if minor_version in ["3.13", "3.14"]: + pytest.xfail("Failes on python 3.13+") if arch == "arm64": pytest.xfail("Failes on arm64") @@ -1922,9 +1880,7 @@ def test_install_editable_package(pipexec, pyexec, build, minor_version, tmp_pat @pytest.mark.skip_unless_on_linux -def test_install_editable_package_in_extras( - pipexec, pyexec, build, minor_version, tmp_path -): +def test_install_editable_package_in_extras(pipexec, pyexec, build, minor_version, tmp_path): _install_ppbt(pyexec) sitepkgs = pathlib.Path(build) / "lib" / f"python{minor_version}" / "site-packages" @@ -1946,9 +1902,7 @@ def test_install_editable_package_in_extras( env=env, ) assert p.returncode == 0 - p = subprocess.run( - [str(pipexec), "install", f"--target={extras}", "-e", "saltext-zabbix"], env=env - ) + p = subprocess.run([str(pipexec), "install", f"--target={extras}", "-e", "saltext-zabbix"], env=env) assert p.returncode == 0 p = subprocess.run([str(pyexec), "-c", "import saltext.zabbix"], env=env) assert p.returncode == 0 @@ -2024,9 +1978,7 @@ def test_no_openssl_binary(rockycontainer, pipexec, pyexec, build): key=lambda p: len(p.name), ) if not bz2_sources: - pytest.fail( - "libbz2.so not found in relenv build; cryptography build cannot proceed" - ) + pytest.fail("libbz2.so not found in relenv build; cryptography build cannot proceed") for bz2_source in bz2_sources: target = sysroot_lib / bz2_source.name if target.exists() or target.is_symlink(): @@ -2072,9 +2024,7 @@ def test_darwin_python_linking(pipexec, pyexec, build, minor_version): def test_import_ssl_module(pyexec): - proc = subprocess.run( - [pyexec, "-c", "import ssl"], capture_output=True, check=False - ) + proc = subprocess.run([pyexec, "-c", "import ssl"], capture_output=True, check=False) assert proc.returncode == 0 assert proc.stdout.decode() == "" assert proc.stderr.decode() == "" @@ -2140,9 +2090,7 @@ def test_expat_version(pyexec): actual_version_str = proc.stdout.decode().strip() # Format is "expat_X_Y_Z", extract version - assert actual_version_str.startswith( - "expat_" - ), f"Unexpected EXPAT_VERSION format: {actual_version_str}" + assert actual_version_str.startswith("expat_"), f"Unexpected EXPAT_VERSION format: {actual_version_str}" # Convert "expat_2_7_3" -> "2.7.3" actual_version = actual_version_str.replace("expat_", "").replace("_", ".") @@ -2197,8 +2145,7 @@ def test_sqlite_version(pyexec): expected_version = ".".join(expected_version.split(".")[:3]) assert actual_version == expected_version, ( - f"SQLite version mismatch on {platform}: expected {expected_version}, " - f"found {actual_version}" + f"SQLite version mismatch on {platform}: expected {expected_version}, found {actual_version}" ) @@ -2228,9 +2175,7 @@ def test_openssl_version(pyexec): # Get expected version from python-versions.json openssl_info = get_dependency_version("openssl", platform) if not openssl_info: - pytest.skip( - f"No openssl version defined in python-versions.json for {platform}" - ) + pytest.skip(f"No openssl version defined in python-versions.json for {platform}") expected_version = openssl_info["version"] @@ -2325,8 +2270,5 @@ def test_xz_lzma_functionality(pyexec): check=False, ) - assert proc.returncode == 0, ( - f"LZMA functionality test failed (exit code {proc.returncode}): " - f"{proc.stderr.decode()}" - ) + assert proc.returncode == 0, f"LZMA functionality test failed (exit code {proc.returncode}): {proc.stderr.decode()}" assert proc.stdout.decode().strip() == "OK"