diff --git a/.claude/skills/upgrade-python/SKILL.md b/.claude/skills/upgrade-python/SKILL.md new file mode 100644 index 00000000..39fef1d2 --- /dev/null +++ b/.claude/skills/upgrade-python/SKILL.md @@ -0,0 +1,237 @@ +--- +name: upgrade-python +description: workflow to rebuild all packages against a newer python version +--- + +Adds support for a new Python version by building all packages, identifying failures, and marking them with `python_versions` restrictions. + +## Usage + +``` +/upgrade-python +``` + +Example: `/upgrade-python 3.14.3` + +The argument is the full Python version (e.g., `3.14.3`). Derive the major.minor (`3.14`) and cpython tag (`cp314`) from it. + +## Step 1: Modify files for single-version build mode + +Edit these files to build ONLY the new Python version (this speeds up CI dramatically): + +### `build.py` line 35 +Change `PYTHONS` to only the new version: +```python +PYTHONS = ((3, 14),) # temporarily building only new version +``` + +### `validate.py` line 19 +Same change: +```python +PYTHONS = ((3, 14),) # temporarily building only new version +``` + +### `docker/install-pythons` line 9 +Change `VERSIONS` to only the new full version: +```python +VERSIONS = ("3.14.3",) # temporarily building only new version +``` + +### `docker/Dockerfile` + +**Line 39** — keep only the new cpython PATH entry (the base image already provides python3.11 on PATH): +```dockerfile + PATH=/venv/bin:/opt/python/cp314-cp314/bin:$PATH \ +``` + +**Line 51** — use `python3.11` directly (provided by the base image) instead of the installed cpython path: +```dockerfile + && python3.11 -m venv /venv \ +``` + +### `.github/workflows/build.yml` + +The production workflow uses pre-built `:latest` images. During a Python upgrade, the Dockerfile has changes that aren't in the production image yet, so you must temporarily add an `image` job that builds and pushes the modified image, and wire the `linux` job to use it. + +**Add a temporary `image` job** before the `linux` job: +```yaml + image: + strategy: + matrix: + include: + - {arch: amd64, os: ubuntu-latest} + - {arch: arm64, os: ubuntu-24.04-arm} + runs-on: ${{ matrix.os }} + permissions: + packages: write + steps: + - uses: actions/checkout@v3 + - run: docker login --username '${{ github.actor }}' --password-stdin ghcr.io <<< '${{ secrets.GITHUB_TOKEN }}' + - run: | + docker buildx build \ + --cache-from ghcr.io/getsentry/pypi-manylinux-${{ matrix.arch }}-ci:latest \ + --cache-to type=inline \ + --platform linux/${{ matrix.arch }} \ + --tag ghcr.io/getsentry/pypi-manylinux-${{ matrix.arch }}-ci:${{ github.sha }} \ + ${{ github.ref == 'refs/heads/main' && format('--tag ghcr.io/getsentry/pypi-manylinux-{0}-ci:latest', matrix.arch) || '' }} \ + --push \ + docker +``` + +**Modify the `linux` job** to depend on `image` and use the SHA-tagged image: +```yaml + linux: + needs: [image] + ... + container: ghcr.io/getsentry/pypi-manylinux-${{ matrix.arch }}-ci:${{ github.sha }} +``` + +**macOS PATH entries** — keep only the new cpython PATH entry: +```yaml + - run: | + echo "$PWD/pythons/cp314-cp314/bin" >> "$GITHUB_PATH" + echo "$PWD/venv/bin" >> "$GITHUB_PATH" +``` + +**Add `--upgrade-python` flag** to both linux and macos build commands: +```yaml + - run: python3 -um build --pypi-url https://pypi.devinfra.sentry.io --upgrade-python +``` + +## Step 2: Commit and push + +Commit all changes with a message like "build: single-version mode for Python 3.14 upgrade" and push. + +## Step 3: Wait for CI, then download and parse logs + +Wait for CI to complete (it will likely fail — that's expected). +Use `gh run watch` to avoid rate limits if polling. + +Download logs from each build job using the GitHub CLI: + +```bash +# Get the workflow run +gh run list --branch --limit 1 + +# Get job IDs from the run +gh run view --json jobs --jq '.jobs[] | {name: .name, id: .databaseId, conclusion: .conclusion}' + +# Download logs for each job +gh api repos/getsentry/pypi/actions/jobs//logs > job-.log +``` + +Parse the logs to identify: +- **Succeeded packages**: lines matching `=== ==@` that are NOT followed by `!!! FAILED:` or `!!! SKIPPED` +- **Failed packages**: lines matching `!!! FAILED: ==: ` +- **Skipped packages**: lines matching `!!! SKIPPED (newer version already failed): ==` — these are older versions that were auto-skipped because a newer version of the same package already failed + +A package is considered failed if it failed or was skipped on ANY platform (linux-amd64, linux-arm64, macos). + +## Step 4: Update `packages.ini` + +In one pass: + +1. **Remove** all packages that **succeeded on all platforms** — delete their entire section (`[name==version]` header + all config lines). The `format-packages-ini` pre-commit hook uses `configparser` which strips `#` comments, so commenting out doesn't work. Just delete succeeded sections outright. They don't need to rebuild. + +2. **Add `python_versions = :packages.ini']).decode() +old = configparser.RawConfigParser(strict=False) +old.read_string(old_content) + +with open('packages.ini') as f: + cur = configparser.RawConfigParser(strict=False) + cur.read_string(f.read()) + +missing = set(old.sections()) - set(cur.sections()) +# Exclude any packages intentionally not restored + +# Append missing sections to packages.ini +with open('packages.ini', 'a') as f: + for section in sorted(missing): + f.write(f'\n[{section}]\n') + for k, v in old[section].items(): + v = v.strip() + if '\n' in v: + f.write(f'{k} =\n') + for part in v.split('\n'): + if part.strip(): + f.write(f' {part.strip()}\n') + else: + f.write(f'{k} = {v}\n') +``` + +Then run `python3 -m format_ini packages.ini` to sort and format. The formatter handles ordering automatically. + +Commit and push. Verify CI passes — all packages should either download pre-built wheels or build successfully. + +## Step 8: Revert single-version mode + +After all packages are restored and CI passes, revert the single-version mode changes from step 1 to build all Python versions again: + +1. **`build.py`**: Change `PYTHONS` back to all versions (e.g., `((3, 11), (3, 12), (3, 13), (3, 14))`) +2. **`validate.py`**: Same change to `PYTHONS` +3. **`docker/install-pythons`**: Change `VERSIONS` back to all versions (e.g., `("3.11.14", "3.12.12", "3.13.12", "3.14.3")`) +4. **`docker/Dockerfile`**: Restore all cpython paths in `PATH` env var +5. **`.github/workflows/build.yml`**: + - Remove the temporary `image` job entirely + - Remove `needs: [image]` from the `linux` job + - Change the `linux` container back to `:latest` tag (from `:${{ github.sha }}`) + - Remove `--upgrade-python` flag from both linux and macos build commands + - Restore all cpython PATH entries in the macos job + +Commit with a message like "revert single-version mode, build all python versions" and push. Verify CI passes with all Python versions building. + +## Important notes + +- The `--upgrade-python` flag in `build.py` enables continue-on-failure mode with a 10-minute timeout per package. Without it, builds fail on first error (normal behavior). +- The `=== name==version@python`, `!!! FAILED: name==version: error`, and `!!! SKIPPED (newer version already failed): name==version` log lines are the markers used to parse results. +- In `--upgrade-python` mode, packages are sorted newest-version-first within each name. If the newest version fails, all older versions are automatically skipped. +- Do NOT try to comment out sections with `#` — the `format-packages-ini` pre-commit hook uses Python's `configparser` which strips all `#` comments. Instead, delete succeeded sections entirely. +- The `format-packages-ini` hook also sorts and reformats `packages.ini`, so ordering is handled automatically. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ae452b54..b27836c9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,7 +2,7 @@ name: build on: pull_request: push: - branches: [main, test-me-*] + branches: [main] concurrency: # serialize runs on the default branch @@ -11,12 +11,13 @@ concurrency: jobs: linux: strategy: + fail-fast: false matrix: include: - {arch: amd64, os: ubuntu-latest} - {arch: arm64, os: ubuntu-24.04-arm} runs-on: ${{ matrix.os }} - container: ghcr.io/getsentry/pypi-manylinux-${{ matrix.arch }}-ci + container: ghcr.io/getsentry/pypi-manylinux-${{ matrix.arch }}-ci:${{ github.head_ref == 'feat/python314-support' && '769374905268e37a20b467d487926b78d56eb96c' || 'latest' }} steps: - uses: actions/checkout@v3 - run: python3 -um build --pypi-url https://pypi.devinfra.sentry.io @@ -28,19 +29,17 @@ jobs: macos: strategy: matrix: - runs-on: [macos-14] + runs-on: [macos-15] runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 - - run: | - # work around https://github.com/indygreg/python-build-standalone/issues/208 - HOMEBREW_NO_AUTO_UPDATE=1 brew install gnu-tar - echo "$(brew --prefix gnu-tar)/libexec/gnubin" >> "$GITHUB_PATH" + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6 - run: python3 -u docker/install-pythons --dest pythons - run: | echo "$PWD/pythons/cp311-cp311/bin" >> "$GITHUB_PATH" echo "$PWD/pythons/cp312-cp312/bin" >> "$GITHUB_PATH" echo "$PWD/pythons/cp313-cp313/bin" >> "$GITHUB_PATH" + echo "$PWD/pythons/cp314-cp314/bin" >> "$GITHUB_PATH" echo "$PWD/venv/bin" >> "$GITHUB_PATH" - run: python3 -um venv venv && pip install -r docker/requirements.txt - run: python3 -um build --pypi-url https://pypi.devinfra.sentry.io diff --git a/PYTHON-3.14-UPGRADE.md b/PYTHON-3.14-UPGRADE.md new file mode 100644 index 00000000..1e786a8b --- /dev/null +++ b/PYTHON-3.14-UPGRADE.md @@ -0,0 +1,147 @@ +# Python 3.14 Upgrade — Build Failures + +Date: 2026-02-24 +Python version: 3.14.3 +CI run: https://github.com/getsentry/pypi/actions/runs/22364364543 + +## Summary by category + +| Category | Count | Packages | +|----------|-------|----------| +| PyO3 too old for 3.14 | 7 | jiter, orjson, pydantic-core, sentry-streams, tiktoken, vroomrs, rpds-py | +| No sdist on PyPI | 3 | backports-zstd, lief, sentry-forked-jsonnet | +| CPython 3.14 C API change | 1 | pyuwsgi | +| Missing build dependency | 1 | xmlsec | +| Build timeout | 1 | grpcio (1.73.1 on linux-amd64 only) | +| Platform-specific issues | 3 | grpcio (1.67.0 macos), pillow (linux-amd64), p4python (macos) | +| Validation failure (import) | 1 | uvloop | + +Note: The last two categories (timeout + platform-specific) are NOT Python 3.14 incompatibilities. +These packages built successfully on at least one platform and were commented out (not restricted). + +## Detailed failures + +### PyO3 too old for Python 3.14 + +These packages use PyO3 versions that cap support at Python 3.13. They need upstream +releases with PyO3 0.25+ (which adds 3.14 support). + +#### jiter==0.9.0 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.24.0 — `the maximum Python version is 3.13, found 3.14` +- **Note**: jiter==0.10.0 builds fine on 3.14 + +#### orjson==3.10.10 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.23.0-dev — `the maximum Python version is 3.13, found 3.14` + +#### pydantic-core==2.33.2 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.24.1 — `the maximum Python version is 3.13, found 3.14` +- **Skipped older versions**: 2.24.2, 2.23.4 + +#### sentry-streams==0.0.35 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.24.0 — `the maximum Python version is 3.13, found 3.14` +- **Skipped older versions**: 0.0.17 through 0.0.34 +- **Note**: Already had `python_versions = <3.14` from previous pass + +#### tiktoken==0.8.0 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.22.6 — `the maximum Python version is 3.13, found 3.14` + +#### vroomrs==0.1.19 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.24.1 — `the maximum Python version is 3.13, found 3.14` +- **Skipped older versions**: 0.1.2 through 0.1.18 + +#### rpds-py==0.20.0 +- **Platforms**: linux-amd64, macos +- **Category**: PyO3 too old +- **Error**: pyo3 0.22.2 — `the maximum Python version is 3.13, found 3.14` +- **Note**: Already had `python_versions = <3.14` from previous pass + +### No sdist on PyPI + +These packages only publish binary wheels (no source distribution), and no cp314 wheel exists yet. + +#### backports-zstd==1.3.0 +- **Platforms**: all +- **Category**: No sdist +- **Error**: `pip download` found no matching distribution — wheel-only package with no cp314 wheel + +#### lief==0.16.6 +- **Platforms**: linux-amd64, macos +- **Category**: No sdist +- **Error**: `pip download` found no matching distribution — no sdist for 0.16.6 + +#### sentry-forked-jsonnet==0.20.0.post4 +- **Platforms**: linux-amd64, macos +- **Category**: No sdist +- **Error**: `pip download` found no matching distribution +- **Note**: Already had `python_versions = <3.14` from previous pass + +### CPython 3.14 C API changes + +#### pyuwsgi==2.0.29 +- **Platforms**: linux-amd64, macos +- **Category**: CPython 3.14 API change +- **Error**: `c_recursion_remaining` removed from `PyThreadState` in CPython 3.14 +- **Skipped older versions**: 2.0.28.post1, 2.0.27.post1 +- **Note**: pyuwsgi==2.0.30 builds fine on 3.14 + +### Missing build dependencies + +#### xmlsec==1.3.14 +- **Platforms**: linux-amd64, macos +- **Category**: Missing build dep +- **Error**: Build dependency `lxml` has no cp314 wheel on the internal PyPI index +- **Note**: lxml==5.3.0 itself builds fine on 3.14, but xmlsec needs it at build time from the index +- **Status**: Unblocked once lxml==5.3.0 cp314 wheel is deployed to internal index. After merging (which builds and deploys lxml's cp314 wheel), remove `python_versions = <3.14` from xmlsec==1.3.14 in a follow-up commit. + +### Platform-specific issues (NOT Python 3.14 incompatibilities) + +These packages were NOT marked with `python_versions = <3.14` because they can build +on Python 3.14 — they just had issues on specific platforms. + +#### grpcio==1.73.1 +- **Platforms**: linux-amd64 only (succeeded on linux-arm64 and macos) +- **Category**: Build timeout +- **Error**: Build timed out after 600 seconds on linux-amd64 (compiled successfully on macos in ~8.5 min) +- **Note**: grpcio==1.75.1 succeeds everywhere + +#### grpcio==1.67.0 +- **Platforms**: macos only (succeeded on linux-arm64) +- **Category**: C compilation error +- **Error**: Bundled zlib `fdopen` macro conflicts with macOS `_stdio.h` + +#### pillow==11.2.1 +- **Platforms**: linux-amd64 only (succeeded on macos) +- **Category**: Missing system library +- **Error**: Missing `libjpeg` headers in the Docker build container +- **Note**: pillow==11.3.0 succeeds everywhere +- **Fix**: Added `libjpeg-dev` to Dockerfile and re-added pillow versions to packages.ini + +#### p4python==2025.1.2767466 +- **Platforms**: macos only (succeeded on linux-amd64) +- **Category**: Missing build configuration +- **Error**: `setup.py` requires `--ssl` parameter and P4API on macOS; multiple env issues +- **Status**: Marked `python_versions = <3.14` — too many macOS-specific build issues (SSL detection, P4API download) to fix in this pass + +### Validation failures + +These packages built successfully but failed during import validation on Python 3.14. + +#### uvloop==0.21.0 +- **Platforms**: all +- **Category**: CPython 3.14 API removal +- **Error**: `ImportError: cannot import name 'BaseDefaultEventLoopPolicy' from 'asyncio.events'` +- **Note**: `asyncio.events.BaseDefaultEventLoopPolicy` was removed in CPython 3.14 + +This needs to be figured out for granian to pass validation as well. diff --git a/build.py b/build.py index 16f1189d..529f408b 100644 --- a/build.py +++ b/build.py @@ -32,7 +32,7 @@ from packaging.utils import parse_wheel_filename from packaging.version import Version -PYTHONS = ((3, 11), (3, 12), (3, 13)) +PYTHONS = ((3, 11), (3, 12), (3, 13), (3, 14)) BINARY_EXTS = frozenset( (".c", ".cc", ".cpp", ".cxx", ".pxd", ".pxi", ".pyx", ".go", ".rs") @@ -257,18 +257,22 @@ def _apt_install(packages: tuple[str, ...]) -> Generator[None, None, None]: installed_before = _linux_installed_packages() - subprocess.check_call( + ret = subprocess.run( ( "apt-get", + "-o", + "Acquire::Retries=3", "install", "-qqy", "--no-install-recommends", *packages, ), env={**os.environ, "DEBIAN_FRONTEND": "noninteractive"}, - stderr=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, + capture_output=True, ) + if ret.returncode: + sys.stderr.buffer.write(ret.stderr) + ret.check_returncode() try: yield @@ -553,7 +557,13 @@ def _produced_binary(wheel: str) -> bool: return False -def _build(package: Package, python: Python, dest: str, index_url: str) -> str: +def _build( + package: Package, + python: Python, + dest: str, + index_url: str, + timeout: int | None = None, +) -> str: with tempfile.TemporaryDirectory() as tmpdir: pip = (python.exe, "-mpip") @@ -569,7 +579,8 @@ def _build(package: Package, python: Python, dest: str, index_url: str) -> str: "--no-deps", f"--no-binary={package.name}", f"{package.name}=={package.version}", - ) + ), + timeout=timeout, ) (sdist,) = os.listdir(sdist_dir) sdist = os.path.join(sdist_dir, sdist) @@ -589,6 +600,7 @@ def _build(package: Package, python: Python, dest: str, index_url: str) -> str: # disable bulky "universal2" building "ARCHFLAGS": "", }, + timeout=timeout, ) (filename,) = os.listdir(build_dir) filename_full = os.path.join(build_dir, filename) @@ -616,6 +628,7 @@ def main() -> int: parser.add_argument("--pypi-url", required=True) parser.add_argument("--packages-ini", default="packages.ini") parser.add_argument("--dest", default="dist") + parser.add_argument("--upgrade-python", action="store_true") args = parser.parse_args() cfg = configparser.RawConfigParser() @@ -633,27 +646,76 @@ def main() -> int: internal_wheels = _internal_wheels(args.pypi_url) built: dict[str, list[tuple[Version, frozenset[Tag]]]] = {} + timeout = 600 if args.upgrade_python else None + failures: list[str] = [] + failed_names: set[str] = set() + all_packages = [Package.make(k, cfg[k]) for k in cfg.sections()] + + if args.upgrade_python: + # sort newest versions first within each package name so we can + # bail out early: if the newest version fails, older ones will too + by_name: dict[str, list[Package]] = {} + for p in all_packages: + by_name.setdefault(p.name, []).append(p) + all_packages = [ + p + for pkgs in by_name.values() + for p in sorted(pkgs, key=lambda p: p.version, reverse=True) + ] + for package, python in itertools.product(all_packages, pythons): if package.satisfied_by(internal_wheels, python.tags): continue elif python.version_string not in package.python_versions: continue + pkg_id = f"{package.name}=={package.version}" + + if args.upgrade_python and package.name in failed_names: + print(f"=== {pkg_id}@{python.version}") + print(f"!!! SKIPPED (newer version already failed): {pkg_id}") + failures.append(pkg_id) + continue + print(f"=== {package.name}=={package.version}@{python.version}") if package.satisfied_by(built, python.tags): print("-> just built!") else: - print("-> building...") - downloaded_wheel = _download(package, python, args.dest) - if downloaded_wheel is not None: - _add_wheel(built, downloaded_wheel) - print(f"-> downloaded! {downloaded_wheel}") - else: - built_wheel = _build(package, python, args.dest, index_url) - _add_wheel(built, built_wheel) - print(f"-> built! {built_wheel}") + try: + print("-> building...") + downloaded_wheel = _download(package, python, args.dest) + if downloaded_wheel is not None: + _add_wheel(built, downloaded_wheel) + print(f"-> downloaded! {downloaded_wheel}") + else: + built_wheel = _build( + package, + python, + args.dest, + index_url, + timeout=timeout, + ) + _add_wheel(built, built_wheel) + print(f"-> built! {built_wheel}") + except ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + SystemExit, + ) as e: + if args.upgrade_python: + print(f"!!! FAILED: {pkg_id}: {e}") + failures.append(pkg_id) + failed_names.add(package.name) + else: + raise + + if failures: + print(f"\nFAILED PACKAGES ({len(failures)}):") + for f in failures: + print(f" - {f}") + return 1 return 0 diff --git a/docker/Dockerfile b/docker/Dockerfile index b400598c..3da5fb40 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,27 +2,30 @@ FROM python:3.11.4-slim-bullseye RUN : \ && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install \ + && DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install \ -y --no-install-recommends \ automake \ ca-certificates \ + clang \ curl \ dumb-init \ - g++ \ - gcc \ git \ libbz2-dev \ libdb-dev \ libexpat1-dev \ libffi-dev \ libgdbm-dev \ + libjpeg-dev \ libltdl-dev \ liblzma-dev \ libncursesw5-dev \ libreadline-dev \ libsqlite3-dev \ + libcurl4-openssl-dev \ + libsasl2-dev \ libssl-dev \ libtool \ + libzstd-dev \ make \ pkg-config \ uuid-dev \ @@ -31,20 +34,6 @@ RUN : \ zstd \ && rm -rf /var/lib/apt/lists/* -# https://github.com/pypa/auditwheel/issues/229 -# libc's libcrypt1 uses GLIBC_PRIVATE so we must build our own -RUN : \ - && git clone https://github.com/pypa/manylinux /tmp/manylinux \ - && cd /tmp/manylinux \ - && git checkout 075550587bb428c01ed2dd31f9b6e0b089d62802 \ - && \ - AUDITWHEEL_POLICY= \ - LIBXCRYPT_VERSION=4.4.28 \ - LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 \ - LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive \ - /tmp/manylinux/docker/build_scripts/install-libxcrypt.sh \ - && rm -rf /tmp/manylinux - ENV \ BUILD_IN_CONTAINER=1 \ PATH=/venv/bin:/opt/python/cp311-cp311/bin:/opt/python/cp312-cp312/bin:/opt/python/cp313-cp313/bin:/opt/python/cp314-cp314/bin:$PATH \ @@ -52,12 +41,18 @@ ENV \ PIP_NO_CACHE_DIR=1 \ PIP_NO_WARN_ABOUT_ROOT_USER=0 +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + COPY install-pythons /tmp/install-pythons -RUN /tmp/install-pythons +RUN /tmp/install-pythons \ + # the base image installs libcrypt.so.1 to /usr/local/lib/ which the + # linker finds before the system copy. this breaks perl (needed by dpkg) + # because the base image's copy lacks the XCRYPT_2.0 symbol. + && rm -f /usr/local/lib/libcrypt.so* COPY requirements.txt /tmp/requirements.txt RUN : \ - && /opt/python/cp311-cp311/bin/python3.11 -m venv /venv \ + && python3.11 -m venv /venv \ && /venv/bin/pip install --no-cache-dir -r /tmp/requirements.txt ENTRYPOINT ["dumb-init", "--"] diff --git a/docker/install-pythons b/docker/install-pythons index 031cc50a..ed745b25 100755 --- a/docker/install-pythons +++ b/docker/install-pythons @@ -2,61 +2,11 @@ from __future__ import annotations import argparse -import hashlib -import os.path -import platform -import re -import secrets +import glob +import os import subprocess -import sys -import tempfile -RELEASE = ( - "https://github.com/indygreg/python-build-standalone/releases/download/20260211/" -) -# curl --silent --location https://github.com/indygreg/python-build-standalone/releases/download/20260211/SHA256SUMS | grep -E '(aarch64-apple-darwin-pgo\+lto-full|x86_64-apple-darwin-pgo\+lto-full|aarch64-unknown-linux-gnu-pgo\+lto-full|x86_64-unknown-linux-gnu-pgo\+lto-full)' | grep -Ev '(cpython-3\.(8|9|10|15)|freethreaded)' -CHECKSUMS = """\ -ceda72c76ecfd4294ae3fdf275202a3cfe912cf1dc7076c9526171aaaedbd3e3 cpython-3.11.14+20260211-aarch64-apple-darwin-pgo+lto-full.tar.zst -355c4a10233a2e3ac1e511e7cf052116404e197bc70fcc22b67aba635e209808 cpython-3.11.14+20260211-aarch64-unknown-linux-gnu-pgo+lto-full.tar.zst -898995bc172df26f5e0ce9bac316254756094466b9d472234e62e4045e1ddbf6 cpython-3.11.14+20260211-x86_64-apple-darwin-pgo+lto-full.tar.zst -9dd3129d78fc42a63690b09f9f48b27d35b0fbe7b580fd1cd85bb554c82671b6 cpython-3.11.14+20260211-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst -bf70a8ba4d44eb243af9dc3485656e0ce3757588eefe27e1801b36ff9773805a cpython-3.12.12+20260211-aarch64-apple-darwin-pgo+lto-full.tar.zst -48cccc8970f32586b60125199c955da870c5b9c52c05afb2bce28714eeb17cc6 cpython-3.12.12+20260211-aarch64-unknown-linux-gnu-pgo+lto-full.tar.zst -14fe4f2213f9b89d5649b2c50636be20393ec0092960d1acd11f7c84a4e1b2e9 cpython-3.12.12+20260211-x86_64-apple-darwin-pgo+lto-full.tar.zst -75428635145d4eb8de86cff5d00a823009a21fe8c173c7899959d0f41f73ad4a cpython-3.12.12+20260211-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst -3baef69715ffc554a7f173e9419cfb75ddf25b7cae91ab141032843d53fa34c4 cpython-3.13.12+20260211-aarch64-apple-darwin-pgo+lto-full.tar.zst -0ad848cab9031fc80c64442698f6eff112d81d45eaf53f49ece6ecbfc97f6ea6 cpython-3.13.12+20260211-aarch64-unknown-linux-gnu-pgo+lto-full.tar.zst -8ad36a0b44b03f2c236d05135600d626ae73245eae0361a17ddabb9e7163e50b cpython-3.13.12+20260211-x86_64-apple-darwin-pgo+lto-full.tar.zst -2483028342db1e31a8a4004a859a856fade2563bae97f18812a2d27a123773e6 cpython-3.13.12+20260211-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst -d016c5a16c6a246f56cf2fae2a4150a339311a30787347ff1c1e063295c82401 cpython-3.14.3+20260211-aarch64-apple-darwin-pgo+lto-full.tar.zst -13a08dca6f29df3701f1846184db78499d23014f6d5a70fa6c2c1f29baee350a cpython-3.14.3+20260211-aarch64-unknown-linux-gnu-pgo+lto-full.tar.zst -3e55c3d0914e7e4f2e7a135c80077a0ac635de9dcfa0c08f2544fc2165e264a4 cpython-3.14.3+20260211-x86_64-apple-darwin-pgo+lto-full.tar.zst -96c6684fffd6da9d219400b2e3c020d9bc2c838cbb4ac202e2dd652dda3d1914 cpython-3.14.3+20260211-x86_64-unknown-linux-gnu-pgo+lto-full.tar.zst -""" VERSIONS = ("3.11.14", "3.12.12", "3.13.12", "3.14.3") -ARCH_MAP = {"arm64": "aarch64"} -ARCH = ARCH_MAP.get(platform.machine(), platform.machine()) - -CLANG_PP = re.compile(r"\bclang\+\+") -CLANG = re.compile(r"\bclang\b") - - -def _must_sub(reg: re.Pattern[str], new: str, s: str) -> str: - after = reg.sub(new, s) - if after == s: - raise AssertionError(f"expected replacement by {reg} => {new}!") - return after - - -def _checksum_url(version: str) -> tuple[str, str]: - for line in CHECKSUMS.splitlines(): - sha256, filename = line.split() - _, f_version_release, arch, _, plat, *_ = filename.split("-") - f_version, _ = f_version_release.split("+") - if version == f_version and sys.platform == plat and ARCH == arch: - return (sha256, f"{RELEASE}/{filename}") - else: - raise NotImplementedError(version, sys.platform, platform.machine()) def main() -> int: @@ -66,57 +16,49 @@ def main() -> int: os.makedirs(args.dest, exist_ok=True) - for version in VERSIONS: - with tempfile.TemporaryDirectory() as tmpdir: - expected, url = _checksum_url(version) - - major, minor, *_ = version.split(".") - dest = os.path.join(args.dest, f"cp{major}{minor}-cp{major}{minor}") - tgz_dest = os.path.join(tmpdir, "python.tgz") + subprocess.check_call( + ("uv", "python", "install", *VERSIONS, "--install-dir", args.dest), + ) - curl_cmd = ("curl", "--silent", "--location", "--output", tgz_dest, url) - subprocess.check_call(curl_cmd) - - with open(tgz_dest, "rb") as f: - sha256 = hashlib.sha256(f.read()).hexdigest() - if not secrets.compare_digest(sha256, expected): - raise AssertionError(f"checksum mismatch {sha256=} {expected=}") - - os.makedirs(dest, exist_ok=True) - tar_cmd = ( - "tar", - "-C", - dest, - "--strip-components=2", - "-xf", - tgz_dest, - "python/install", - ) - subprocess.check_call(tar_cmd) - - # https://github.com/indygreg/python-build-standalone/issues/209 - if sys.platform == "linux" and ARCH == "x86_64": - for fname in ( - f"{dest}/lib/python{major}.{minor}/config-{major}.{minor}-x86_64-linux-gnu/Makefile", - f"{dest}/lib/python{major}.{minor}/_sysconfigdata__linux_x86_64-linux-gnu.py", - ): - print(f"XXX: fixing up build metadata in {fname}") - with open(fname) as f: - contents = f.read() - contents = _must_sub(CLANG_PP, "c++", contents) - contents = _must_sub(CLANG, "cc", contents) - with open(fname, "w") as f: - f.write(contents) - - py = os.path.join(dest, "bin", "python3") - subprocess.check_call((py, "-mensurepip")) - subprocess.check_call( - ( - *(py, "-mpip", "install"), - *("pip==25.0.1", "setuptools==75.8.0", "wheel==0.45.1"), - ) + for version in VERSIONS: + major, minor, *_ = version.split(".") + + # uv installs to e.g. cpython-3.14.3-linux-aarch64-gnu/ + matches = glob.glob(os.path.join(args.dest, f"cpython-{major}.{minor}.*")) + if len(matches) != 1: + raise AssertionError( + f"expected exactly one match for cpython-{major}.{minor}.*, " + f"got {matches}" ) - subprocess.check_call((py, "--version", "--version")) + installed_dir = matches[0] + + # symlink cpXY-cpXY -> the uv-installed directory for PATH compat + link = os.path.join(args.dest, f"cp{major}{minor}-cp{major}{minor}") + os.symlink(os.path.basename(installed_dir), link) + + # remove PEP 668 marker so pip works normally + marker = os.path.join( + installed_dir, + "lib", + f"python{major}.{minor}", + "EXTERNALLY-MANAGED", + ) + if os.path.exists(marker): + os.remove(marker) + + py = os.path.join(link, "bin", "python3") + subprocess.check_call((py, "-mensurepip")) + subprocess.check_call( + ( + py, + "-mpip", + "install", + "pip==25.0.1", + "setuptools==75.8.0", + "wheel==0.45.1", + ), + ) + subprocess.check_call((py, "--version", "--version")) return 0 diff --git a/packages.ini b/packages.ini index 645da7e9..68cf7e49 100644 --- a/packages.ini +++ b/packages.ini @@ -79,6 +79,7 @@ python_versions = <3.13 [babel==2.17.0] [backports-zstd==1.3.0] +python_versions = <3.14 [backrefs==5.9] @@ -284,24 +285,28 @@ apt_requires = wget brew_requires = wget custom_prebuild = prebuild/librdkafka v2.3.0 +python_versions = <3.14 [confluent-kafka==2.7.0] apt_requires = patch wget brew_requires = wget custom_prebuild = prebuild/librdkafka v2.6.1 +python_versions = <3.14 [confluent-kafka==2.8.0] apt_requires = patch wget brew_requires = wget custom_prebuild = prebuild/librdkafka v2.8.0 +python_versions = <3.14 [confluent-kafka==2.9.0] apt_requires = patch wget brew_requires = wget custom_prebuild = prebuild/librdkafka v2.8.0 +python_versions = <3.14 [confluent-kafka==2.12.2] apt_requires = patch @@ -826,18 +831,25 @@ custom_prebuild = prebuild/crc32c 1.1.2 [googleapis-common-protos==1.70.0] [granian==2.5.4] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.5.5] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.5.6] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.5.7] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.6.0] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.6.1] +python_versions = <3.14 validate_extras = pname,reload,uvloop [granian==2.7.0] +python_versions = <3.14 validate_extras = pname,reload,uvloop [greenlet==1.1.3] @@ -871,8 +883,11 @@ python_versions = <3.13 [grpcio==1.66.1] python_versions = <3.13 [grpcio==1.67.0] +python_versions = <3.14 [grpcio==1.72.0rc1] +python_versions = <3.14 [grpcio==1.73.1] +python_versions = <3.14 [grpcio==1.75.1] [grpcio-status==1.47.0] @@ -886,9 +901,13 @@ python_versions = <3.13 [grpcio-status==1.62.2] [grpcio-status==1.66.1] [grpcio-status==1.67.0] +python_versions = <3.14 [grpcio-status==1.72.0rc1] +python_versions = <3.14 [grpcio-status==1.73.1] +python_versions = <3.14 [grpcio-status==1.75.1] +python_versions = <3.14 [h11==0.9.0] [h11==0.12.0] @@ -1006,6 +1025,7 @@ python_versions = <3.12 [jinja2==3.1.6] [jiter==0.9.0] +python_versions = <3.14 [jiter==0.10.0] [jmespath==0.10.0] @@ -1072,6 +1092,7 @@ python_versions = <3.13 [librt==0.7.8] [lief==0.16.6] +python_versions = <3.14 [looseversion==1.0.2] @@ -1318,11 +1339,13 @@ python_versions = <3.13 [orjson==3.10.3] python_versions = <3.13 [orjson==3.10.10] +python_versions = <3.14 [outcome==1.2.0] [outcome==1.3.0.post0] [p4python==2025.1.2767466] +python_versions = <3.14 [packaging==21.3] [packaging==22.0] @@ -1587,8 +1610,11 @@ python_versions = <3.13 [pydantic-core==2.18.4] python_versions = <3.13 [pydantic-core==2.23.4] +python_versions = <3.14 [pydantic-core==2.24.2] +python_versions = <3.14 [pydantic-core==2.33.2] +python_versions = <3.14 [pyelftools==0.28] [pyelftools==0.29] @@ -1782,10 +1808,13 @@ validate_skip_imports = uwsgidecorators python_versions = <3.13 validate_skip_imports = uwsgidecorators [pyuwsgi==2.0.27.post1] +python_versions = <3.14 validate_skip_imports = uwsgidecorators [pyuwsgi==2.0.28.post1] +python_versions = <3.14 validate_skip_imports = uwsgidecorators [pyuwsgi==2.0.29] +python_versions = <3.14 validate_skip_imports = uwsgidecorators [pyuwsgi==2.0.30] validate_skip_imports = uwsgidecorators @@ -1904,6 +1933,7 @@ python_versions = <3.13 [rpds-py==0.15.2] python_versions = <3.13 [rpds-py==0.20.0] +python_versions = <3.14 [rq==1.0] @@ -2268,6 +2298,7 @@ python_versions = <3.13 [sentry-forked-email-reply-parser==0.5.12.post1] [sentry-forked-jsonnet==0.20.0.post4] +python_versions = <3.14 [sentry-infra-event-notifier==0.0.3] [sentry-infra-event-notifier==0.0.4] @@ -3026,19 +3057,32 @@ python_versions = >=3.10 [sentry-streams==0.0.15] [sentry-streams==0.0.16] [sentry-streams==0.0.17] +python_versions = <3.14 [sentry-streams==0.0.18] +python_versions = <3.14 [sentry-streams==0.0.19] +python_versions = <3.14 [sentry-streams==0.0.20] +python_versions = <3.14 [sentry-streams==0.0.21] +python_versions = <3.14 [sentry-streams==0.0.22] +python_versions = <3.14 [sentry-streams==0.0.23] +python_versions = <3.14 [sentry-streams==0.0.24] +python_versions = <3.14 [sentry-streams==0.0.31] +python_versions = <3.14 [sentry-streams==0.0.34] +python_versions = <3.14 [sentry-streams==0.0.35] +python_versions = <3.14 [sentry-streams-k8s==0.0.1] +python_versions = <3.14 [sentry-streams-k8s==0.0.2] +python_versions = <3.14 [sentry-streams-k8s==0.0.3] [sentry-streams-k8s==0.0.4] @@ -3285,6 +3329,7 @@ python_versions = <3.12 [tiktoken==0.6.0] python_versions = <3.13 [tiktoken==0.8.0] +python_versions = <3.14 [time-machine==2.12.0] python_versions = <3.13 @@ -3554,6 +3599,7 @@ python_versions = <3.13 [uv-build==0.9.28] [uvloop==0.21.0] +python_versions = <3.14 [vine==1.3.0] [vine==5.0.0] @@ -3573,23 +3619,41 @@ python_versions = <3.13 [virtualenv==20.36.1] [vroomrs==0.1.2] +python_versions = <3.14 [vroomrs==0.1.3] +python_versions = <3.14 [vroomrs==0.1.4] +python_versions = <3.14 [vroomrs==0.1.5] +python_versions = <3.14 [vroomrs==0.1.6] +python_versions = <3.14 [vroomrs==0.1.7] +python_versions = <3.14 [vroomrs==0.1.8] +python_versions = <3.14 [vroomrs==0.1.9] +python_versions = <3.14 [vroomrs==0.1.10] +python_versions = <3.14 [vroomrs==0.1.11] +python_versions = <3.14 [vroomrs==0.1.12] +python_versions = <3.14 [vroomrs==0.1.13] +python_versions = <3.14 [vroomrs==0.1.14] +python_versions = <3.14 [vroomrs==0.1.15] +python_versions = <3.14 [vroomrs==0.1.16] +python_versions = <3.14 [vroomrs==0.1.17] +python_versions = <3.14 [vroomrs==0.1.18] +python_versions = <3.14 [vroomrs==0.1.19] +python_versions = <3.14 [watchdog==2.1.9] python_versions = <3.12 @@ -3664,6 +3728,7 @@ python_versions = <3.13 apt_requires = pkg-config brew_requires = pkg-config custom_prebuild = prebuild/xmlsec-1-3-14-deps +python_versions = <3.14 [yamlfix==1.3.0] diff --git a/prebuild/crc32c b/prebuild/crc32c index 6663a84c..223f1f2b 100755 --- a/prebuild/crc32c +++ b/prebuild/crc32c @@ -49,6 +49,7 @@ def main() -> int: subprocess.check_call( ( "cmake", + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", f"-DCMAKE_INSTALL_PREFIX={args.prefix}", "-DCRC32C_BUILD_TESTS=no", "-DCRC32C_BUILD_BENCHMARKS=no", diff --git a/prebuild/librdkafka b/prebuild/librdkafka index ceb75ed9..fd60a6f6 100755 --- a/prebuild/librdkafka +++ b/prebuild/librdkafka @@ -39,20 +39,17 @@ def main() -> int: "./configure", "--enable-static", "--install-deps", - "--source-deps-only", f"--prefix={args.prefix}", ), cwd=tmpdir, ) print("building...") - subprocess.check_call(("make", "-j"), cwd=tmpdir) - print("testing...") - subprocess.check_call( - ("examples/rdkafka_example", "-X", "builtin.features"), cwd=tmpdir - ) + subprocess.check_call(("make", "-j", "-C", "src"), cwd=tmpdir) + subprocess.check_call(("make", "-j", "-C", "src-cpp"), cwd=tmpdir) print("installing...") - subprocess.check_call(("make", "install"), cwd=tmpdir) + subprocess.check_call(("make", "-C", "src", "install"), cwd=tmpdir) + subprocess.check_call(("make", "-C", "src-cpp", "install"), cwd=tmpdir) return 0 diff --git a/tests/validate_test.py b/tests/validate_test.py index 6484069d..116196cc 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -3,6 +3,7 @@ import zipfile import pytest +from packaging.specifiers import SpecifierSet from packaging.tags import parse_tag import validate @@ -11,6 +12,7 @@ def test_info_nothing_supplied(): info = validate.Info.from_dct({}) expected = validate.Info( + python_versions=SpecifierSet(), validate_extras=None, validate_incorrect_missing_deps=(), validate_skip_imports=(), @@ -27,6 +29,7 @@ def test_info_all_supplied(): } ) expected = validate.Info( + python_versions=SpecifierSet(), validate_extras="d", validate_incorrect_missing_deps=("six",), validate_skip_imports=("uwsgidecorators",), @@ -43,24 +46,24 @@ def test_pythons_to_check_no_pythons_raises_error(): def test_pythons_to_check_py2_ignored(): ret = validate._pythons_to_check(parse_tag("py2.py3-none-any")) - assert ret == ("python3.11", "python3.12", "python3.13") + assert ret == ("python3.11", "python3.12", "python3.13", "python3.14") def test_pythons_to_check_py3_gives_all(): ret = validate._pythons_to_check(parse_tag("py3-none-any")) - assert ret == ("python3.11", "python3.12", "python3.13") + assert ret == ("python3.11", "python3.12", "python3.13", "python3.14") def test_pythons_to_check_abi3(): tag = "cp37-abi3-manylinux1_x86_64" ret = validate._pythons_to_check(parse_tag(tag)) - assert ret == ("python3.11", "python3.12", "python3.13") + assert ret == ("python3.11", "python3.12", "python3.13", "python3.14") def test_pythons_to_check_minimum_abi3(): tag = "cp312-abi3-manylinux1_x86_64" ret = validate._pythons_to_check(parse_tag(tag)) - assert ret == ("python3.12", "python3.13") + assert ret == ("python3.12", "python3.13", "python3.14") def test_pythons_to_check_specific_cpython_tag(): @@ -76,7 +79,7 @@ def test_pythons_to_check_multi_platform_with_musllinux(): tags = parse_tag("py3-none-any") | parse_tag("py3-none-musllinux_1_2_x86_64") ret = validate._pythons_to_check(tags) # Should succeed because at least one tag (py3-none-any) is compatible - assert ret == ("python3.11", "python3.12", "python3.13") + assert ret == ("python3.11", "python3.12", "python3.13", "python3.14") def test_top_imports_record(tmp_path): diff --git a/validate.py b/validate.py index a04acb01..c1a72f33 100644 --- a/validate.py +++ b/validate.py @@ -12,15 +12,17 @@ from typing import NamedTuple import packaging.tags +from packaging.specifiers import SpecifierSet from packaging.tags import Tag from packaging.utils import parse_wheel_filename from packaging.version import Version -PYTHONS = ((3, 11), (3, 12), (3, 13)) +PYTHONS = ((3, 11), (3, 12), (3, 13), (3, 14)) DIST_INFO_RE = re.compile(r"^[^/]+.dist-info/[^/]+$") class Info(NamedTuple): + python_versions: SpecifierSet validate_extras: str | None validate_incorrect_missing_deps: tuple[str, ...] validate_skip_imports: tuple[str, ...] @@ -28,6 +30,7 @@ class Info(NamedTuple): @classmethod def from_dct(cls, dct: Mapping[str, str]) -> Info: return cls( + python_versions=SpecifierSet(dct.get("python_versions", "")), validate_extras=dct.get("validate_extras") or None, validate_incorrect_missing_deps=tuple( dct.get("validate_incorrect_missing_deps", "").split() @@ -160,16 +163,31 @@ def main() -> int: pkg, _, version_s = k.partition("==") packages[(pkg, Version(version_s))] = Info.from_dct(cfg[k]) + failed = [] for filename in sorted(os.listdir(args.dist)): name, version, _, wheel_tags = parse_wheel_filename(filename) info = packages[(name, version)] for python in _pythons_to_check(wheel_tags): - _validate( - python=python, - filename=os.path.join(args.dist, filename), - info=info, - index_url=args.index_url, - ) + # e.g. "python3.11" -> "3.11" + py_version = python[len("python") :] + if py_version not in info.python_versions: + continue + try: + _validate( + python=python, + filename=os.path.join(args.dist, filename), + info=info, + index_url=args.index_url, + ) + except subprocess.CalledProcessError: + failed.append(f"{name}=={version} ({python})") + print(f"!!! FAILED validation: {name}=={version} ({python})") + + if failed: + print(f"\nFAILED VALIDATIONS ({len(failed)}):") + for f in failed: + print(f" - {f}") + return 1 return 0