diff --git a/.github/workflows/build_release_artifacts.yml b/.github/workflows/build_release_artifacts.yml index d82e9186..932feac3 100644 --- a/.github/workflows/build_release_artifacts.yml +++ b/.github/workflows/build_release_artifacts.yml @@ -1,4 +1,4 @@ -# Builds every artifact an `xy` release publishes: eleven cross-compiled platform +# Builds every artifact an `xy` release publishes: eleven runtime-verified native # wheels, a runtime-verified PyEmscripten wheel and an sdist. No single runner can # produce that set, which is why this is a repository-supplied build workflow # rather than the `uv build` the release pipeline runs by default. @@ -45,7 +45,7 @@ on: required: true type: string # Dry run: build and verify the full matrix without a release. Use it to - # exercise the cross-compile legs (musllinux, win-arm64, wasm, ...) after + # exercise the native targets (Linux, macOS, Windows, wasm, ...) after # touching the matrix, the toolchains or the wheel tags — a target added but # never exercised is unverified, not working. Nothing here can publish: the # upload path lives in publish.yml, which this workflow never calls. @@ -99,30 +99,21 @@ jobs: fail-fast: false matrix: include: - # Linux glibc (manylinux_2_17 floor) — cross-compiled with zig. - - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zigtarget: x86_64-unknown-linux-gnu.2.17, plat: manylinux_2_17_x86_64, zig: true, native: true } - - { os: ubuntu-latest, target: aarch64-unknown-linux-gnu, zigtarget: aarch64-unknown-linux-gnu.2.17, plat: manylinux_2_17_aarch64, zig: true, native: false } - - { os: ubuntu-latest, target: armv7-unknown-linux-gnueabihf, zigtarget: armv7-unknown-linux-gnueabihf.2.17, plat: manylinux_2_17_armv7l, zig: true, native: false } - # Linux musl / Alpine. crt-static is musl's default, and rustc - # silently *drops* cdylib output under it (a warning, not an error) - # since a fully-static binary can't also be a shared library — - # -C target-feature=-crt-static switches to dynamic linking against - # musl libc so a real cdylib gets produced. - - { os: ubuntu-latest, target: x86_64-unknown-linux-musl, zigtarget: x86_64-unknown-linux-musl, plat: musllinux_1_2_x86_64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - - { os: ubuntu-latest, target: aarch64-unknown-linux-musl, zigtarget: aarch64-unknown-linux-musl, plat: musllinux_1_2_aarch64, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - - { os: ubuntu-latest, target: armv7-unknown-linux-musleabihf, zigtarget: armv7-unknown-linux-musleabihf, plat: musllinux_1_2_armv7l, zig: true, native: false, rustflags: "-C target-feature=-crt-static" } - # macOS. Both build on the arm64 runner (macos-14): Apple Silicon - # runners are plentiful, while the last Intel runner (macos-13) is - # deprecated and frequently unschedulable. The Apple toolchain - # cross-links x86_64 Mach-O natively, so the Intel wheel is a cross - # build (native: false -> content-verified, not import-smoked, since - # an x86_64 lib can't be imported on the arm64 runner). - - { os: macos-14, target: x86_64-apple-darwin, plat: macosx_10_12_x86_64, zig: false, native: false, deployment: "10.12" } - - { os: macos-14, target: aarch64-apple-darwin, plat: macosx_11_0_arm64, zig: false, native: true, deployment: "11.0" } - # Windows x64 (native), x86 and arm64 (cross-compiled). - - { os: windows-latest, target: x86_64-pc-windows-msvc, plat: win_amd64, zig: false, native: true } - - { os: windows-latest, target: i686-pc-windows-msvc, plat: win32, zig: false, native: false } - - { os: windows-latest, target: aarch64-pc-windows-msvc, plat: win_arm64, zig: false, native: false } + # Linux glibc. ARM64 builds on its native runner; armv7 is tested under QEMU. + - { os: ubuntu-latest, target: x86_64-unknown-linux-gnu, zigtarget: x86_64-unknown-linux-gnu.2.17, plat: manylinux_2_17_x86_64, zig: true, native: true, smoke: host } + - { os: ubuntu-24.04-arm, target: aarch64-unknown-linux-gnu, zigtarget: aarch64-unknown-linux-gnu.2.17, plat: manylinux_2_17_aarch64, zig: true, native: true, smoke: host } + - { os: ubuntu-latest, target: armv7-unknown-linux-gnueabihf, zigtarget: armv7-unknown-linux-gnueabihf.2.17, plat: manylinux_2_17_armv7l, zig: true, native: true, smoke: manylinux-armv7 } + # musllinux wheels are runtime-smoked in Alpine, with QEMU for armv7. + - { os: ubuntu-latest, target: x86_64-unknown-linux-musl, zigtarget: x86_64-unknown-linux-musl, plat: musllinux_1_2_x86_64, zig: true, native: true, smoke: alpine, rustflags: "-C target-feature=-crt-static" } + - { os: ubuntu-24.04-arm, target: aarch64-unknown-linux-musl, zigtarget: aarch64-unknown-linux-musl, plat: musllinux_1_2_aarch64, zig: true, native: true, smoke: alpine, rustflags: "-C target-feature=-crt-static" } + - { os: ubuntu-latest, target: armv7-unknown-linux-musleabihf, zigtarget: armv7-unknown-linux-musleabihf, plat: musllinux_1_2_armv7l, zig: true, native: true, smoke: alpine-armv7, rustflags: "-C target-feature=-crt-static" } + # Native macOS runners cover both published architectures. + - { os: macos-15-intel, target: x86_64-apple-darwin, plat: macosx_10_12_x86_64, zig: false, native: true, smoke: host, deployment: "10.12" } + - { os: macos-14, target: aarch64-apple-darwin, plat: macosx_11_0_arm64, zig: false, native: true, smoke: host, deployment: "11.0" } + # Windows x64/ARM64 run natively; win32 uses an x86 Python. + - { os: windows-latest, target: x86_64-pc-windows-msvc, plat: win_amd64, zig: false, native: true, smoke: host } + - { os: windows-latest, target: i686-pc-windows-msvc, plat: win32, zig: false, native: true, smoke: host, python-architecture: x86 } + - { os: windows-11-arm, target: aarch64-pc-windows-msvc, plat: win_arm64, zig: false, native: true, smoke: host } steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -149,6 +140,11 @@ jobs: - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: "22" + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: matrix.python-architecture != '' + with: + python-version: "3.12" + architecture: ${{ matrix.python-architecture }} - name: Install JS build toolchain run: npm ci - name: Cross-compile the native core (zig) @@ -176,12 +172,12 @@ jobs: XY_SKIP_CARGO: "1" XY_CARGO_TARGET: ${{ matrix.target }} XY_WHEEL_PLATFORM: ${{ matrix.plat }} - run: uv build --wheel + run: uv build --python python --wheel - name: Verify wheel contents shell: bash run: | whl=$(ls dist/*.whl) - python scripts/verify_wheel.py "$whl" --expect-native + python scripts/verify_wheel.py "$whl" --expect-native --expect-platform "${{ matrix.plat }}" --require-symbol xy_abi_version --require-linkage - name: Install-size budget (<= 15 MB) shell: bash run: | @@ -189,16 +185,50 @@ jobs: size=$(wc -c < "$whl" | tr -d ' ') echo "wheel size: $size bytes ($whl)" test "$size" -le 15728640 + - name: Enable QEMU for ARMv7 runtime checks + if: matrix.smoke == 'manylinux-armv7' || matrix.smoke == 'alpine-armv7' + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + platforms: arm - name: Verify the wheel installs and loads the native core - # Only host-arch wheels can be imported on the runner; cross-compiled - # arches (aarch64/armv7/win-arm64/win32) are content-verified instead. - if: matrix.native + if: matrix.smoke == 'host' shell: bash run: | - uv venv smoke + uv venv smoke --python python uv pip install -p smoke dist/*.whl numpy anywidget "reflex>=0.9.6" - ./smoke/bin/python -c "import importlib.metadata as m, reflex_xy, xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; assert reflex_xy.__version__ == m.version('xy'); print('native', k.__file__)" \ - || ./smoke/Scripts/python.exe -c "import importlib.metadata as m, reflex_xy, xy.kernels as k; assert k.BACKEND=='native', k.BACKEND; assert reflex_xy.__version__ == m.version('xy'); print('native')" + if [ -f ./smoke/bin/python ]; then py=./smoke/bin/python; else py=./smoke/Scripts/python.exe; fi + "$py" scripts/wheel_smoke.py + - name: Verify ARMv7 manylinux wheel in a runtime container + if: matrix.smoke == 'manylinux-armv7' + shell: bash + run: | + docker run --rm --platform linux/arm/v7 -v "$GITHUB_WORKSPACE:/workspace" -w /workspace quay.io/pypa/manylinux2014_armv7l:latest sh -eu -c ' + /opt/python/cp311-cp311/bin/python -m venv /tmp/xy-smoke + /tmp/xy-smoke/bin/pip install numpy anywidget "reflex>=0.9.6" dist/*.whl + /tmp/xy-smoke/bin/python scripts/wheel_smoke.py + ' + - name: Verify musllinux wheel in Alpine runtime + if: matrix.smoke == 'alpine' + shell: bash + run: | + alpine_platform=linux/amd64 + if [ "${{ matrix.plat }}" = musllinux_1_2_aarch64 ]; then alpine_platform=linux/arm64; fi + docker run --rm --platform "$alpine_platform" -v "$GITHUB_WORKSPACE:/workspace" -w /workspace alpine:3.22 sh -eu -c ' + apk add --no-cache python3 py3-numpy py3-pip + python3 -m venv --system-site-packages /tmp/xy-smoke + /tmp/xy-smoke/bin/pip install anywidget "reflex>=0.9.6" dist/*.whl + /tmp/xy-smoke/bin/python scripts/wheel_smoke.py + ' + - name: Verify armv7 musllinux wheel in Alpine runtime + if: matrix.smoke == 'alpine-armv7' + shell: bash + run: | + docker run --rm --platform linux/arm/v7 -v "$GITHUB_WORKSPACE:/workspace" -w /workspace alpine:3.22 sh -eu -c ' + apk add --no-cache python3 py3-numpy py3-pip + python3 -m venv --system-site-packages /tmp/xy-smoke + /tmp/xy-smoke/bin/pip install anywidget "reflex>=0.9.6" dist/*.whl + /tmp/xy-smoke/bin/python scripts/wheel_smoke.py + ' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ inputs.artifact-prefix }}${{ matrix.plat }} @@ -365,6 +395,6 @@ jobs: # Merge the matrix output into the single artifact the gated job uploads, and # re-verify it as a batch: `verify-dist` fails unless *every* artifact — all - # eleven wheels, the wasm wheel and the sdist — declares the version being + # eleven native wheels, the wasm wheel and the sdist — declares the version being # released, which is what catches a build that silently fell back to the # derived development version. diff --git a/docs/app/tests/test_docs_site.py b/docs/app/tests/test_docs_site.py index 26a69e10..4619fb61 100644 --- a/docs/app/tests/test_docs_site.py +++ b/docs/app/tests/test_docs_site.py @@ -1604,17 +1604,28 @@ def test_installation_uses_uv_first_package_manager_tabs() -> None: def test_installation_distinguishes_supported_targets_from_pypi_artifacts() -> None: - """Do not present missing release wheels as missing platform support.""" + """Distinguish XY's supported targets from the files in the 0.0.1 upload.""" content = (DOCS_ROOT / "overview" / "installation.md").read_text(encoding="utf-8") + macos_row = next(line for line in content.splitlines() if line.startswith("| macOS |")) + glibc_row = next(line for line in content.splitlines() if "manylinux_2_17" in line) + musl_row = next(line for line in content.splitlines() if "musllinux_1_2" in line) windows_row = next(line for line in content.splitlines() if line.startswith("| Windows |")) wasm_row = next(line for line in content.splitlines() if line.startswith("| WebAssembly |")) - assert all(value in windows_row for value in ("`x86_64`", "`x86`", "`arm64`")) - assert "| Supported | Not included |" in windows_row + assert "`x86_64`, `arm64`" in macos_row + assert "macOS 10.12+ on Intel" in macos_row + assert "Included" in macos_row + assert "`x86_64`, `aarch64`, `armv7l`" in glibc_row + assert "Included" in glibc_row + assert "`x86_64`, `aarch64`, `armv7l`" in musl_row + assert "Included" in musl_row + assert "`x86_64`, `x86`, `arm64`" in windows_row + assert "Supported" in windows_row + assert "Not included" in windows_row assert "Pyodide 314" in wasm_row - assert "`wasm32` | Supported | Not in 0.0.1 (on PyPI since 0.0.3) |" in wasm_row - assert "Windows is supported by XY's native core and release pipeline." in content - assert "0.0.1 PyPI upload does not include Windows wheels" in content + assert "`wasm32`" in wasm_row + assert "Not in 0.0.1" in wasm_row + assert "Windows is supported by XY's native core and release pipeline" in content assert "runtime-verified WebAssembly wheel" in content assert "`pyemscripten_2026_0_wasm32` platform" in content assert "%pip install xy" in content diff --git a/news/473.misc.md b/news/473.misc.md new file mode 100644 index 00000000..d2a8cdc7 --- /dev/null +++ b/news/473.misc.md @@ -0,0 +1 @@ +Validate native wheel ABI, linkage and platform contracts, including normalized numeric development versions. diff --git a/pyproject.toml b/pyproject.toml index 33c1e35c..94282cfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -288,7 +288,7 @@ release-timezone = "America/Los_Angeles" tag-prefix = "v" # `xy` cannot be built by `uv build` on one runner: it publishes eleven -# cross-compiled platform wheels, a runtime-verified PyEmscripten wheel and an +# runtime-verified native platform wheels, a runtime-verified PyEmscripten wheel and an # sdist, each needing its own operating system or toolchain. So the build is # delegated to a workflow this repository owns, and the pipeline keeps # everything either side of it — same changelog detection, same version, same diff --git a/python/xy/kernels.py b/python/xy/kernels.py index fc40e861..3d31d748 100644 --- a/python/xy/kernels.py +++ b/python/xy/kernels.py @@ -17,10 +17,13 @@ except ImportError as err: # pragma: no cover - platform-dependent raise ImportError( "xy requires its native Rust core, which could not be loaded " - f"({err}). Prebuilt wheels cover Linux glibc and musl (x86-64, aarch64, " - "armv7), macOS (x86-64, Apple Silicon), and Windows (x86, x64, arm64); " - "on those platforms `pip install xy` needs no toolchain. On any " - "other platform, install a Rust toolchain (https://rustup.rs) and " + f"({err}). Native wheels cover Linux glibc and musl/Alpine " + "(x86-64, aarch64, and armv7), macOS Intel (10.12+) and Apple Silicon " + "(11+), and Windows x86, x64, and ARM64. Pyodide 314 has a " + "runtime-verified Emscripten wheel. Those installs need no toolchain. " + "On other platforms, " + "install a Rust toolchain " + "(https://rustup.rs) and " "reinstall from source (or run `cargo build --release`)." ) from err diff --git a/scripts/verify_wheel.py b/scripts/verify_wheel.py index 78e6115f..6aa7e07f 100644 --- a/scripts/verify_wheel.py +++ b/scripts/verify_wheel.py @@ -14,6 +14,7 @@ import csv import hashlib import re +import struct import sys import zipfile from dataclasses import dataclass @@ -107,6 +108,578 @@ class WheelInfo: tags: list[str] +@dataclass(frozen=True) +class NativeBinaryInfo: + """Architecture facts read from a packaged native library header.""" + + format: str + machine: str + bits: int + exported_symbols: frozenset[str] = frozenset() + + +_ELF_MACHINES = {40: "arm", 62: "x86_64", 183: "aarch64"} +_PE_MACHINES = {0x14C: "x86", 0x8664: "x86_64", 0xAA64: "aarch64"} +_MACHO_CPU_TYPES = {0x01000007: "x86_64", 0x0100000C: "aarch64"} +_WHEEL_TARGETS = { + "manylinux_2_17_x86_64": ("ELF", "x86_64", 64), + "manylinux_2_17_aarch64": ("ELF", "aarch64", 64), + "manylinux_2_17_armv7l": ("ELF", "arm", 32), + "musllinux_1_2_x86_64": ("ELF", "x86_64", 64), + "musllinux_1_2_aarch64": ("ELF", "aarch64", 64), + "musllinux_1_2_armv7l": ("ELF", "arm", 32), + "macosx_10_12_x86_64": ("Mach-O", "x86_64", 64), + "macosx_11_0_arm64": ("Mach-O", "aarch64", 64), + "win_amd64": ("PE", "x86_64", 64), + "win32": ("PE", "x86", 32), + "win_arm64": ("PE", "aarch64", 64), +} +_ELF_ALLOWED_GLIBC = { + "libc.so.6", + "libm.so.6", + "libgcc_s.so.1", + "libpthread.so.0", + "libdl.so.2", + "librt.so.1", + "libutil.so.1", + "ld-linux-x86-64.so.2", + "ld-linux-aarch64.so.1", + "ld-linux-armhf.so.3", +} +_ELF_ALLOWED_MUSL_PREFIXES = ("libc.musl-", "ld-musl-", "libgcc_s.so.") +_PE_ALLOWED_IMPORT_PREFIXES = ("api-ms-win-core-", "api-ms-win-crt-") +_PE_ALLOWED_IMPORTS = { + "bcryptprimitives.dll", + "kernel32.dll", + "ntdll.dll", + "msvcrt.dll", + "ucrtbase.dll", + "vcruntime140.dll", +} + + +def _inspect_native_binary(name: str, data: bytes) -> NativeBinaryInfo: + """Read format, machine, and bitness without platform-specific tools.""" + if data[:4] == b"\x7fELF": + if len(data) < 20: + raise AssertionError(f"{name} has a truncated ELF header") + elf_class, endian = data[4], data[5] + if elf_class not in {1, 2} or endian not in {1, 2}: + raise AssertionError(f"{name} has an invalid ELF class or byte order") + prefix = "<" if endian == 1 else ">" + machine = struct.unpack_from(prefix + "H", data, 18)[0] + if machine not in _ELF_MACHINES: + raise AssertionError(f"{name} has unsupported ELF machine {machine}") + bits = 32 if elf_class == 1 else 64 + symbols = _elf_exported_symbols(name, data, prefix, bits) + return NativeBinaryInfo("ELF", _ELF_MACHINES[machine], bits, symbols) + + if len(data) >= 8 and data[:4] in { + b"\xfe\xed\xfa\xce", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xcf\xfa\xed\xfe", + }: + prefix = ">" if data[:4] in {b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf"} else "<" + cputype = struct.unpack_from(prefix + "I", data, 4)[0] + if cputype not in _MACHO_CPU_TYPES: + raise AssertionError(f"{name} has unsupported Mach-O CPU type {cputype}") + bits = 64 if data[:4] in {b"\xfe\xed\xfa\xcf", b"\xcf\xfa\xed\xfe"} else 32 + symbols = _macho_exported_symbols(name, data, prefix, bits) + return NativeBinaryInfo("Mach-O", _MACHO_CPU_TYPES[cputype], bits, symbols) + + if len(data) >= 0x40 and data[:2] == b"MZ": + pe_offset = struct.unpack_from(" len(data) or data[pe_offset : pe_offset + 4] != b"PE\0\0": + raise AssertionError(f"{name} has an invalid PE header") + machine = struct.unpack_from(" frozenset[str]: + """Read defined global/weak names from an ELF dynamic symbol table.""" + if len(data) < (64 if bits == 64 else 52): + return frozenset() + if bits == 64: + section_offset, section_count = ( + struct.unpack_from(prefix + "QI", data, 40)[0], + struct.unpack_from(prefix + "H", data, 60)[0], + ) + section_size, _section_name_index = ( + struct.unpack_from(prefix + "H", data, 58)[0], + struct.unpack_from(prefix + "H", data, 62)[0], + ) + section_header_size = section_size + section_fields = (24, 32, 40, 56) + else: + section_offset = struct.unpack_from(prefix + "I", data, 32)[0] + section_count = struct.unpack_from(prefix + "H", data, 48)[0] + section_header_size = struct.unpack_from(prefix + "H", data, 46)[0] + _section_name_index = struct.unpack_from(prefix + "H", data, 50)[0] + section_fields = (16, 20, 24, 36) + if not section_offset or not section_count or not section_header_size: + return frozenset() + + sections: list[tuple[int, int, int, int, int]] = [] + for index in range(section_count): + offset = section_offset + index * section_header_size + if offset + section_header_size > len(data): + raise AssertionError(f"{name} has a truncated ELF section table") + section_type = struct.unpack_from(prefix + "I", data, offset + 4)[0] + section_file_offset = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[0] + )[0] + section_length = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[1] + )[0] + section_link = struct.unpack_from(prefix + "I", data, offset + section_fields[2])[0] + section_entry_size = struct.unpack_from( + prefix + ("Q" if bits == 64 else "I"), data, offset + section_fields[3] + )[0] + sections.append( + ( + section_type, + section_file_offset, + section_length, + section_link, + section_entry_size, + ) + ) + + dynamic_index = next((i for i, section in enumerate(sections) if section[0] == 11), None) + if dynamic_index is None: + return frozenset() + _, symbol_offset, symbol_length, string_index, entry_size = sections[dynamic_index] + if string_index >= len(sections): + raise AssertionError(f"{name} has an invalid ELF dynamic string-table link") + _, string_offset, string_length, _, _ = sections[string_index] + symbol_size = 24 if bits == 64 else 16 + entry_size = entry_size or symbol_size + if entry_size < symbol_size or symbol_offset + symbol_length > len(data): + raise AssertionError(f"{name} has an invalid ELF dynamic symbol table") + if string_offset + string_length > len(data): + raise AssertionError(f"{name} has a truncated ELF dynamic string table") + strings = data[string_offset : string_offset + string_length] + exported: set[str] = set() + symbol_end = symbol_offset + symbol_length - (symbol_length % entry_size) + for offset in range(symbol_offset, symbol_end, entry_size): + if bits == 64: + string_name, info, section_index = ( + struct.unpack_from(prefix + "I", data, offset)[0], + data[offset + 4], + struct.unpack_from(prefix + "H", data, offset + 6)[0], + ) + else: + string_name = struct.unpack_from(prefix + "I", data, offset)[0] + info = data[offset + 12] + section_index = struct.unpack_from(prefix + "H", data, offset + 14)[0] + if section_index == 0 or info >> 4 not in {1, 2} or string_name >= len(strings): + continue + end = strings.find(b"\0", string_name) + if end > string_name: + exported.add(strings[string_name:end].decode("utf-8", errors="replace")) + return frozenset(exported) + + +def _macho_exported_symbols(name: str, data: bytes, prefix: str, bits: int) -> frozenset[str]: + """Read externally defined names from a thin Mach-O symbol table.""" + header_size = 32 if bits == 64 else 28 + if len(data) < header_size: + return frozenset() + ncommands = struct.unpack_from(prefix + "I", data, 16)[0] + command_offset = header_size + symbol_table: tuple[int, int, int, int] | None = None + for _ in range(ncommands): + if command_offset + 8 > len(data): + raise AssertionError(f"{name} has a truncated Mach-O load-command table") + command, command_size = struct.unpack_from(prefix + "II", data, command_offset) + if command_size < 8 or command_offset + command_size > len(data): + raise AssertionError(f"{name} has an invalid Mach-O load command") + if command == 2 and command_size >= 24: # LC_SYMTAB + symbol_table = struct.unpack_from(prefix + "IIII", data, command_offset + 8) + break + command_offset += command_size + if symbol_table is None: + return frozenset() + symbol_offset, symbol_count, string_offset, string_size = symbol_table + entry_size = 16 if bits == 64 else 12 + if string_offset + string_size > len(data): + raise AssertionError(f"{name} has a truncated Mach-O string table") + strings = data[string_offset : string_offset + string_size] + exported: set[str] = set() + for index in range(symbol_count): + offset = symbol_offset + index * entry_size + if offset + entry_size > len(data): + raise AssertionError(f"{name} has a truncated Mach-O symbol table") + string_name = struct.unpack_from(prefix + "I", data, offset)[0] + symbol_type = data[offset + 4] + if not symbol_type & 0x01 or symbol_type & 0x0E == 0x00 or string_name >= len(strings): + continue + end = strings.find(b"\0", string_name) + if end > string_name: + exported.add(strings[string_name:end].decode("utf-8", errors="replace").lstrip("_")) + return frozenset(exported) + + +def _pe_exported_symbols( + name: str, data: bytes, pe_offset: int, optional_magic: int, bits: int +) -> frozenset[str]: + """Read names from the PE export directory.""" + optional_offset = pe_offset + 24 + data_directory_offset = optional_offset + (96 if optional_magic == 0x10B else 112) + if data_directory_offset + 8 > len(data): + return frozenset() + export_rva, export_size = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE section table") + virtual_size, virtual_address, raw_size, raw_offset = struct.unpack_from( + " int: + for virtual_address, size, raw_offset, _raw_size in sections: + if virtual_address <= rva < virtual_address + size: + return raw_offset + (rva - virtual_address) + raise AssertionError(f"{name} has an export RVA outside its PE sections") + + export_offset = file_offset(export_rva) + if export_offset + 40 > len(data): + raise AssertionError(f"{name} has a truncated PE export directory") + number_of_names = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE export-name table") + string_offset = file_offset(struct.unpack_from(" string_offset: + exported.add(data[string_offset:end].decode("ascii", errors="replace")) + return frozenset(exported) + + +def _macho_linkage( + name: str, data: bytes, prefix: str, bits: int +) -> tuple[tuple[str, ...], tuple[int, int] | None]: + """Read loaded dylibs and the minimum macOS version from Mach-O commands.""" + header_size = 32 if bits == 64 else 28 + if len(data) < header_size: + raise AssertionError(f"{name} has a truncated Mach-O header") + ncommands = struct.unpack_from(prefix + "I", data, 16)[0] + command_offset = header_size + dependencies: list[str] = [] + minimum: tuple[int, int] | None = None + dylib_commands = {0xC, 0x18 | 0x80000000, 0x1F | 0x80000000} + for _ in range(ncommands): + if command_offset + 8 > len(data): + raise AssertionError(f"{name} has a truncated Mach-O load-command table") + command, command_size = struct.unpack_from(prefix + "II", data, command_offset) + if command_size < 8 or command_offset + command_size > len(data): + raise AssertionError(f"{name} has an invalid Mach-O load command") + if command in dylib_commands and command_size >= 24: + name_offset = struct.unpack_from(prefix + "I", data, command_offset + 8)[0] + start = command_offset + name_offset + end = data.find(b"\0", start, command_offset + command_size) + if end > start: + dependencies.append(data[start:end].decode("utf-8", errors="replace")) + elif command == 0x32 and command_size >= 16: # LC_BUILD_VERSION + version = struct.unpack_from(prefix + "I", data, command_offset + 12)[0] + minimum = (version >> 16, (version >> 8) & 0xFF) + elif command == 0x24 and command_size >= 16: # LC_VERSION_MIN_MACOSX + version = struct.unpack_from(prefix + "I", data, command_offset + 8)[0] + minimum = (version >> 16, (version >> 8) & 0xFF) + command_offset += command_size + return tuple(dependencies), minimum + + +def _pe_imports(name: str, data: bytes, pe_offset: int, optional_magic: int) -> tuple[str, ...]: + """Read DLL names from the PE import and delay-load directories.""" + optional_offset = pe_offset + 24 + data_directory_offset = optional_offset + (96 if optional_magic == 0x10B else 112) + directory_count_offset = optional_offset + (92 if optional_magic == 0x10B else 108) + optional_size = struct.unpack_from(" optional_end: + return () + directory_count = struct.unpack_from(" len(data): + raise AssertionError(f"{name} has a truncated PE section table") + virtual_size, virtual_address, raw_size, raw_offset = struct.unpack_from( + " int: + for virtual_address, size, raw_offset in sections: + if virtual_address <= rva < virtual_address + size: + offset = raw_offset + rva - virtual_address + if offset < len(data): + return offset + break + raise AssertionError(f"{name} has an import RVA outside its PE sections") + + imports: list[str] = [] + image_base = struct.unpack_from( + " len(data): + raise AssertionError(f"{name} has a truncated PE import directory") + if is_delay: + attributes, name_value = struct.unpack_from(" None: + expected = _WHEEL_TARGETS.get(platform) + if expected is None: + raise AssertionError( + f"no native binary contract is defined for wheel platform {platform!r}" + ) + actual = _inspect_native_binary(name, data) + if (actual.format, actual.machine, actual.bits) != expected: + raise AssertionError( + f"{name} identifies as {actual.format}/{actual.machine}/{actual.bits}-bit, " + f"expected {expected[0]}/{expected[1]}/{expected[2]}-bit for {platform}" + ) + + +def _require_exported_symbols(name: str, data: bytes, required: set[str]) -> None: + """Require ABI symbols from the binary's export table.""" + info = _inspect_native_binary(name, data) + missing = sorted(required - info.exported_symbols) + if missing: + raise AssertionError(f"{name} is missing exported ABI symbols: {missing}") + + +def _elf_linkage(name: str, data: bytes, prefix: str, bits: int) -> tuple[str, tuple[str, ...]]: + """Return the ELF interpreter and DT_NEEDED names.""" + if bits == 64: + if len(data) < 64: + raise AssertionError(f"{name} has a truncated ELF header") + program_offset = struct.unpack_from(prefix + "Q", data, 32)[0] + program_entry_size = struct.unpack_from(prefix + "H", data, 54)[0] + program_count = struct.unpack_from(prefix + "H", data, 56)[0] + program_fields = (8, 16, 32) + dynamic_entry_size = 16 + else: + if len(data) < 52: + raise AssertionError(f"{name} has a truncated ELF header") + program_offset = struct.unpack_from(prefix + "I", data, 28)[0] + program_entry_size = struct.unpack_from(prefix + "H", data, 42)[0] + program_count = struct.unpack_from(prefix + "H", data, 44)[0] + program_fields = (4, 8, 16) + dynamic_entry_size = 8 + if not program_offset or not program_entry_size or not program_count: + raise AssertionError(f"{name} has no ELF program headers") + + load_segments: list[tuple[int, int, int, int]] = [] + interpreter: str | None = None + dynamic: tuple[int, int] | None = None + number_size = "Q" if bits == 64 else "I" + for index in range(program_count): + offset = program_offset + index * program_entry_size + if offset + program_entry_size > len(data): + raise AssertionError(f"{name} has a truncated ELF program-header table") + program_type = struct.unpack_from(prefix + "I", data, offset)[0] + file_offset = struct.unpack_from(prefix + number_size, data, offset + program_fields[0])[0] + virtual_address = struct.unpack_from( + prefix + number_size, data, offset + program_fields[1] + )[0] + file_size = struct.unpack_from(prefix + number_size, data, offset + program_fields[2])[0] + if program_type == 1: + load_segments.append((virtual_address, file_offset, file_size, file_size)) + elif program_type == 3: + raw = data[file_offset : file_offset + file_size] + interpreter = raw.split(b"\0", 1)[0].decode("utf-8", errors="replace") + elif program_type == 2: + dynamic = (file_offset, file_size) + if dynamic is None: + return interpreter or "", () + + def virtual_to_file(address: int) -> int: + for virtual_address, file_offset, file_size, _ in load_segments: + if virtual_address <= address < virtual_address + file_size: + return file_offset + address - virtual_address + raise AssertionError(f"{name} has an ELF dynamic string table outside load segments") + + dynamic_offset, dynamic_size = dynamic + needed: list[int] = [] + string_address = string_size = None + for offset in range(dynamic_offset, dynamic_offset + dynamic_size, dynamic_entry_size): + if offset + dynamic_entry_size > len(data): + raise AssertionError(f"{name} has a truncated ELF dynamic section") + tag = struct.unpack_from(prefix + number_size, data, offset)[0] + value = struct.unpack_from(prefix + number_size, data, offset + (8 if bits == 64 else 4))[0] + if tag == 0: + break + if tag == 1: + needed.append(value) + elif tag == 5: + string_address = value + elif tag == 10: + string_size = value + if string_address is None or string_size is None: + raise AssertionError(f"{name} has DT_NEEDED entries but no complete dynamic string table") + string_offset = virtual_to_file(string_address) + strings = data[string_offset : string_offset + string_size] + if len(strings) != string_size: + raise AssertionError(f"{name} has a truncated ELF dynamic string table") + dependencies = [] + for string_index in needed: + if string_index >= len(strings): + raise AssertionError(f"{name} has an invalid DT_NEEDED string index") + end = strings.find(b"\0", string_index) + if end < string_index: + raise AssertionError(f"{name} has an unterminated DT_NEEDED name") + dependencies.append(strings[string_index:end].decode("utf-8", errors="replace")) + return interpreter or "", tuple(dependencies) + + +def _require_elf_linkage(name: str, data: bytes, platform: str) -> None: + """Validate libc family and dependency policy for Linux wheel targets.""" + if data[:4] != b"\x7fELF": + raise AssertionError(f"{name} is not an ELF binary for Linux linkage validation") + endian = data[5] + prefix = "<" if endian == 1 else ">" if endian == 2 else "" + if not prefix: + raise AssertionError(f"{name} has an invalid ELF byte order") + bits = 32 if data[4] == 1 else 64 if data[4] == 2 else 0 + if not bits: + raise AssertionError(f"{name} has an invalid ELF class") + interpreter, dependencies = _elf_linkage(name, data, prefix, bits) + if platform.startswith("manylinux_"): + if interpreter and "ld-linux" not in interpreter: + raise AssertionError(f"{name} is not linked against glibc: interpreter={interpreter!r}") + # Shared objects normally have no PT_INTERP. A Rust cdylib with no + # dynamic libc dependency is also valid: it is self-contained and + # compatible with the tagged glibc floor. + unexpected = sorted(set(dependencies) - _ELF_ALLOWED_GLIBC) + if unexpected: + raise AssertionError(f"{name} has unsupported glibc dependencies: {unexpected}") + versions = { + (int(match.group(1)), int(match.group(2))) + for match in re.finditer(rb"GLIBC_(\d+)\.(\d+)", data) + } + if versions and max(versions) > (2, 17): + version = ".".join(map(str, max(versions))) + raise AssertionError(f"{name} requires glibc {version}, above manylinux_2_17") + elif platform.startswith("musllinux_"): + if interpreter and "musl" not in interpreter: + raise AssertionError(f"{name} is not linked against musl: interpreter={interpreter!r}") + unexpected = sorted( + dependency + for dependency in dependencies + if not dependency.startswith(_ELF_ALLOWED_MUSL_PREFIXES) + ) + if unexpected: + raise AssertionError(f"{name} has unsupported musl dependencies: {unexpected}") + + +def _require_macho_linkage(name: str, data: bytes, platform: str) -> None: + magic = data[:4] + if magic not in { + b"\xfe\xed\xfa\xce", + b"\xce\xfa\xed\xfe", + b"\xfe\xed\xfa\xcf", + b"\xcf\xfa\xed\xfe", + }: + raise AssertionError(f"{name} is not a Mach-O binary for macOS linkage validation") + prefix = ">" if magic in {b"\xfe\xed\xfa\xce", b"\xfe\xed\xfa\xcf"} else "<" + bits = 64 if magic in {b"\xfe\xed\xfa\xcf", b"\xcf\xfa\xed\xfe"} else 32 + dependencies, minimum = _macho_linkage(name, data, prefix, bits) + unexpected = sorted( + dependency + for dependency in dependencies + if not dependency.startswith(("/usr/lib/", "/System/Library/")) + ) + if unexpected: + raise AssertionError(f"{name} has unsupported macOS dependencies: {unexpected}") + expected_floor = (11, 0) if platform.endswith("arm64") else (10, 12) + if minimum is None: + raise AssertionError(f"{name} has no macOS deployment target") + if minimum > expected_floor: + raise AssertionError( + f"{name} targets macOS {minimum[0]}.{minimum[1]}, above {expected_floor[0]}.{expected_floor[1]}" + ) + + +def _require_pe_linkage(name: str, data: bytes) -> None: + if len(data) < 0x40 or data[:2] != b"MZ": + raise AssertionError(f"{name} is not a PE binary for Windows linkage validation") + pe_offset = struct.unpack_from(" len(data) or data[pe_offset : pe_offset + 4] != b"PE\0\0": + raise AssertionError(f"{name} has an invalid PE header") + optional_magic = struct.unpack_from(" str: matches = [n for n in names if n.endswith(f".dist-info/{filename}")] if len(matches) != 1: @@ -189,6 +762,13 @@ def _require_filename_tag(path: Path, tags: list[str]) -> None: ) +def _require_expected_platform_tag(platform: str, tags: list[str]) -> None: + if not any(tag.rsplit("-", 1)[-1] == platform for tag in tags): + raise AssertionError( + f"expected platform {platform!r} is not advertised in WHEEL tags {tags}" + ) + + def _normalize_numeric_local_version(version: str) -> str: # Hatchling normalizes an all-digit commit hash in the filename/dist-info, # while uv-dynamic-versioning can preserve its leading zeros in METADATA. @@ -326,13 +906,28 @@ def _require_record(zf: zipfile.ZipFile, names: set[str]) -> None: ) -def verify_wheel(path: Path, *, expect_native: Optional[bool]) -> None: +def verify_wheel( + path: Path, + *, + expect_native: Optional[bool], + expect_platform: Optional[str] = None, + required_symbols: Optional[set[str]] = None, + require_linkage: bool = False, +) -> None: + if require_linkage and (expect_native is not True or expect_platform is None): + raise AssertionError("linkage validation requires an expected native wheel platform tag") + if (expect_platform is not None or required_symbols) and expect_native is not True: + raise AssertionError("platform and ABI symbol checks require --expect-native") + if (expect_platform is not None or required_symbols) and expect_native is not True: + raise AssertionError("platform and ABI symbol checks require --expect-native") with zipfile.ZipFile(path) as zf: _require_unique_archive_members(zf.infolist()) names = set(zf.namelist()) _require_only_shippable_roots(names) wheel = _parse_wheel(names, zf.read(_dist_info_name(names, "WHEEL"))) _require_filename_tag(path, wheel.tags) + if expect_platform is not None: + _require_expected_platform_tag(expect_platform, wheel.tags) expected_version = _filename_version(path) _require_dist_info_version(names, expected_version) _require_metadata(names, zf.read(_dist_info_name(names, "METADATA")), expected_version) @@ -435,6 +1030,24 @@ def verify_wheel(path: Path, *, expect_native: Optional[bool]) -> None: raise AssertionError("native wheel must set Root-Is-Purelib: false") if any(tag == "py3-none-any" for tag in wheel.tags): raise AssertionError(f"native wheel must not use a pure tag: {wheel.tags}") + if expect_platform is not None or required_symbols: + with zipfile.ZipFile(path) as zf: + native_data = zf.read(native_libs[0]) + if expect_platform is not None: + _require_native_target(native_libs[0], native_data, expect_platform) + if required_symbols: + _require_exported_symbols(native_libs[0], native_data, required_symbols) + if require_linkage: + if expect_platform.startswith(("manylinux_", "musllinux_")): + _require_elf_linkage(native_libs[0], native_data, expect_platform) + elif expect_platform.startswith("macosx_"): + _require_macho_linkage(native_libs[0], native_data, expect_platform) + elif expect_platform.startswith("win_") or expect_platform == "win32": + _require_pe_linkage(native_libs[0], native_data) + else: + raise AssertionError( + f"linkage inspection is not implemented for wheel platform {expect_platform!r}" + ) elif expect_native is False: if native_libs: raise AssertionError( @@ -458,12 +1071,33 @@ def main(argv: Optional[list[str]] = None) -> int: group = parser.add_mutually_exclusive_group() group.add_argument("--expect-native", action="store_true") group.add_argument("--expect-pure", action="store_true") + parser.add_argument( + "--expect-platform", + help="validate the native library header against this wheel platform tag", + ) + parser.add_argument( + "--require-symbol", + action="append", + default=[], + help="require an exported native ABI symbol (repeatable)", + ) + parser.add_argument( + "--require-linkage", + action="store_true", + help="validate native dynamic linkage against --expect-platform", + ) args = parser.parse_args(argv) expect_native = True if args.expect_native else False if args.expect_pure else None try: - verify_wheel(args.wheel, expect_native=expect_native) - except (AssertionError, KeyError, zipfile.BadZipFile) as e: + verify_wheel( + args.wheel, + expect_native=expect_native, + expect_platform=args.expect_platform, + required_symbols=set(args.require_symbol), + require_linkage=args.require_linkage, + ) + except (AssertionError, KeyError, struct.error, zipfile.BadZipFile) as e: print(f"wheel verification failed for {args.wheel}: {e}", file=sys.stderr) return 1 print(f"wheel verification OK: {args.wheel}") diff --git a/scripts/wheel_smoke.py b/scripts/wheel_smoke.py new file mode 100644 index 00000000..7ac3c67e --- /dev/null +++ b/scripts/wheel_smoke.py @@ -0,0 +1,42 @@ +"""Run the representative native-kernel smoke test against an installed wheel.""" + +import importlib.metadata as metadata +from pathlib import Path + +import numpy as np + +import reflex_xy +import xy +import xy.kernels as kernels + + +def main() -> None: + if kernels.BACKEND != "native": + raise RuntimeError(f"expected native backend, got {kernels.BACKEND!r}") + distribution = metadata.distribution("xy") + installed_files = { + str(file).replace("\\", "/"): distribution.locate_file(file).resolve() + for file in distribution.files or () + } + for member, imported_path in ( + ("xy/__init__.py", xy.__file__), + ("xy/kernels.py", kernels.__file__), + ("reflex_xy/__init__.py", reflex_xy.__file__), + ): + if installed_files.get(member) != Path(imported_path).resolve(): + raise RuntimeError(f"{member} was not imported from the installed xy wheel") + + native_members = [member for member in installed_files if member.startswith("xy/_native_lib/")] + native_path = Path(kernels._impl._lib._name).resolve() + if len(native_members) != 1 or installed_files[native_members[0]] != native_path: + raise RuntimeError("native core was not loaded from the installed xy wheel") + if xy.__version__ != distribution.version or reflex_xy.__version__ != distribution.version: + raise RuntimeError("installed reflex_xy and xy versions do not match") + codes, unique = kernels.factorize_fixed(np.asarray(["a", "b", "a"], dtype="S1")) + if codes.tolist() != [0, 1, 0] or unique.tolist() != [0, 1]: + raise RuntimeError(f"unexpected factorize_fixed result: {codes}, {unique}") + print("native", kernels.__file__) + + +if __name__ == "__main__": + main() diff --git a/spec/design-dossier.md b/spec/design-dossier.md index ac0fcc82..89bc73c4 100644 --- a/spec/design-dossier.md +++ b/spec/design-dossier.md @@ -1232,8 +1232,11 @@ hits a source build requiring a Rust toolchain — an instant adoption cliff. from Python with `ctypes`. There is no CPython extension ABI at all, so one `py3-none-` wheel covers every supported Python version on that platform without PyO3 or `abi3`. Wheel matrix in CI, release-blocking: manylinux - (x86_64 + aarch64), macOS (arm64 + x86_64), Windows x86_64. A missing native wheel + (x86_64 + aarch64 + armv7), musllinux (x86_64 + aarch64 + armv7), macOS + (arm64 + x86_64), and Windows (x86_64 + x86 + arm64). Each published wheel + gets an install and kernel smoke on a native runner or under emulation. A missing native wheel is a release failure, not an end-user surprise. + The release also includes a separate runtime-verified PyEmscripten wheel. 2. **The JS/WebGL2 render client as bundled static assets** inside the same wheel — versioned, no CDN dependency (notebooks are often airgapped; §23's CSP rules apply). The bundles (`python/xy/static/*.js`) are a **generated artifact, not @@ -1951,8 +1954,9 @@ detail or honesty. ### F1 — Packaging & distribution is unspecified. For Python-only, it's the highest risk. [Critical] **Failure scenario.** `pip install ` must deliver three separately-hard things: -(a) the **native Rust core** as prebuilt wheels across the matrix — manylinux (x86_64 + -aarch64), macOS (arm64 + x86_64), Windows — via a plain C-ABI `cdylib` built by +(a) the **native Rust core** as prebuilt wheels across the runtime-verified matrix — +manylinux and musllinux (x86_64, aarch64, and armv7), macOS (arm64 and x86_64), +and Windows (x86_64, x86, and arm64) — via a plain C-ABI `cdylib` built by Hatchling, so the Python-version cross-product disappears without PyO3 or `abi3`; (b) the compiled **JS/WebGL2 render client** as bundled static assets; (c) a **notebook front-end integration** that injects that client diff --git a/spec/process/production-readiness.md b/spec/process/production-readiness.md index f7d916a0..a5822706 100644 --- a/spec/process/production-readiness.md +++ b/spec/process/production-readiness.md @@ -586,7 +586,7 @@ artifacts carry it. Three consequences worth knowing: | `dispatch_release.yml` | manual | Materializes `news/` fragments into `CHANGELOG.md` at the next version. Final releases land through a pull request; prereleases go straight to an `r/pre-*` branch. | | `release_from_changelog.yml` | push to `main`, `r/pre-**`, `r/hotfix/**` | Publishes any changelog version that has no git tag, by calling `publish.yml`. | | `publish.yml` | called by the above, or manual | Validates the request, calls the build workflow below, waits for `pypi` environment approval, uploads, then tags and creates the GitHub release. | -| `build_release_artifacts.yml` | called by `publish.yml`, or manual for a dry run | **This repository's own**: the release matrix — eleven platform wheels, the runtime-verified PyEmscripten wheel, and the sdist. | +| `build_release_artifacts.yml` | called by `publish.yml`, or manual for a dry run | **This repository's own**: the release matrix — eleven runtime-verified native wheels, the runtime-verified PyEmscripten wheel, and the sdist. | | `deploy-docs-stg.yml` | dispatched by `publish.yml` per published tag, or manual | **This repository's own**: builds and deploys the docs site for the version just published. | | `deploy-docs-site.yml` | manual (`source_ref`, default `main`) | **This repository's own**: builds the docs site from a branch, tag, or commit and promotes the same images through staging and production approval, independently of a library release. | @@ -629,7 +629,7 @@ pull request, so a stale workflow (or a configuration change made without re-syncing) is a red PR rather than a surprise at release time. What this repository owns is `build_release_artifacts.yml`, because `xy` cannot -be built by `uv build` on one runner: eleven cross-compiled platform wheels, a +be built by `uv build` on one runner: eleven runtime-verified native platform wheels, a runtime-verified PyEmscripten wheel and an sdist, each needing its own operating system or toolchain. `[[tool.reflex-release.custom-build]]` points the pipeline at it, and `publish.yml` calls it in place of its own build job. Everything @@ -803,16 +803,15 @@ Before merging a release pull request: keeps the generated four honest, and `collect` verifies the artifacts at release time. - Before the first release after a change to the wheel matrix (new target, - cross-compile toolchain, or tagging scheme), manually run - `build_release_artifacts.yml` (`workflow_dispatch`) and confirm every leg of - the cross-compile matrix — including the newer aarch64/armv7/musllinux/ - win-arm64 targets and the wasm job — actually builds, since a target added to - the matrix but never exercised in CI is unverified, not working. That run - reaches no publish path at all. Add the new platform to `expect-artifacts` in - the same change, or a release can ship without it. -- Confirm CI built and verified native wheels for Linux glibc and musl/Alpine + toolchain, or tagging scheme), manually run `build_release_artifacts.yml` + (`workflow_dispatch`) and confirm every native target and the wasm job builds + and passes its runtime gate, including native or emulated install and kernel + smokes for each published architecture. That run reaches no publish path. Add each + published platform to `expect-artifacts` in the same change, or a release can + ship without it. +- Confirm CI built and installed the native wheels for Linux glibc and musl/Alpine (x86-64, aarch64, armv7), macOS (x86-64, Apple Silicon), and Windows (x86, x64, - arm64). + arm64). ARMv7 smokes run in QEMU containers; other targets use native runners. - Confirm the Pyodide/Emscripten wheel passes its runtime load gate, not only its structural wheel check. The tested toolchain is Rust 1.97.0 with `panic=abort`, Emscripten 5.0.3, cibuildwheel 4.1.0, the PEP 783 @@ -833,8 +832,10 @@ Before merging a release pull request: runtime dependencies, and Matplotlib/Reflex extras. It must exclude repository-only docs, tests, gallery evidence, scripts, benchmarks, examples, native binaries, generated caches, unsafe paths, and duplicate members. -- Confirm each platform wheel passes `scripts/verify_wheel.py --expect-native` - and its install smoke loads `xy.kernels.BACKEND == "native"`. Confirm the +- Confirm each native platform wheel passes `scripts/verify_wheel.py + --expect-native --expect-platform --require-symbol xy_abi_version + --require-linkage` and its install smoke calls a representative kernel and + loads `xy.kernels.BACKEND == "native"`. Confirm the fallback `py3-none-any` wheel passes `--expect-pure` and fails compute with the documented native-core error. Wheel `METADATA` must keep `Name: xy`, `Requires-Python: >=3.11`, diff --git a/tests/test_release_process.py b/tests/test_release_process.py index 31ade178..5c0e582a 100644 --- a/tests/test_release_process.py +++ b/tests/test_release_process.py @@ -196,6 +196,39 @@ def test_expect_artifacts_matches_what_the_build_workflow_produces() -> None: assert "*.tar.gz" in entry["expect-artifacts"] +def test_every_native_wheel_target_runs_the_install_smoke() -> None: + entry = RELEASE_CONFIG["custom-build"][0] + build = (ROOT / ".github" / "workflows" / entry["workflow"]).read_text(encoding="utf-8") + wheel_entries = [ + line for line in build.splitlines() if line.lstrip().startswith("- {") and "plat:" in line + ] + + assert wheel_entries, "the native wheel matrix must not be empty" + assert all(re.search(r"\bnative: true\b", line) for line in wheel_entries), wheel_entries + targets = set(re.findall(r"plat: ([A-Za-z0-9_]+)", build)) + assert targets == { + "manylinux_2_17_x86_64", + "manylinux_2_17_aarch64", + "manylinux_2_17_armv7l", + "musllinux_1_2_x86_64", + "musllinux_1_2_aarch64", + "musllinux_1_2_armv7l", + "macosx_10_12_x86_64", + "macosx_11_0_arm64", + "win_amd64", + "win32", + "win_arm64", + } + smoke_modes = set(re.findall(r"\bsmoke: ([A-Za-z0-9-]+)", "\n".join(wheel_entries))) + assert smoke_modes == {"host", "manylinux-armv7", "alpine", "alpine-armv7"} + assert "if: matrix.smoke == 'host'" in build + assert "if: matrix.smoke == 'manylinux-armv7'" in build + assert "if: matrix.smoke == 'alpine'" in build + assert "if: matrix.smoke == 'alpine-armv7'" in build + assert "docker/setup-qemu-action@" in build + assert "scripts/wheel_smoke.py" in build + + def test_post_release_workflow_declares_the_dispatch_contract() -> None: # The release pipeline dispatches this workflow with `tag`, `package` and # `version` once a tag is published. reflex-release checks only that the name diff --git a/tests/test_verify_wheel.py b/tests/test_verify_wheel.py index 9ee6ecff..c82cd8da 100644 --- a/tests/test_verify_wheel.py +++ b/tests/test_verify_wheel.py @@ -3,6 +3,7 @@ import base64 import hashlib import importlib.util +import struct import sys import zipfile from pathlib import Path @@ -106,6 +107,272 @@ def _load_verify_module(): verify_wheel = _load_verify_module() +@pytest.mark.parametrize( + ("platform", "binary"), + [ + ( + "manylinux_2_17_x86_64", + b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + verify_wheel._require_native_target("native", binary, platform) + + +def test_native_binary_header_rejects_wrong_architecture() -> None: + binary = b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + binary = b"\x7fELF" + bytes([2, 1, 1, 0]) + bytes(10) + struct.pack(" None: + data = bytearray(1024) + data[:4] = b"\x7fELF" + data[4:8] = bytes([2, 1, 1, 0]) + struct.pack_into(" None: + data = bytearray(48) + data[:4] = b"\xcf\xfa\xed\xfe" + struct.pack_into(" None: + data = bytearray(104) + data[:4] = b"\xcf\xfa\xed\xfe" + struct.pack_into(" None: + with pytest.raises(AssertionError, match="requires an expected native wheel platform"): + verify_wheel.verify_wheel(Path("missing.whl"), expect_native=True, require_linkage=True) + + +def test_linkage_validation_requires_native_wheel() -> None: + with pytest.raises(AssertionError, match="requires an expected native wheel platform"): + verify_wheel.verify_wheel( + Path("missing.whl"), + expect_native=None, + expect_platform="win_amd64", + require_linkage=True, + ) + + +@pytest.mark.parametrize( + "options", + [ + {"expect_platform": "win_amd64"}, + {"required_symbols": {"xy_abi_version"}}, + ], +) +def test_platform_and_symbol_checks_require_native_mode(options: dict[str, object]) -> None: + with pytest.raises(AssertionError, match="require --expect-native"): + verify_wheel.verify_wheel(Path("missing.whl"), expect_native=None, **options) + + +def _elf_linkage_fixture( + interpreter: bytes, dependency: bytes | tuple[bytes, ...], version: bytes = b"" +) -> bytes: + dependencies = (dependency,) if isinstance(dependency, bytes) else dependency + dynamic_size = (3 + len(dependencies)) * 16 + data = bytearray(768) + data[:4] = b"\x7fELF" + data[4:8] = bytes([2, 1, 1, 0]) + struct.pack_into(" None: + binary = _elf_linkage_fixture(b"/lib64/ld-linux-x86-64.so.2\0", b"libc.so.6", b"GLIBC_2.17") + + verify_wheel._require_elf_linkage("native", binary, "manylinux_2_17_x86_64") + + with pytest.raises(AssertionError, match="above manylinux_2_17"): + verify_wheel._require_elf_linkage( + "native", + _elf_linkage_fixture(b"/lib64/ld-linux-x86-64.so.2\0", b"libc.so.6", b"GLIBC_2.28"), + "manylinux_2_17_x86_64", + ) + + with pytest.raises(AssertionError, match="unsupported glibc dependencies"): + verify_wheel._require_elf_linkage( + "native", + _elf_linkage_fixture( + b"/lib64/ld-linux-x86-64.so.2\0", b"libnotallowed.so.1", b"GLIBC_2.17" + ), + "manylinux_2_17_x86_64", + ) + + +def test_musl_shared_library_without_interpreter_uses_needed_libc() -> None: + binary = _elf_linkage_fixture(b"", b"libc.musl-x86_64.so.1") + verify_wheel._require_elf_linkage("native", binary, "musllinux_1_2_x86_64") + + +@pytest.mark.parametrize("platform", ["manylinux_2_17_x86_64", "musllinux_1_2_x86_64"]) +def test_self_contained_elf_shared_library_needs_no_interpreter_or_libc(platform: str) -> None: + # ELF shared objects do not use PT_INTERP. A self-contained cdylib can + # also have no dynamic libc dependency and still be compatible with either + # platform's ABI floor. + binary = _elf_linkage_fixture(b"", ()) + + verify_wheel._require_elf_linkage("native", binary, platform) + + +@pytest.mark.parametrize( + "dependency", + [b"libpthread.so.0", b"libdl.so.2", b"librt.so.1", b"libutil.so.1"], +) +def test_elf_linkage_accepts_manylinux2014_system_libraries(dependency: bytes) -> None: + binary = _elf_linkage_fixture(b"/lib64/ld-linux-x86-64.so.2\0", dependency, b"GLIBC_2.17") + verify_wheel._require_elf_linkage("native", binary, "manylinux_2_17_x86_64") + + +def test_elf_linkage_accepts_combined_zig_manylinux_system_dependencies() -> None: + binary = _elf_linkage_fixture( + b"/lib64/ld-linux-x86-64.so.2\0", + ( + b"libc.so.6", + b"libm.so.6", + b"libgcc_s.so.1", + b"ld-linux-x86-64.so.2", + b"libpthread.so.0", + b"libdl.so.2", + ), + b"GLIBC_2.17", + ) + verify_wheel._require_elf_linkage("native", binary, "manylinux_2_17_x86_64") + + +def _pe_linkage_fixture(dependency: bytes, *, delay_load: bool = False) -> bytes: + data = bytearray(0x400) + data[:2] = b"MZ" + struct.pack_into(" None: + verify_wheel._require_pe_linkage("native", _pe_linkage_fixture(dependency)) + + +def test_pe_linkage_rejects_imports_that_only_prefix_match_a_system_dll() -> None: + with pytest.raises(AssertionError, match="unsupported Windows imports"): + verify_wheel._require_pe_linkage("native", _pe_linkage_fixture(b"kernel32.dll.evil\0")) + + +def test_pe_linkage_rejects_unsupported_delay_load_imports() -> None: + with pytest.raises(AssertionError, match="unsupported Windows imports"): + verify_wheel._require_pe_linkage( + "native", _pe_linkage_fixture(b"evil.dll\0", delay_load=True) + ) + + +def test_expected_platform_must_match_wheel_tag(tmp_path: Path) -> None: + whl = tmp_path / "xy-0.0.1-py3-none-macosx_11_0_arm64.whl" + _write_wheel(whl) + with pytest.raises(AssertionError, match="expected platform 'win_amd64'"): + verify_wheel.verify_wheel(whl, expect_native=True, expect_platform="win_amd64") + + def _record_hash(data: bytes) -> str: digest = hashlib.sha256(data).digest() return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii")