From 35b17b5dd0c5af3e67bb194166258f1468ec9239 Mon Sep 17 00:00:00 2001 From: jkh Date: Mon, 21 Sep 2026 17:46:04 -0700 Subject: [PATCH 1/2] Build libpython one compiler at a time on macOS Intel The macos-x86_64 job fails repeatedly with the cross compiler dying inside CPython: Python/symtable.c:1798 internal compiler error: Segmentation fault Objects/descrobject.o internal compiler error: Segmentation fault Python/Python-ast.c:9836 internal compiler error: Segmentation fault A different translation unit each run, always one of the larger generated ones, and only on that runner -- Linux x86_64 and arm64 pass every time. setup_cpython.sh drives the archive with -j$(nproc), which inside the 6 GB colima VM means three concurrent cc1 processes on CPython's heaviest files. PYTHONOS_BUILD_JOBS caps that, and the macOS Intel matrix entry sets it to 1. Anything unset, empty, non-numeric or non-positive falls back to nproc, so no other host changes behaviour and a typo cannot silently serialise every build. This is the second attempt at this failure -- 6f9c698 moved the VM from VZ to QEMU and it kept crashing. If serialising the compiler does not settle it either, the next thing to rule out is the QEMU backend itself rather than the build. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 5 +++++ GNUmakefile | 3 ++- tools/setup_cpython.sh | 11 ++++++++++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56864f3b..39ca06f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,9 @@ jobs: PYTHONOS_QEMU_ACCEL: tcg PYTHONOS_GUI_BOOT_TIMEOUT: "90" PYTHONOS_AUDIO_BOOT_TIMEOUT: "90" + # Empty everywhere except the macOS Intel runner, where setup_cpython.sh + # falls back to nproc. + PYTHONOS_BUILD_JOBS: ${{ matrix.build_jobs }} strategy: fail-fast: false matrix: @@ -41,6 +44,8 @@ jobs: # replace with macos-latest (Apple Silicon) without validating Docker. - name: macos-x86_64 runner: macos-15-intel + # One compiler at a time: see PYTHONOS_BUILD_JOBS below. + build_jobs: "1" qemu_bin: qemu-system-x86_64 artifact: pythonos-macos-x86_64 artifact_path: build/pythonos.iso diff --git a/GNUmakefile b/GNUmakefile index 2217773d..14a5b29c 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -464,7 +464,8 @@ docker-build: # ── CPython library (slow, cached — only rebuild if missing) ───────────────── $(LIBPYTHON): tools/setup_cpython.sh .docker-image - $(DOCKER_RUN) -e PYTHONOS_FREE_THREADING=$(PYTHONOS_FREE_THREADING) $(DOCKER_IMG) \ + $(DOCKER_RUN) -e PYTHONOS_FREE_THREADING=$(PYTHONOS_FREE_THREADING) \ + -e PYTHONOS_BUILD_JOBS=$(PYTHONOS_BUILD_JOBS) $(DOCKER_IMG) \ ./tools/setup_cpython.sh --build # ── Source file sets ($(wildcard) is evaluated by Make, not a subshell) ────── diff --git a/tools/setup_cpython.sh b/tools/setup_cpython.sh index 588291f4..5cdcc6fb 100755 --- a/tools/setup_cpython.sh +++ b/tools/setup_cpython.sh @@ -404,7 +404,16 @@ if [[ "$BUILD_REQUESTED" == "1" ]]; then # PYTHON_FOR_FREEZE — use python3.14 for non-bootstrap freeze # -W Makefile -W Modules/config.c: additionally tell make both are up-to-date # so the self-regen recipe is skipped if possible. - make -j"$(nproc)" "$LIBPYTHON_ARCHIVE" \ + # PYTHONOS_BUILD_JOBS caps parallelism for hosts that cannot feed nproc + # compilers at once. The macOS Intel runner builds inside a 6 GB colima VM, + # where three concurrent cc1 processes on CPython's larger generated + # translation units crash the compiler -- a different file each time. + jobs="${PYTHONOS_BUILD_JOBS:-$(nproc)}" + case "$jobs" in + ''|*[!0-9]*|0) jobs="$(nproc)" ;; + esac + echo "[setup_cpython] building libpython with -j$jobs" + make -j"$jobs" "$LIBPYTHON_ARCHIVE" \ -W Makefile \ -W Modules/config.c \ 'FREEZE_MODULE_BOOTSTRAP=python3.14 ./Programs/_freeze_module.py' \ From 2ce3dba2ec0a124ec7dfe95e21fd523d94f7c080 Mon Sep 17 00:00:00 2001 From: jkh Date: Mon, 21 Sep 2026 21:17:49 -0700 Subject: [PATCH 2/2] Give the macOS Intel GUI smoke a budget it can meet With libpython building one compiler at a time the cross build now completes on that runner -- no compiler crash, 59 smoke tests green -- and the job gets as far as test-gui-x86_64, where compositor.start() times out at 60 seconds. That runner boots the guest under TCG inside a QEMU-backed colima VM, so it is slower than the Linux hosts at every stage. PYTHONOS_GUI_COMMAND_- TIMEOUT already existed for exactly this; set it to 240 there. It is a deadline rather than a sleep, so a command that finishes quickly still returns quickly and the Linux jobs are unaffected. The two timeout readers now fall back to their defaults on anything unusable. A matrix key that is only set for one entry reaches the others as an empty string, and bare float("") raises -- the readers would have crashed the test rather than using the default they appear to promise. Empty, non-numeric and non-positive all fall back, so a typo cannot turn a timeout into zero either. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 11 ++++++++--- tests/gui_smoke_test.py | 21 +++++++++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39ca06f7..03464c52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,9 +21,10 @@ jobs: PYTHONOS_QEMU_ACCEL: tcg PYTHONOS_GUI_BOOT_TIMEOUT: "90" PYTHONOS_AUDIO_BOOT_TIMEOUT: "90" - # Empty everywhere except the macOS Intel runner, where setup_cpython.sh - # falls back to nproc. + # Both empty except on the macOS Intel runner, where the readers fall + # back to their defaults. PYTHONOS_BUILD_JOBS: ${{ matrix.build_jobs }} + PYTHONOS_GUI_COMMAND_TIMEOUT: ${{ matrix.gui_command_timeout }} strategy: fail-fast: false matrix: @@ -44,8 +45,12 @@ jobs: # replace with macos-latest (Apple Silicon) without validating Docker. - name: macos-x86_64 runner: macos-15-intel - # One compiler at a time: see PYTHONOS_BUILD_JOBS below. + # One compiler at a time, and a longer GUI command budget: this + # runner builds and boots inside a QEMU-backed colima VM, so both + # the cross compiler and the guest are markedly slower than the + # Linux hosts. build_jobs: "1" + gui_command_timeout: "240" qemu_bin: qemu-system-x86_64 artifact: pythonos-macos-x86_64 artifact_path: build/pythonos.iso diff --git a/tests/gui_smoke_test.py b/tests/gui_smoke_test.py index 4c0cdb06..a064e572 100644 --- a/tests/gui_smoke_test.py +++ b/tests/gui_smoke_test.py @@ -31,9 +31,26 @@ QemuMonitor, parse_ppm, sample_pixel, color_close, ) +def _env_seconds(name: str, default: float) -> float: + """Read a timeout from the environment, falling back on anything unusable. + + A workflow that sets this from an unset matrix key passes an empty string + rather than leaving the variable out, and bare float("") raises. Treat + empty, non-numeric and non-positive values as "not configured" so a typo + cannot turn a timeout into a crash -- or into zero. + """ + raw = os.environ.get(name, "") + try: + value = float(raw) + except ValueError: + return default + return value if value > 0 else default + + ISO = sys.argv[1] if len(sys.argv) > 1 else "build/pythonos.iso" PORT = int(os.environ.get("PYTHONOS_GUI_HOST_PORT", "5559")) -BOOT_TIMEOUT = float(os.environ.get("PYTHONOS_GUI_BOOT_TIMEOUT", "30")) +BOOT_TIMEOUT = _env_seconds("PYTHONOS_GUI_BOOT_TIMEOUT", 30.0) +COMMAND_TIMEOUT = _env_seconds("PYTHONOS_GUI_COMMAND_TIMEOUT", 15.0) SERIAL_LOG = "/tmp/pythonos-gui-smoke.log" MONITOR_SOCK = "/tmp/pythonos-gui-smoke.mon.sock" @@ -84,7 +101,7 @@ def _send(s: socket.socket, line: str, wait: float = 2.5) -> str: chunks = [] # Cross-architecture TCG can exceed the old 1-2.5 second budgets. This is # a deadline, not a sleep: successful commands still return immediately. - deadline = time.time() + max(wait, float(os.environ.get("PYTHONOS_GUI_COMMAND_TIMEOUT", "15"))) + deadline = time.time() + max(wait, COMMAND_TIMEOUT) marker_seen = False complete = False while time.time() < deadline: