From d13b32def0abc405308dcef12fe62fd0a63073a8 Mon Sep 17 00:00:00 2001 From: loonghao Date: Fri, 9 Oct 2026 01:48:01 +0800 Subject: [PATCH 1/2] fix: verify legacy Python distributions and build workflows --- .github/workflows/test-legacy-python.yml | 130 ++++ Cargo.lock | 2 + crates/vx-providers/python/provider.star | 63 +- crates/vx-runtime-http/Cargo.toml | 2 + crates/vx-runtime-http/src/installer.rs | 61 +- .../tests/artifact_integrity_tests.rs | 128 ++++ .../src/manifest_runtime/install.rs | 137 +++- crates/vx-runtime/src/manifest_runtime/mod.rs | 34 + crates/vx-runtime/src/traits.rs | 5 + .../tests/manifest_install_lock_tests.rs | 159 ++++ crates/vx-starlark/src/provider/execute.rs | 53 +- crates/vx-starlark/src/provider/types.rs | 14 + .../tests/archive_integrity_tests.rs | 81 ++ .../vx-starlark/tests/python_legacy_tests.rs | 117 ++- docs/guide/provider-star-layouts.md | 14 +- docs/tools/python.md | 36 +- docs/zh/tools/python.md | 36 +- scripts/test_legacy_python.py | 718 ++++++++++++++++++ scripts/tests/test_legacy_python_harness.py | 118 +++ tests/fixtures/legacy-python-build/native.c | 19 + .../legacy-python-build/pyproject.toml | 3 + tests/fixtures/legacy-python-build/setup.py | 36 + .../vx_legacy_probe/__init__.py | 12 + 23 files changed, 1885 insertions(+), 93 deletions(-) create mode 100644 .github/workflows/test-legacy-python.yml create mode 100644 crates/vx-runtime-http/tests/artifact_integrity_tests.rs create mode 100644 crates/vx-starlark/tests/archive_integrity_tests.rs create mode 100644 scripts/test_legacy_python.py create mode 100644 scripts/tests/test_legacy_python_harness.py create mode 100644 tests/fixtures/legacy-python-build/native.c create mode 100644 tests/fixtures/legacy-python-build/pyproject.toml create mode 100644 tests/fixtures/legacy-python-build/setup.py create mode 100644 tests/fixtures/legacy-python-build/vx_legacy_probe/__init__.py diff --git a/.github/workflows/test-legacy-python.yml b/.github/workflows/test-legacy-python.yml new file mode 100644 index 000000000..0cdade670 --- /dev/null +++ b/.github/workflows/test-legacy-python.yml @@ -0,0 +1,130 @@ +name: Legacy Python Acceptance + +on: + pull_request: + paths: + - "crates/vx-providers/python/**" + - "crates/vx-providers/uv/**" + - "crates/vx-runtime/**" + - "crates/vx-runtime-http/**" + - "crates/vx-runtime-archive/**" + - "crates/vx-cache/**" + - "crates/vx-paths/**" + - "crates/vx-cli/**" + - "crates/vx-resolver/**" + - "crates/vx-starlark/**" + - "scripts/test_legacy_python.py" + - "scripts/tests/test_legacy_python_harness.py" + - "tests/fixtures/legacy-python-build/**" + - ".github/workflows/test-legacy-python.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build legacy acceptance VX (${{ matrix.platform }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + os: ubuntu-24.04 + binary: vx + - platform: windows-x64 + os: windows-2025 + binary: vx.exe + env: + CARGO_TERM_COLOR: always + RUSTC_WRAPPER: sccache + SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: ./.github/actions/setup-rust-sccache + with: + shared-key: legacy-python-build + - name: Bootstrap VX for build commands + uses: ./ + with: + cache: "false" + setup: "false" + - name: Build this revision + run: vx cargo build --locked --profile dev-fast --bin vx + - uses: actions/upload-artifact@v7 + with: + name: legacy-python-vx-${{ matrix.platform }} + path: target/dev-fast/${{ matrix.binary }} + if-no-files-found: error + retention-days: 3 + + acceptance: + name: Legacy Python 3.7 (${{ matrix.platform }}) + needs: build + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-x64 + os: ubuntu-24.04 + binary: vx + - platform: windows-x64 + os: windows-2025 + binary: vx.exe + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/download-artifact@v8 + with: + name: legacy-python-vx-${{ matrix.platform }} + path: bin + - name: Make VX executable + if: runner.os != 'Windows' + run: chmod +x bin/vx + - name: Bootstrap modern harness interpreter + uses: ./ + with: + tools: "python@3.12" + cache: "false" + setup: "false" + - name: Bind existing Windows compiler + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + - name: Check acceptance guards + run: vx python@3.12 -m unittest discover -s scripts/tests -p test_legacy_python_harness.py + - name: Run isolated legacy acceptance + shell: bash + env: + VX_ACCEPTANCE_BINARY: ${{ github.workspace }}/bin/${{ matrix.binary }} + VX_ACCEPTANCE_ROOT: ${{ runner.temp }}/vx-legacy-python + VX_ACCEPTANCE_RECEIPT: ${{ github.workspace }}/legacy-python-receipt.json + VX_ACCEPTANCE_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + vx python@3.12 scripts/test_legacy_python.py \ + --vx-binary "$VX_ACCEPTANCE_BINARY" \ + --work-root "$VX_ACCEPTANCE_ROOT" \ + --receipt "$VX_ACCEPTANCE_RECEIPT" \ + --source-revision "$VX_ACCEPTANCE_SOURCE_SHA" + - name: Upload acceptance evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: legacy-python-evidence-${{ matrix.platform }} + path: | + legacy-python-receipt.json + ${{ runner.temp }}/vx-legacy-python/run-*/logs + if-no-files-found: warn + retention-days: 14 diff --git a/Cargo.lock b/Cargo.lock index bffb70410..655cdad21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5218,12 +5218,14 @@ dependencies = [ "bzip2", "flate2", "futures-util", + "hex", "indicatif", "regex", "reqwest", "rstest", "serde_json", "sevenz-rust", + "sha2 0.11.0", "tar", "tempfile", "tokio", diff --git a/crates/vx-providers/python/provider.star b/crates/vx-providers/python/provider.star index 4f2e9420f..40bcb7c7c 100644 --- a/crates/vx-providers/python/provider.star +++ b/crates/vx-providers/python/provider.star @@ -80,11 +80,35 @@ _PBS_TRIPLES = { } _LEGACY_37_ASSETS = { - # python-build-standalone's 20200822 release predates the modern - # cpython-{version}+{build_tag}-{triple}-install_only_stripped naming. - "windows/x64": "cpython-3.7.9-x86_64-pc-windows-msvc-shared-pgo-20200823T0118.tar.zst", - "macos/x64": "cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T0123.tar.zst", - "linux/x64": "cpython-3.7.9-x86_64-unknown-linux-gnu-pgo-20200823T0036.tar.zst", + # Fixed lifecycle catalog: release metadata must not select a different + # artifact after upstream stops publishing this Python line. + # Digests are VX intake pins of official HTTPS release assets; these old + # releases publish no checksums or signatures. They are not publisher hashes. + "windows/x64": { + "release": "20200822", + "asset": "cpython-3.7.9-x86_64-pc-windows-msvc-shared-pgo-20200823T0118.tar.zst", + "sha256": "8769a244cfe54c32c5253b78897a0fe82fe419dfde653b1afc3f5f20594cca89", + "asset_id": 24195911, + "size": 31429615, + "required_paths": ["Lib/site.py", "Lib/venv/__init__.py", "include/Python.h", "libs/python37.lib"], + }, + "macos/x64": { + # The 20200822 build has an unwanted libintl.dylib dependency. + "release": "20200823", + "asset": "cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T2228.tar.zst", + "sha256": "53657e7712cc7b24491fb1fc66dcc8f47a577fc77df137178746987ba4c5afb8", + "asset_id": 24211674, + "size": 26136896, + "required_paths": ["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.dylib"], + }, + "linux/x64": { + "release": "20200822", + "asset": "cpython-3.7.9-x86_64-unknown-linux-gnu-pgo-20200823T0036.tar.zst", + "sha256": "c6d6256d13e929e77e7ee6e53470fe63ad19d173fee6d56bb1b2dbda67081543", + "asset_id": 24195897, + "size": 28357634, + "required_paths": ["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.so.1.0"], + }, } _PYPY27_VERSION = "2.7.18" @@ -102,12 +126,18 @@ def _pbs_triple(ctx): def _platform_key(ctx): return "{}/{}".format(ctx.platform.os, ctx.platform.arch) -def _is_legacy_37(version, build_tag): - return version == "3.7.9" and build_tag == "20200822" - def _is_pypy27(version): return version == _PYPY27_VERSION +def _legacy37_artifact(ctx, version): + if version != "3.7.9": + fail("Python {} has no pinned portable build; request Python 3.7.9 explicitly".format(version)) + platform = _platform_key(ctx) + artifact = _LEGACY_37_ASSETS.get(platform) + if artifact == None: + fail("Python 3.7.9 portable build unavailable for {}; supported platforms: Windows x64, Linux x64, macOS x64".format(platform)) + return artifact + # --------------------------------------------------------------------------- # download_url — python-build-standalone asset # --------------------------------------------------------------------------- @@ -119,17 +149,16 @@ def download_url(ctx, version): return None return "https://downloads.python.org/pypy/{}".format(asset) + if version.startswith("3.7."): + artifact = _legacy37_artifact(ctx, version) + return github_asset_url("astral-sh", "python-build-standalone", artifact["release"], artifact["asset"]) + triple = _pbs_triple(ctx) if not triple: return None build_tag = ctx.version_date if not build_tag: return None - if _is_legacy_37(version, build_tag): - asset = _LEGACY_37_ASSETS.get(_platform_key(ctx)) - if not asset: - return None - return github_asset_url("astral-sh", "python-build-standalone", build_tag, asset) asset = "cpython-{}+{}-{}-install_only_stripped.tar.gz".format(version, build_tag, triple) return github_asset_url("astral-sh", "python-build-standalone", build_tag, asset) @@ -157,9 +186,8 @@ def install_layout(ctx, version): "executable_paths": exe_paths, } - if version == "3.7.9": - if _LEGACY_37_ASSETS.get(_platform_key(ctx)) == None: - return None + if version.startswith("3.7."): + artifact = _legacy37_artifact(ctx, version) if ctx.platform.os == "windows": exe_paths = ["python.exe"] else: @@ -168,6 +196,9 @@ def install_layout(ctx, version): "type": "archive", "strip_prefix": "python/install", "executable_paths": exe_paths, + "required_paths": artifact["required_paths"], + "sha256": artifact["sha256"], + "mirror_urls": [], } # The python-build-standalone tarball has a top-level "python/" directory. diff --git a/crates/vx-runtime-http/Cargo.toml b/crates/vx-runtime-http/Cargo.toml index bfb07cb72..c275b2ecf 100644 --- a/crates/vx-runtime-http/Cargo.toml +++ b/crates/vx-runtime-http/Cargo.toml @@ -11,6 +11,8 @@ repository.workspace = true async-trait = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } +sha2 = { workspace = true } +hex = "0.4" # Async runtime tokio = { workspace = true, features = ["rt", "fs", "io-util", "process"] } diff --git a/crates/vx-runtime-http/src/installer.rs b/crates/vx-runtime-http/src/installer.rs index 5c6d39070..758f6228b 100644 --- a/crates/vx-runtime-http/src/installer.rs +++ b/crates/vx-runtime-http/src/installer.rs @@ -72,7 +72,12 @@ impl RealInstaller { /// the GET response's Content-Disposition header or final redirected URL. /// For APIs like Adoptium that use redirect chains (307 → 302 → CDN), /// this saves ~3-5 seconds by eliminating the redundant HEAD round-trip. - async fn download_and_detect_filename(&self, url: &str, dest: &Path) -> Result> { + async fn download_and_detect_filename( + &self, + url: &str, + dest: &Path, + expected_sha256: Option<&str>, + ) -> Result> { use futures_util::StreamExt; use indicatif::{ProgressBar, ProgressStyle}; use tokio::io::AsyncWriteExt; @@ -97,6 +102,7 @@ impl RealInstaller { } std::fs::copy(&path, dest)?; pb.finish_and_clear(); + self.verify_download(url, dest, expected_sha256)?; tracing::debug!(url = url, cached_path = ?path, "Served from download cache"); // Return cached filename from metadata let cached_filename = if !metadata.filename.is_empty() @@ -125,6 +131,7 @@ impl RealInstaller { } std::fs::copy(&path, dest)?; pb.finish_and_clear(); + self.verify_download(url, dest, expected_sha256)?; tracing::debug!(url = url, cached_path = ?path, "Served from download cache (with ETag)"); return Ok(None); } @@ -268,6 +275,11 @@ impl RealInstaller { progress_bar.finish_and_clear(); file.flush().await?; + drop(file); + + // The catalog digest is authoritative for every transport, including + // mirrors and cache hits. Never admit an unverified payload to cache. + self.verify_download(url, dest, expected_sha256)?; // Store in download cache if enabled if let Some(cache) = &self.http.download_cache @@ -279,6 +291,36 @@ impl RealInstaller { Ok(detected_filename) } + fn verify_download(&self, url: &str, path: &Path, expected: Option<&str>) -> Result<()> { + use sha2::{Digest, Sha256}; + use std::io::Read; + + let Some(expected) = expected else { + return Ok(()); + }; + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = [0; 64 * 1024]; + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.update(&buffer[..count]); + } + let actual = hex::encode(hasher.finalize()); + if !actual.eq_ignore_ascii_case(expected) { + if let Some(cache) = &self.http.download_cache { + let _ = cache.remove(url); + } + anyhow::bail!( + "SHA256 mismatch for {url}: expected {expected}, got {actual}; refusing installation" + ); + } + tracing::info!(url, sha256 = %actual, "Verified artifact SHA256"); + Ok(()) + } + /// Parse filename from Content-Disposition header value fn parse_content_disposition(value: &str) -> Option { // Handle: attachment; filename=xxx.zip @@ -526,7 +568,7 @@ impl Installer for RealInstaller { } async fn download_and_extract(&self, url: &str, dest: &Path) -> Result<()> { - self.download_and_extract_impl(url, dest, false).await + self.download_and_extract_impl(url, dest, false, None).await } async fn download_with_layout( @@ -539,7 +581,13 @@ impl Installer for RealInstaller { // example, NSIS). Explicit archive extensions still take precedence. let skip_embedded_archive_detection = metadata.contains_key("target_name") && metadata.contains_key("target_dir"); - self.download_and_extract_impl(url, dest, skip_embedded_archive_detection) + let expected_sha256 = metadata.get("sha256").map(String::as_str); + if let Some(digest) = expected_sha256 + && (digest.len() != 64 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit())) + { + anyhow::bail!("Invalid artifact SHA256: expected exactly 64 hexadecimal characters"); + } + self.download_and_extract_impl(url, dest, skip_embedded_archive_detection, expected_sha256) .await?; // Debug: log metadata and dest contents @@ -749,6 +797,7 @@ impl RealInstaller { url: &str, dest: &Path, skip_embedded_archive_detection: bool, + expected_sha256: Option<&str>, ) -> Result<()> { // Create temp file for download let temp_dir = tempfile::tempdir()?; @@ -759,7 +808,11 @@ impl RealInstaller { // Download and detect filename in a single GET request (no separate HEAD). let temp_download_path = temp_dir.path().join("download_temp"); let detected_filename = self - .download_and_detect_filename(url_without_fragment, &temp_download_path) + .download_and_detect_filename( + url_without_fragment, + &temp_download_path, + expected_sha256, + ) .await?; let archive_name = detected_filename.unwrap_or_else(|| { diff --git a/crates/vx-runtime-http/tests/artifact_integrity_tests.rs b/crates/vx-runtime-http/tests/artifact_integrity_tests.rs new file mode 100644 index 000000000..b85239e2d --- /dev/null +++ b/crates/vx-runtime-http/tests/artifact_integrity_tests.rs @@ -0,0 +1,128 @@ +//! Fixed artifact digests apply to downloads and cached bytes before installation. + +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; +use std::time::Duration; + +use rstest::rstest; +use sha2::{Digest, Sha256}; +use vx_cache::{CacheLookupResult, DownloadCache}; +use vx_runtime::Installer; +use vx_runtime_http::RealInstaller; + +const PAYLOAD: &[u8] = b"fixed runtime artifact"; + +fn metadata() -> HashMap { + HashMap::from([("sha256".into(), hex::encode(Sha256::digest(PAYLOAD)))]) +} + +fn serve() -> (String, thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/runtime.bin", listener.local_addr().unwrap()); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let count = stream.read(&mut buffer).unwrap(); + assert!(count > 0); + request.extend_from_slice(&buffer[..count]); + } + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + PAYLOAD.len() + ) + .unwrap(); + stream.write_all(PAYLOAD).unwrap(); + }); + (url, server) +} + +#[tokio::test] +async fn verified_download_is_reusable_without_a_server() { + let temp = tempfile::tempdir().unwrap(); + let installer = RealInstaller::with_download_cache(temp.path().join("cache")); + let (url, server) = serve(); + let first = temp.path().join("first"); + installer + .download_with_layout(&url, &first, &metadata()) + .await + .unwrap(); + server.join().unwrap(); + let offline = temp.path().join("offline"); + installer + .download_with_layout(&url, &offline, &metadata()) + .await + .unwrap(); + assert_eq!( + std::fs::read(offline.join("bin/runtime.bin")).unwrap(), + PAYLOAD + ); +} + +#[tokio::test] +async fn wrong_digest_is_neither_installed_nor_cached() { + let temp = tempfile::tempdir().unwrap(); + let cache_dir = temp.path().join("cache"); + let installer = RealInstaller::with_download_cache(cache_dir.clone()); + let (url, server) = serve(); + let dest = temp.path().join("install"); + let digest = HashMap::from([("sha256".into(), "0".repeat(64))]); + let error = installer + .download_with_layout(&url, &dest, &digest) + .await + .unwrap_err(); + server.join().unwrap(); + assert!(error.to_string().contains("SHA256 mismatch")); + assert!(!dest.exists(), "unverified bytes must not be extracted"); + assert!(matches!( + DownloadCache::new(cache_dir).lookup(&url), + CacheLookupResult::Miss + )); +} + +#[tokio::test] +async fn same_size_cache_corruption_is_rejected_and_evicted() { + let temp = tempfile::tempdir().unwrap(); + let cache_dir = temp.path().join("cache"); + let cache = DownloadCache::new(cache_dir.clone()); + let url = "http://127.0.0.1:1/runtime.bin"; + let source = temp.path().join("source"); + std::fs::write(&source, PAYLOAD).unwrap(); + cache.store(url, &source, None, None, None).unwrap(); + let path = cache.file_path(&DownloadCache::cache_key(url)); + let mut corrupt = PAYLOAD.to_vec(); + corrupt[0] ^= 1; + std::fs::write(path, corrupt).unwrap(); + let dest = temp.path().join("install"); + let error = RealInstaller::with_download_cache(cache_dir) + .download_with_layout(url, &dest, &metadata()) + .await + .unwrap_err(); + assert!(error.to_string().contains("SHA256 mismatch")); + assert!(!dest.exists()); + assert!(matches!(cache.lookup(url), CacheLookupResult::Miss)); +} + +#[rstest] +#[case("")] +#[case("abc")] +#[case(&"g".repeat(64))] +#[tokio::test] +async fn malformed_digest_fails_before_network_access(#[case] digest: &str) { + let temp = tempfile::tempdir().unwrap(); + let dest = temp.path().join("install"); + let metadata = HashMap::from([("sha256".into(), digest.into())]); + let error = RealInstaller::new() + .download_with_layout("http://127.0.0.1:1/runtime.bin", &dest, &metadata) + .await + .unwrap_err(); + assert!(error.to_string().contains("Invalid artifact SHA256")); + assert!(!dest.exists()); +} diff --git a/crates/vx-runtime/src/manifest_runtime/install.rs b/crates/vx-runtime/src/manifest_runtime/install.rs index 73ec9ac52..70f6be477 100644 --- a/crates/vx-runtime/src/manifest_runtime/install.rs +++ b/crates/vx-runtime/src/manifest_runtime/install.rs @@ -50,9 +50,8 @@ impl ManifestDrivenRuntime { let _install_lock = InstallLock::acquire(&install_path).await?; // Try Starlark-driven install_layout first (provides URL + strip_prefix + exe paths) - if let Some(ref layout_fn) = self.install_layout_fn - && let Some(layout) = layout_fn(version.to_string()).await? - { + let layout_hint = self.resolve_layout_hint(version).await?; + if let Some(ref layout) = layout_hint { let url = layout .get("url") .and_then(|u| u.as_str()) @@ -65,9 +64,9 @@ impl ManifestDrivenRuntime { ); if ctx.fs.exists(&install_path) - && self.layout_executable_exists(&install_path, &layout, ctx) + && self.layout_executable_exists(&install_path, layout, ctx) { - let exe_path = self.resolve_exe_path_from_layout(&install_path, &layout); + let exe_path = self.resolve_exe_path_from_layout(&install_path, layout); return Ok(InstallResult::already_installed( install_path, exe_path, @@ -78,16 +77,10 @@ impl ManifestDrivenRuntime { ctx.fs.remove_dir_all(&install_path)?; } - let mut layout_meta = HashMap::new(); - if let Some(prefix) = layout.get("strip_prefix").and_then(|s| s.as_str()) { - layout_meta.insert("strip_prefix".to_string(), prefix.to_string()); - } - - ctx.installer - .download_with_layout(&url, &install_path, &layout_meta) + self.download_layout_sources(&url, &install_path, Some(layout), ctx) .await?; - let exe_path = self.resolve_exe_path_from_layout(&install_path, &layout); + let exe_path = self.resolve_exe_path_from_layout(&install_path, layout); return Ok(InstallResult::success( install_path, exe_path, @@ -99,7 +92,13 @@ impl ManifestDrivenRuntime { // Try direct download URL if let Some(url) = self.download_url(version, &platform).await? { return self - .install_via_direct_download(version, &url, &install_path, ctx) + .install_via_direct_download( + version, + &url, + &install_path, + layout_hint.as_ref(), + ctx, + ) .await; } @@ -114,19 +113,16 @@ impl ManifestDrivenRuntime { version: &str, url: &str, install_path: &std::path::Path, + layout_hint: Option<&serde_json::Value>, ctx: &RuntimeContext, ) -> Result { info!("Installing {} via direct download from {}", self.name, url); - // Resolve install_layout once for strip_prefix / executable_paths hints - let layout_hint = self.resolve_layout_hint(version).await; - if ctx.fs.exists(install_path) && layout_hint - .as_ref() .is_none_or(|layout| self.layout_executable_exists(install_path, layout, ctx)) { - let exe_path = if let Some(ref layout) = layout_hint { + let exe_path = if let Some(layout) = layout_hint { self.resolve_exe_path_from_layout(install_path, layout) } else { install_path.join(vx_paths::with_executable_extension(&self.executable)) @@ -141,15 +137,10 @@ impl ManifestDrivenRuntime { ctx.fs.remove_dir_all(install_path)?; } - // Build layout metadata for download_with_layout - let layout_meta = build_layout_meta(layout_hint.as_ref()); - debug!("layout_meta for download_with_layout: {:?}", layout_meta); - - ctx.installer - .download_with_layout(url, install_path, &layout_meta) + self.download_layout_sources(url, install_path, layout_hint, ctx) .await?; - let exe_path = if let Some(ref layout) = layout_hint { + let exe_path = if let Some(layout) = layout_hint { self.resolve_exe_path_from_layout(install_path, layout) } else { install_path.join(vx_paths::with_executable_extension(&self.executable)) @@ -163,24 +154,77 @@ impl ManifestDrivenRuntime { /// Resolve the install_layout hint (strip_prefix, executable_paths) for a version. /// - /// Calls `install_layout_fn` once and caches the result. Returns `None` if + /// Calls `install_layout_fn` once. Returns `None` if /// no layout function is set or the function returns `None`. - async fn resolve_layout_hint(&self, version: &str) -> Option { - let layout_fn = self.install_layout_fn.as_ref()?; - match layout_fn(version.to_string()).await { - Ok(Some(layout)) => { - debug!("install_layout_fn returned: {:?}", layout); - Some(layout) + async fn resolve_layout_hint(&self, version: &str) -> Result> { + let Some(layout_fn) = self.install_layout_fn.as_ref() else { + return Ok(None); + }; + // A malformed integrity contract must never degrade to an unchecked download. + layout_fn(version.to_string()).await + } + + async fn download_layout_sources( + &self, + origin: &str, + install_path: &std::path::Path, + layout: Option<&serde_json::Value>, + ctx: &RuntimeContext, + ) -> Result<()> { + let metadata = build_layout_meta(layout)?; + let mut sources = Vec::new(); + if let Some(mirrors) = layout.and_then(|value| value.get("mirror_urls")) { + let mirrors = mirrors + .as_array() + .ok_or_else(|| anyhow::anyhow!("mirror_urls must be a list"))?; + if !mirrors.is_empty() && !metadata.contains_key("sha256") { + anyhow::bail!("Artifact mirrors require a fixed SHA256 digest"); } - Ok(None) => { - debug!("install_layout_fn returned None"); - None + for mirror in mirrors { + let mirror = mirror + .as_str() + .ok_or_else(|| anyhow::anyhow!("mirror_urls must contain strings"))?; + if !sources.iter().any(|source| source == mirror) { + sources.push(mirror.to_string()); + } } - Err(e) => { - warn!("install_layout_fn failed: {}", e); - None + } + if !sources.iter().any(|source| source == origin) { + sources.push(origin.to_string()); + } + let mut last_error = None; + for source in sources { + let result = ctx + .installer + .download_with_layout(&source, install_path, &metadata) + .await.and_then(|()| { + if let Some(digest) = metadata.get("sha256") { + let layout = layout.expect("a digest originates in a layout"); + if !self.layout_files_exist(install_path, layout, ctx) { + anyhow::bail!("Verified artifact is missing its executable or required build files"); + } + // Readers hold this same store lock. An interrupted write + // leaves an invalid receipt and will force a fresh install. + ctx.fs.write(&install_path.join(".vx-artifact.json"), &serde_json::json!({ + "schema_version": 1, + "sha256": digest, + "source": source, + }).to_string())?; + } + Ok(()) + }); + match result { + Ok(()) => return Ok(()), + Err(error) => { + warn!(%source, %error, "Artifact source failed; retaining the fixed digest for fallback"); + if ctx.fs.exists(install_path) { + ctx.fs.remove_dir_all(install_path)?; + } + last_error = Some(error); + } } } + Err(last_error.expect("the origin always supplies one source")) } /// Install via system package managers and script strategies. @@ -312,10 +356,10 @@ impl ManifestDrivenRuntime { } /// Build layout metadata HashMap from an optional Starlark layout descriptor. -fn build_layout_meta(layout: Option<&serde_json::Value>) -> HashMap { +fn build_layout_meta(layout: Option<&serde_json::Value>) -> Result> { let mut meta = HashMap::new(); let Some(layout) = layout else { - return meta; + return Ok(meta); }; if let Some(prefix) = layout.get("strip_prefix").and_then(|s| s.as_str()) { @@ -331,5 +375,14 @@ fn build_layout_meta(layout: Option<&serde_json::Value>) -> HashMap bool { + if !self.layout_files_exist(install_dir, layout, ctx) { + return false; + } + let Some(digest) = layout.get("sha256") else { + return true; + }; + let Some(digest) = digest.as_str() else { + return false; + }; + let Ok(receipt) = ctx + .fs + .read_to_string(&install_dir.join(".vx-artifact.json")) + else { + return false; + }; + let Ok(receipt) = serde_json::from_str::(&receipt) else { + return false; + }; + receipt + .get("schema_version") + .and_then(|value| value.as_u64()) + == Some(1) + && receipt + .get("sha256") + .and_then(|value| value.as_str()) + .is_some_and(|actual| actual.eq_ignore_ascii_case(digest)) + } + + pub(crate) fn layout_files_exist( + &self, + install_dir: &std::path::Path, + layout: &serde_json::Value, + ctx: &RuntimeContext, ) -> bool { ctx.fs .exists(&self.resolve_exe_path_from_layout(install_dir, layout)) diff --git a/crates/vx-runtime/src/traits.rs b/crates/vx-runtime/src/traits.rs index f61240bb9..a9969e482 100644 --- a/crates/vx-runtime/src/traits.rs +++ b/crates/vx-runtime/src/traits.rs @@ -221,6 +221,8 @@ pub trait Installer: Send + Sync { /// Download and install with layout configuration (RFC 0019) /// /// This method accepts layout metadata to handle file renaming, moving, and permissions. + /// Implementations accepting `sha256` must verify downloaded and cached bytes + /// before extraction. The default implementation rejects that contract. /// If not implemented, falls back to `download_and_extract`. async fn download_with_layout( &self, @@ -228,6 +230,9 @@ pub trait Installer: Send + Sync { dest: &Path, metadata: &std::collections::HashMap, ) -> Result<()> { + if metadata.contains_key("sha256") { + anyhow::bail!("This installer does not support fixed artifact SHA256 verification"); + } // Default implementation - use metadata for post-processing self.download_and_extract(url, dest).await?; diff --git a/crates/vx-runtime/tests/manifest_install_lock_tests.rs b/crates/vx-runtime/tests/manifest_install_lock_tests.rs index f4a5b7d7f..ea0adb5f3 100644 --- a/crates/vx-runtime/tests/manifest_install_lock_tests.rs +++ b/crates/vx-runtime/tests/manifest_install_lock_tests.rs @@ -1,6 +1,8 @@ //! Manifest installations and completeness readers share the physical store lock. +use std::collections::HashMap; use std::path::Path; +use std::sync::Mutex; use std::sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -123,6 +125,163 @@ fn runtime(inline_url: bool) -> ManifestDrivenRuntime { .with_install_layout(move |_| Box::pin(async move { Ok(Some(layout(inline_url))) })) } +type RecordedDownload = (String, HashMap); + +#[derive(Default)] +struct RecordingInstaller { + calls: Mutex>, +} + +#[async_trait] +impl Installer for RecordingInstaller { + async fn extract(&self, _archive: &Path, _dest: &Path) -> Result<()> { + bail!("only layout downloads are allowed") + } + + async fn download_and_extract(&self, _url: &str, _dest: &Path) -> Result<()> { + bail!("the integrity contract must be retained") + } + + async fn download_with_layout( + &self, + url: &str, + dest: &Path, + metadata: &HashMap, + ) -> Result<()> { + self.calls + .lock() + .unwrap() + .push((url.to_string(), metadata.clone())); + std::fs::create_dir_all(dest.join("bin"))?; + if url.contains("mirror.invalid") { + std::fs::write(dest.join("partial"), b"partial extraction")?; + bail!("mirror unavailable") + } + assert!(!dest.join("partial").exists(), "fallback must start clean"); + std::fs::write(dest.join("bin").join(executable()), b"verified sdk")?; + Ok(()) + } +} + +#[rstest] +#[case::inline_layout_url(true)] +#[case::separate_download_url(false)] +#[tokio::test] +async fn mirror_fallback_preserves_the_fixed_digest(#[case] inline_url: bool) { + let temp = tempfile::tempdir().unwrap(); + let installer = Arc::new(RecordingInstaller::default()); + let ctx = RuntimeContext::new( + Arc::new(RealPathProvider::with_base_dir(temp.path())), + Arc::new(NoopHttpClient), + Arc::new(RealFileSystem::new()), + installer.clone(), + ); + let runtime = runtime(inline_url).with_install_layout(move |_| { + Box::pin(async move { + let mut layout = layout(inline_url); + layout["sha256"] = "e".repeat(64).into(); + layout["mirror_urls"] = serde_json::json!(["https://mirror.invalid/sdk.zip"]); + Ok(Some(layout)) + }) + }); + runtime.install(VERSION, &ctx).await.unwrap(); + let calls = installer.calls.lock().unwrap(); + assert_eq!(calls.len(), 2); + assert_eq!(calls[0].0, "https://mirror.invalid/sdk.zip"); + assert_eq!(calls[1].0, "https://example.invalid/sdk.zip"); + assert_eq!(calls[0].1, calls[1].1); + assert_eq!(calls[0].1["sha256"], "e".repeat(64)); +} + +#[rstest] +#[case::layout_error(None)] +#[case::wrong_digest_type(Some(serde_json::json!({"sha256": 42})))] +#[case::mirror_without_digest(Some(serde_json::json!({"mirror_urls": ["https://mirror.invalid/sdk.zip"]})))] +#[tokio::test] +async fn invalid_integrity_contract_never_falls_back_to_an_unchecked_download( + #[case] invalid: Option, +) { + let temp = tempfile::tempdir().unwrap(); + let installer = Arc::new(RecordingInstaller::default()); + let ctx = RuntimeContext::new( + Arc::new(RealPathProvider::with_base_dir(temp.path())), + Arc::new(NoopHttpClient), + Arc::new(RealFileSystem::new()), + installer.clone(), + ); + let runtime = runtime(false).with_install_layout(move |_| { + let invalid = invalid.clone(); + Box::pin(async move { + match invalid { + Some(invalid) => Ok(Some(invalid)), + None => bail!("malformed provider layout"), + } + }) + }); + assert!(runtime.install(VERSION, &ctx).await.is_err()); + assert!(installer.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn verified_layout_replaces_an_existing_install_without_a_matching_receipt() { + let temp = tempfile::tempdir().unwrap(); + let installer = Arc::new(RecordingInstaller::default()); + let ctx = RuntimeContext::new( + Arc::new(RealPathProvider::with_base_dir(temp.path())), + Arc::new(NoopHttpClient), + Arc::new(RealFileSystem::new()), + installer.clone(), + ); + let runtime = runtime(false).with_install_layout(|_| { + Box::pin(async { + let mut layout = layout(false); + layout["sha256"] = "e".repeat(64).into(); + Ok(Some(layout)) + }) + }); + let install_dir = ctx.paths.version_store_dir("sdk", VERSION); + std::fs::create_dir_all(install_dir.join("bin")).unwrap(); + std::fs::write( + install_dir.join("bin").join(executable()), + b"unverified sdk", + ) + .unwrap(); + assert!(!runtime.is_installed(VERSION, &ctx).await.unwrap()); + runtime.install(VERSION, &ctx).await.unwrap(); + assert!(runtime.is_installed(VERSION, &ctx).await.unwrap()); + assert_eq!(installer.calls.lock().unwrap().len(), 1); + // A receipt from another catalog digest cannot validate this installation. + std::fs::write( + install_dir.join(".vx-artifact.json"), + r#"{"schema_version":1,"sha256":"old"}"#, + ) + .unwrap(); + assert!(!runtime.is_installed(VERSION, &ctx).await.unwrap()); +} + +#[tokio::test] +async fn incomplete_verified_layout_never_gets_a_completion_receipt() { + let temp = tempfile::tempdir().unwrap(); + let installer = Arc::new(RecordingInstaller::default()); + let ctx = RuntimeContext::new( + Arc::new(RealPathProvider::with_base_dir(temp.path())), + Arc::new(NoopHttpClient), + Arc::new(RealFileSystem::new()), + installer, + ); + let runtime = runtime(false).with_install_layout(|_| { + Box::pin(async { + let mut layout = layout(false); + layout["sha256"] = "e".repeat(64).into(); + layout["required_paths"] = serde_json::json!(["include/Python.h"]); + Ok(Some(layout)) + }) + }); + assert!(runtime.install(VERSION, &ctx).await.is_err()); + let install_dir = ctx.paths.version_store_dir("sdk", VERSION); + assert!(!install_dir.exists()); +} + #[rstest] #[case::layout_url_before_executable(true, false)] #[case::layout_url_after_executable(true, true)] diff --git a/crates/vx-starlark/src/provider/execute.rs b/crates/vx-starlark/src/provider/execute.rs index 4ba49bf04..85aaff2e1 100644 --- a/crates/vx-starlark/src/provider/execute.rs +++ b/crates/vx-starlark/src/provider/execute.rs @@ -8,6 +8,21 @@ use tracing::debug; use super::StarlarkProvider; use super::types::{EnvOp, InstallLayout, PostExtractAction, PreRunAction, VersionInfoResult}; +/// Parse declared archive fields without silently discarding invalid types. +fn archive_layout_field( + descriptor: &serde_json::Value, + field: &str, +) -> Result { + match descriptor.get(field) { + None => Ok(T::default()), + Some(value) => serde_json::from_value(value.clone()).map_err(|error| { + Error::EvalError(format!( + "Invalid archive descriptor field '{field}': {error}" + )) + }), + } +} + impl StarlarkProvider { pub(super) async fn execute_install( &self, @@ -284,38 +299,24 @@ impl StarlarkProvider { "archive_install" | "archive" => { // "archive" is the legacy format (no URL, just layout hints) // "archive_install" includes the URL for complete install info - let url = json - .get("url") - .and_then(|u| u.as_str()) - .map(|s| s.to_string()); - let strip_prefix = json - .get("strip_prefix") - .and_then(|s| s.as_str()) - .map(|s| s.to_string()); - let executable_paths = json - .get("executable_paths") - .and_then(|p| p.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(); - let required_paths = json - .get("required_paths") - .and_then(|p| p.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(); + let url = archive_layout_field(&json, "url")?; + let strip_prefix = archive_layout_field(&json, "strip_prefix")?; + let executable_paths = archive_layout_field(&json, "executable_paths")?; + let required_paths = archive_layout_field(&json, "required_paths")?; + let sha256 = if json.get("sha256").is_some() { + Some(archive_layout_field::(&json, "sha256")?) + } else { + None + }; + let mirror_urls = archive_layout_field(&json, "mirror_urls")?; debug!(provider = %self.meta.name, url = ?url, strip_prefix = ?strip_prefix, "Resolved archive_install/archive descriptor"); Ok(Some(InstallLayout::Archive { url, strip_prefix, executable_paths, required_paths, + sha256, + mirror_urls, })) } "binary_install" | "binary" => { diff --git a/crates/vx-starlark/src/provider/types.rs b/crates/vx-starlark/src/provider/types.rs index fa3cb1e6a..1e2353957 100644 --- a/crates/vx-starlark/src/provider/types.rs +++ b/crates/vx-starlark/src/provider/types.rs @@ -26,6 +26,12 @@ pub enum InstallLayout { strip_prefix: Option, executable_paths: Vec, required_paths: Vec, + /// Expected SHA-256 of the complete archive before extraction. + #[serde(default)] + sha256: Option, + /// Additional download sources for the same verified archive bytes. + #[serde(default)] + mirror_urls: Vec, }, /// Single binary installation Binary { @@ -59,6 +65,8 @@ impl InstallLayout { strip_prefix, executable_paths, required_paths, + sha256, + mirror_urls, } => { let mut map = serde_json::Map::new(); if let Some(u) = url { @@ -67,6 +75,12 @@ impl InstallLayout { if let Some(sp) = strip_prefix { map.insert("strip_prefix".into(), serde_json::Value::String(sp)); } + if let Some(digest) = sha256 { + map.insert("sha256".into(), serde_json::Value::String(digest)); + } + if !mirror_urls.is_empty() { + map.insert("mirror_urls".into(), serde_json::json!(mirror_urls)); + } map.insert( "executable_paths".into(), serde_json::Value::Array( diff --git a/crates/vx-starlark/tests/archive_integrity_tests.rs b/crates/vx-starlark/tests/archive_integrity_tests.rs new file mode 100644 index 000000000..5be79b053 --- /dev/null +++ b/crates/vx-starlark/tests/archive_integrity_tests.rs @@ -0,0 +1,81 @@ +//! Archive integrity and mirror descriptors must survive the typed bridge. + +use rstest::rstest; +use vx_starlark::StarlarkProvider; +use vx_starlark::provider::InstallLayout; + +async fn provider_with_layout(fields: &str) -> StarlarkProvider { + let content = format!( + r#" +name = "archive-integrity" +description = "Archive descriptor contract" +runtimes = [{{"name": "archive-integrity", "executable": "example"}}] +def install_layout(_ctx, _version): + return {{"type": "archive", {fields}}} +"# + ); + StarlarkProvider::from_content("archive-integrity", &content) + .await + .unwrap() +} + +#[tokio::test] +async fn integrity_fields_survive_typed_layout_and_flattening() { + let digest = "12".repeat(32); + let provider = provider_with_layout(&format!( + r#""sha256": "{digest}", "mirror_urls": ["https://mirror.example/python.tar.zst"]"# + )) + .await; + let layout = provider.install_layout("3.7.9").await.unwrap().unwrap(); + let flat = layout.to_flat_json(); + assert_eq!(flat["sha256"], digest); + assert_eq!( + flat["mirror_urls"], + serde_json::json!(["https://mirror.example/python.tar.zst"]) + ); +} + +#[rstest] +#[case("sha256", "123")] +#[case("sha256", "[]")] +#[case("sha256", "None")] +#[case("mirror_urls", "\"https://mirror.example/python.tar.zst\"")] +#[case("mirror_urls", "[\"https://mirror.example/python.tar.zst\", 123]")] +#[case("mirror_urls", "None")] +#[case("url", "123")] +#[case("strip_prefix", "[]")] +#[case("executable_paths", "[\"python.exe\", None]")] +#[case("required_paths", "{\"path\": \"python.exe\"}")] +#[tokio::test] +async fn incorrectly_typed_archive_fields_fail(#[case] field: &str, #[case] value: &str) { + let provider = provider_with_layout(&format!("\"{field}\": {value}")).await; + let error = provider.install_layout("3.7.9").await.unwrap_err(); + assert!( + error.to_string().contains(&format!("field '{field}'")), + "unexpected diagnostic: {error}" + ); +} + +#[test] +fn previously_serialized_archive_layouts_default_integrity_fields() { + let layout: InstallLayout = serde_json::from_value(serde_json::json!({ + "Archive": { + "url": null, + "strip_prefix": null, + "executable_paths": [], + "required_paths": [] + } + })) + .unwrap(); + match layout { + InstallLayout::Archive { + sha256, + mirror_urls, + .. + } => { + assert!(sha256.is_none()); + assert!(mirror_urls.is_empty()); + } + other => panic!("unexpected layout: {other:?}"), + } +} diff --git a/crates/vx-starlark/tests/python_legacy_tests.rs b/crates/vx-starlark/tests/python_legacy_tests.rs index 851db4f7e..72093761e 100644 --- a/crates/vx-starlark/tests/python_legacy_tests.rs +++ b/crates/vx-starlark/tests/python_legacy_tests.rs @@ -5,6 +5,7 @@ //! Python 2.7 compatibility is provided via PyPy2.7 portable archives because //! python-build-standalone does not publish CPython 2.7 artifacts. +use rstest::rstest; use vx_starlark::StarlarkEngine; fn load_provider_content(provider_name: &str) -> (std::path::PathBuf, String) { @@ -45,7 +46,7 @@ async fn test_python37_download_url_windows_uses_legacy_asset() { } #[tokio::test] -async fn test_python37_download_url_unsupported_arm64_returns_none() { +async fn test_python37_download_url_unsupported_arm64_reports_error() { let (star_path, content) = load_provider_content("python"); let engine = StarlarkEngine::new(); let mut ctx = vx_starlark::ProviderContext::new("python", std::env::temp_dir().join("vx-test")); @@ -54,6 +55,55 @@ async fn test_python37_download_url_unsupported_arm64_returns_none() { ctx.version_date = Some("20200822".to_string()); let result = engine + .call_function( + &star_path, + &content, + "download_url", + &ctx, + &[serde_json::json!("3.7.9")], + ) + .unwrap_err(); + + assert!( + result + .to_string() + .contains("portable build unavailable for linux/arm64") + ); +} + +#[rstest] +#[case( + "windows", + "8769a244cfe54c32c5253b78897a0fe82fe419dfde653b1afc3f5f20594cca89", + "20200822", + "20200823T0118" +)] +#[case( + "linux", + "c6d6256d13e929e77e7ee6e53470fe63ad19d173fee6d56bb1b2dbda67081543", + "20200822", + "20200823T0036" +)] +#[case( + "macos", + "53657e7712cc7b24491fb1fc66dcc8f47a577fc77df137178746987ba4c5afb8", + "20200823", + "20200823T2228" +)] +fn test_python37_pinned_catalog_without_release_discovery( + #[case] os: &str, + #[case] digest: &str, + #[case] release: &str, + #[case] timestamp: &str, +) { + let (star_path, content) = load_provider_content("python"); + let engine = StarlarkEngine::new(); + let mut ctx = vx_starlark::ProviderContext::new("python", std::env::temp_dir().join("vx-test")); + ctx.platform.os = os.to_string(); + ctx.platform.arch = "x64".to_string(); + assert!(ctx.version_date.is_none()); + + let url = engine .call_function( &star_path, &content, @@ -62,8 +112,71 @@ async fn test_python37_download_url_unsupported_arm64_returns_none() { &[serde_json::json!("3.7.9")], ) .unwrap(); + let url = url.as_str().unwrap(); + assert!(url.contains(&format!("/download/{release}/")), "{url}"); + assert!(url.contains(timestamp), "{url}"); + + let layout = engine + .call_function( + &star_path, + &content, + "install_layout", + &ctx, + &[serde_json::json!("3.7.9")], + ) + .unwrap(); + assert_eq!(layout["sha256"], digest); + assert_eq!(layout["mirror_urls"], serde_json::json!([])); +} - assert!(result.is_null()); +#[rstest] +#[case("windows", vec!["Lib/site.py", "Lib/venv/__init__.py", "include/Python.h", "libs/python37.lib"])] +#[case("linux", vec!["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.so.1.0"])] +#[case("macos", vec!["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.dylib"])] +fn test_python37_completeness_requires_full_environment_and_build_support( + #[case] os: &str, + #[case] required_paths: Vec<&str>, +) { + let (star_path, content) = load_provider_content("python"); + let engine = StarlarkEngine::new(); + let mut ctx = vx_starlark::ProviderContext::new("python", std::env::temp_dir().join("vx-test")); + ctx.platform.os = os.to_string(); + ctx.platform.arch = "x64".to_string(); + let layout = engine + .call_function( + &star_path, + &content, + "install_layout", + &ctx, + &[serde_json::json!("3.7.9")], + ) + .unwrap(); + assert_eq!(layout["required_paths"], serde_json::json!(required_paths)); +} + +#[rstest] +#[case("download_url")] +#[case("install_layout")] +fn test_python37_other_patch_versions_are_not_substituted(#[case] function: &str) { + let (star_path, content) = load_provider_content("python"); + let engine = StarlarkEngine::new(); + let mut ctx = vx_starlark::ProviderContext::new("python", std::env::temp_dir().join("vx-test")); + ctx.platform.os = "windows".to_string(); + ctx.platform.arch = "x64".to_string(); + let error = engine + .call_function( + &star_path, + &content, + function, + &ctx, + &[serde_json::json!("3.7.17")], + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("Python 3.7.17 has no pinned portable build") + ); } #[tokio::test] diff --git a/docs/guide/provider-star-layouts.md b/docs/guide/provider-star-layouts.md index c2864ecef..deb6fc677 100644 --- a/docs/guide/provider-star-layouts.md +++ b/docs/guide/provider-star-layouts.md @@ -20,7 +20,7 @@ The `install_layout()` function returns a descriptor dict. The `__type` (or `typ | Type | Required Fields | Optional Fields | Use Case | |------|----------------|-----------------|----------| -| `"archive"` | `type` | `strip_prefix`, `executable_paths` | tar.gz, zip archives | +| `"archive"` | `type` | `strip_prefix`, `executable_paths`, `required_paths`, `sha256`, `mirror_urls` | tar.gz, zip archives | | `"binary"` | `type` | `executable_name`, `permissions` | Direct executable download | | `"msi"` | `type`, `url` | `executable_paths`, `strip_prefix`, `extra_args` | Windows MSI installer | | `"system_find"` | `type`, `executable` | `system_paths`, `hint` | System-installed tool lookup | @@ -36,6 +36,18 @@ def install_layout(ctx, version): } ``` +For a retained runtime distribution, set `sha256` to the fixed SHA256 of the +complete archive and `required_paths` to the files needed for a usable runtime +and its supported build workflows. `mirror_urls` is an ordered list of sources +for identical archive bytes; the `download_url()` origin is tried last. Mirrors +require a fixed digest, and every source and cache hit is verified before +extraction. A failed source cannot relax the digest for the next source. + +After extraction, VX checks the executable and required files before writing +`.vx-artifact.json` with the verified digest. An old installation without a +matching completion receipt is reacquired. This records archive verification; +it does not rehash every installed file on each execution. + ### Binary Layout ```python diff --git a/docs/tools/python.md b/docs/tools/python.md index 89acd76ca..ed134d1c2 100644 --- a/docs/tools/python.md +++ b/docs/tools/python.md @@ -24,7 +24,7 @@ vx uses [python-build-standalone](https://github.com/astral-sh/python-build-stan | Python 3.10 | Active | Stable | | Python 3.9 | EOL | Last build: 20251120 | | Python 3.8 | EOL | Limited availability | -| Python 3.7 | EOL | Legacy support only | +| Python 3.7 | EOL | Pinned CPython 3.7.9 on Windows, Linux and macOS x64 | | Python 2.7 | EOL | Legacy compatibility via PyPy2.7 | > **Note**: Python versions that have reached End-of-Life (EOL) may have limited availability. Python 2.7 is intended for legacy test and migration workflows; CPython-specific native extensions may require a system CPython 2.7 installation. @@ -92,6 +92,40 @@ vx uv venv .venv27 --python 2.7 When `uv` receives a simple version through `vx uv ... --python `, vx resolves that version with the vx Python provider first and passes the installed interpreter path to uv. Python 2.7 is a special legacy case: uv itself requires Python 3.6+, so `vx uv venv ... --python 2.7` creates the environment with PyPA's Python 2.7 `virtualenv.pyz` while preserving the same vx command shape. +### Python 3.7 lifecycle contract + +uv still lists Python 3.7 as [Tier 2 supported](https://docs.astral.sh/uv/reference/policies/python/). Its [0.7.0 release removed Python 3.7 from the managed download catalog](https://github.com/astral-sh/uv/pull/13022); interpreter support and downloadable distributions are separate capabilities. A current uv can use an existing CPython 3.7 interpreter even though `uv python install 3.7` reports that no download is available. + +VX owns acquisition for this line. The Python provider pins CPython **3.7.9** release assets, archive SHA-256 values and platform identity independently of the current upstream release list. Supported targets are Windows x64, Linux x64 with glibc, and macOS x64. Unsupported targets and other 3.7 patch versions produce an explicit error. VX does not substitute PyPy or another Python minor version. The macOS asset comes from [20200823](https://github.com/astral-sh/python-build-standalone/releases/tag/20200823), which fixes the unwanted `libintl.dylib` dependency in 20200822. + +Completeness checks require the standard library, `venv`, Python headers and link libraries, in addition to the interpreter executable. An embedded-only distribution or interrupted extraction does not satisfy this contract. + +The historical publisher supplied no archive checksums or signatures. These SHA-256 values are VX intake pins calculated from official HTTPS assets and recorded with their GitHub asset IDs and byte sizes. They detect changed downloads and cached archives; they are **not publisher-signed provenance**. A mirror must serve the identical pinned archive bytes. No public mirror is enabled by this catalog entry. + +```bash +vx install python@3.7.9 +vx python@3.7.9 -c "import sys, ssl, sqlite3; assert sys.version_info[:3] == (3, 7, 9); print(sys.executable)" +vx uv venv .venv37 --python 3.7.9 +vx uv pip install --python .venv37 "requests==2.31.0" +vx uv pip check --python .venv37 +``` + +Build dependencies have their own Python requirements. For a project that needs to execute its PEP 517 backend on 3.7, pin compatible backend dependencies in `pyproject.toml`, for example: + +```toml +[build-system] +requires = ["setuptools==67.8.0", "wheel==0.42.0"] +build-backend = "setuptools.build_meta" +``` + +```bash +vx uv build --python .venv37 --no-python-downloads --out-dir dist37 +``` + +Use `--offline` for uv operations after their package and build dependencies have been cached. Retain both the VX interpreter/archive cache and uv package cache for offline workflows. A successful download alone does not establish that a project's dependencies or native extension build supports Python 3.7. + +uv also exposes [a custom Python download catalog](https://docs.astral.sh/uv/reference/environment/#uv_python_downloads_json_url) and [Python archive mirrors](https://docs.astral.sh/uv/reference/environment/#uv_python_install_mirror). These can support managed distribution deployments without freezing an old uv executable; the normal `vx uv ... --python 3.7.9` workflow uses the interpreter acquired by VX. + ### Project Management ```bash diff --git a/docs/zh/tools/python.md b/docs/zh/tools/python.md index de5e71e9e..b56a4cf07 100644 --- a/docs/zh/tools/python.md +++ b/docs/zh/tools/python.md @@ -24,7 +24,7 @@ vx 使用 Astral 的 [python-build-standalone](https://github.com/astral-sh/pyth | Python 3.10 | 活跃 | 稳定版 | | Python 3.9 | 已终止 | 最后构建: 20251120 | | Python 3.8 | 已终止 | 有限可用性 | -| Python 3.7 | 已终止 | 仅遗留支持 | +| Python 3.7 | 已终止 | 固定 CPython 3.7.9,支持 Windows、Linux、macOS x64 | | Python 2.7 | 已终止 | 通过 PyPy2.7 提供遗留兼容 | > **注意**: 已达到生命周期终点 (EOL) 的 Python 版本可用性有限。Python 2.7 面向遗留测试和迁移场景;依赖 CPython 原生扩展的项目可能仍需要系统 CPython 2.7。 @@ -92,6 +92,40 @@ vx uv venv .venv27 --python 2.7 当 `uv` 通过 `vx uv ... --python ` 接收到简单版本号时,vx 会先用自己的 Python Provider 解析并安装该版本,再把解释器路径传给 uv。Python 2.7 是特殊遗留场景:uv 本身要求 Python 3.6+,因此 `vx uv venv ... --python 2.7` 会保持同样的 vx 命令形态,但底层使用 PyPA 的 Python 2.7 `virtualenv.pyz` 创建环境。 +### Python 3.7 生命周期契约 + +uv 当前仍将 Python 3.7 列为 [Tier 2 支持](https://docs.astral.sh/uv/reference/policies/python/)。[uv 0.7.0 移除了托管下载清单中的 Python 3.7](https://github.com/astral-sh/uv/pull/13022),解释器运行支持与可下载发行版属于不同能力。因此,当前 uv 可以使用已有 CPython 3.7,但 `uv python install 3.7` 会提示没有可下载版本。 + +这条版本线由 VX 负责获取。Python Provider 固定 CPython **3.7.9** 的发布版本、归档 SHA-256 和平台身份,不依赖上游当前发布清单。支持 Windows x64、使用 glibc 的 Linux x64、macOS x64;不支持的平台或其他 3.7 补丁版本会明确报错,不替换成 PyPy 或其他 Python 次版本。macOS 使用 [20200823 修复版](https://github.com/astral-sh/python-build-standalone/releases/tag/20200823),避免 20200822 对 `libintl.dylib` 的额外依赖。 + +完整性检查除了解释器可执行文件,还要求标准库、`venv`、Python 头文件和链接库。只有嵌入版解释器或未完成的解压目录不满足这个契约。 + +历史发布者没有提供归档校验值或签名。这里的 SHA-256 是 VX 从官方 HTTPS 资产接收时计算的固定值,同时记录 GitHub 资产 ID 和字节数,用于检测下载文件及缓存归档变化,**不代表发布者签名证明**。镜像必须提供完全相同的固定归档字节;此目录条目尚未启用公共镜像。 + +```bash +vx install python@3.7.9 +vx python@3.7.9 -c "import sys, ssl, sqlite3; assert sys.version_info[:3] == (3, 7, 9); print(sys.executable)" +vx uv venv .venv37 --python 3.7.9 +vx uv pip install --python .venv37 "requests==2.31.0" +vx uv pip check --python .venv37 +``` + +构建依赖有自己的 Python 版本要求。如果项目必须在 3.7 解释器上执行 PEP 517 后端,需要在 `pyproject.toml` 中固定兼容版本,例如: + +```toml +[build-system] +requires = ["setuptools==67.8.0", "wheel==0.42.0"] +build-backend = "setuptools.build_meta" +``` + +```bash +vx uv build --python .venv37 --no-python-downloads --out-dir dist37 +``` + +缓存过所需包和构建依赖后,可以给 uv 操作添加 `--offline`。离线工作流需要同时保留 VX 解释器及归档缓存与 uv 包缓存。下载成功本身不能证明某个项目的依赖或原生扩展支持 Python 3.7。 + +uv 还提供[自定义 Python 下载目录](https://docs.astral.sh/uv/reference/environment/#uv_python_downloads_json_url)和 [Python 归档镜像](https://docs.astral.sh/uv/reference/environment/#uv_python_install_mirror)能力,可以持续维护发行版部署而无需永久冻结旧版 uv。常规 `vx uv ... --python 3.7.9` 工作流使用 VX 获取的解释器。 + ### 项目管理 ```bash diff --git a/scripts/test_legacy_python.py b/scripts/test_legacy_python.py new file mode 100644 index 000000000..d7edfdb76 --- /dev/null +++ b/scripts/test_legacy_python.py @@ -0,0 +1,718 @@ +#!/usr/bin/env python3 +"""Accept a built VX against the fixed Python 3.7 lifecycle contract. + +Run with a modern harness interpreter, for example:: + + vx python@3.12 scripts/test_legacy_python.py --vx-binary target/debug/vx.exe + +Every run creates its own empty VX_HOME and UV_CACHE_DIR below --work-root. +Only its Python store directory is removed, for the archive-cache reinstall +gate. The run directory, command logs and receipt are retained for inspection. +This is a live acceptance check, separate from mocked unit tests. +""" + +from __future__ import annotations + +import argparse +import ast +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time +from typing import Any, Callable + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "tests" / "fixtures" / "legacy-python-build" +PYTHON_VERSION = "3.7.9" +UV_VERSION = "0.12.23" +# These last compatible backend releases are explicit so acceptance does not +# depend on a resolver selecting newer packages that dropped Python 3.7. +BUILD_REQUIREMENTS = ["setuptools==67.8.0", "wheel==0.41.3", "build==1.0.3"] +IDENTITY_CODE = """ +import json, ssl, sys, sysconfig +print(json.dumps({ + 'implementation': sys.implementation.name, + 'version': list(sys.version_info[:3]), + 'executable': sys.executable, + 'prefix': sys.prefix, + 'base_prefix': sys.base_prefix, + 'include': sysconfig.get_path('include'), + 'openssl': ssl.OPENSSL_VERSION, +})) +""" +WHEEL_CODE = """ +import json, sys +import vx_legacy_probe +print(json.dumps({ + 'version': list(sys.version_info[:3]), + 'prefix': sys.prefix, + 'base_prefix': sys.base_prefix, + 'module': vx_legacy_probe.__file__, + 'native_versions': vx_legacy_probe.versions(), + 'build': vx_legacy_probe.build_provenance(), +})) +""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def within(path: Path, directory: Path) -> bool: + return path.resolve().is_relative_to(directory.resolve()) + + +def platform_key() -> str: + operating_system = {"Windows": "windows", "Linux": "linux", "Darwin": "macos"} + architecture = { + "AMD64": "x64", + "x86_64": "x64", + "aarch64": "arm64", + "arm64": "arm64", + } + return "{}/{}".format( + operating_system.get(platform.system(), platform.system().lower()), + architecture.get(platform.machine(), platform.machine()), + ) + + +def artifact_contract(provider: Path) -> dict[str, Any]: + """Read the checked-in literal lifecycle catalog without executing the DSL.""" + tree = ast.parse(provider.read_text(encoding="utf-8"), filename=str(provider)) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "_LEGACY_37_ASSETS" + for target in node.targets + ): + catalog = ast.literal_eval(node.value) + artifact = dict(catalog[platform_key()]) + require( + re.fullmatch(r"[0-9a-fA-F]{64}", artifact.get("sha256") or "") + is not None, + "The lifecycle catalog must contain a trusted SHA256 before acceptance", + ) + artifact["sha256"] = artifact["sha256"].lower() + artifact["url"] = artifact.get("url") or ( + "https://github.com/astral-sh/python-build-standalone/releases/download/" + + artifact["release"] + + "/" + + artifact["asset"] + ) + return artifact + raise AssertionError("Python provider has no literal _LEGACY_37_ASSETS catalog") + + +def existing_compiler_environment( + environment: dict[str, str], +) -> tuple[dict[str, str], str | None]: + """Bind an already installed VS compiler when the invoking shell did not. + + vswhere and vcvars64 come from the existing Visual Studio installation. + This probe neither downloads a compiler nor changes persistent environment. + """ + if os.name != "nt" or shutil.which("cl", path=environment.get("PATH")): + return environment, None + installer = Path(environment.get("ProgramFiles(x86)", "C:/Program Files (x86)")) + vswhere = installer / "Microsoft Visual Studio" / "Installer" / "vswhere.exe" + if not vswhere.is_file(): + return environment, None + location = subprocess.run( + [ + str(vswhere), + "-latest", + "-products", + "*", + "-requires", + "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "-property", + "installationPath", + ], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + if location.returncode or not location.stdout.strip(): + return environment, None + vcvars = ( + Path(location.stdout.strip()) / "VC" / "Auxiliary" / "Build" / "vcvars64.bat" + ) + if not vcvars.is_file(): + return environment, None + # Pass cmd its original command line: list2cmdline escapes the batch file's + # nested quotes, which cmd interprets as a UNC path instead of a script. + command = '"{}" /d /s /c ""{}" >nul && set"'.format( + environment.get("COMSPEC", "cmd.exe"), vcvars + ) + bound = subprocess.run( + command, + env=environment, + capture_output=True, + text=True, + errors="replace", + timeout=60, + check=False, + ) + if bound.returncode: + raise AssertionError( + "Existing MSVC environment failed: " + bound.stderr[-1000:] + ) + result = dict(environment) + for line in bound.stdout.splitlines(): + if "=" in line and not line.startswith("="): + name, value = line.split("=", 1) + # Windows names are case-insensitive; avoid retaining both Path/PATH. + for previous in list(result): + if previous.casefold() == name.casefold(): + del result[previous] + result[name.upper() if name.casefold() == "path" else name] = value + result.update(DISTUTILS_USE_SDK="1", MSSdk="1") + return result, str(vcvars) + + +class Acceptance: + def __init__(self, arguments: argparse.Namespace) -> None: + self.vx = arguments.vx_binary.resolve(strict=True) + work_root = arguments.work_root.resolve() + work_root.mkdir(parents=True, exist_ok=True) + self.directory = Path(tempfile.mkdtemp(prefix="run-", dir=work_root)).resolve() + self.marker = self.directory / ".vx-legacy-acceptance-owner" + self.marker.write_text(str(self.directory), encoding="utf-8") + self.vx_home = self.directory / "vx-home" + self.uv_cache = self.directory / "uv-cache" + self.logs = self.directory / "logs" + self.logs.mkdir() + self.cwd = self.directory / "project" + self.cwd.mkdir() + self.uv = "uv@" + arguments.uv_version + self.timeout = arguments.command_timeout + self.receipt_path = ( + arguments.receipt or self.directory / "receipt.json" + ).resolve() + self.environment = dict(os.environ) + for name in list(self.environment): + if ( + name.startswith(("UV_", "VX_", "PYTHON", "PIP_")) + or name == "VIRTUAL_ENV" + ): + del self.environment[name] + self.environment.update( + { + "VX_HOME": str(self.vx_home), + "UV_CACHE_DIR": str(self.uv_cache), + "UV_PYTHON_DOWNLOADS": "never", + "UV_NO_CONFIG": "1", + "UV_LINK_MODE": "copy", + "PYTHONNOUSERSITE": "1", + "RUST_LOG": "vx_runtime_http=debug,vx_runtime=info", + "NO_COLOR": "1", + } + ) + self.environment, compiler_source = existing_compiler_environment( + self.environment + ) + self.provider = arguments.provider.resolve() + self.state: dict[str, Any] = {} + self.receipt: dict[str, Any] = { + "schema_version": 1, + "started_at": datetime.now(timezone.utc).isoformat(), + "platform": platform_key(), + "host": platform.platform(), + "python_request": PYTHON_VERSION, + "uv_request": arguments.uv_version, + "vx_binary": str(self.vx), + "vx_binary_sha256": sha256(self.vx), + "source_revision": getattr(arguments, "source_revision", None), + "run_directory": str(self.directory), + "vx_home": str(self.vx_home), + "uv_cache": str(self.uv_cache), + "compiler": { + "executable": shutil.which( + "cl" if os.name == "nt" else "cc", path=self.environment.get("PATH") + ), + "environment_source": compiler_source, + "CC": self.environment.get("CC"), + "DISTUTILS_USE_SDK": self.environment.get("DISTUTILS_USE_SDK"), + }, + "gates": [], + "commands": [], + } + + def save(self) -> None: + self.receipt_path.parent.mkdir(parents=True, exist_ok=True) + data = json.dumps(self.receipt, indent=2) + "\n" + self.receipt_path.write_text(data, encoding="utf-8") + local_receipt = self.directory / "receipt.json" + if local_receipt != self.receipt_path: + local_receipt.write_text(data, encoding="utf-8") + + def run(self, label: str, *arguments: str, offline: bool = False) -> str: + command = [str(self.vx), *arguments] + environment = dict(self.environment) + if offline: + environment.update( + { + "UV_OFFLINE": "1", + "HTTP_PROXY": "http://127.0.0.1:9", + "HTTPS_PROXY": "http://127.0.0.1:9", + "ALL_PROXY": "http://127.0.0.1:9", + "http_proxy": "http://127.0.0.1:9", + "https_proxy": "http://127.0.0.1:9", + "all_proxy": "http://127.0.0.1:9", + "NO_PROXY": "", + "no_proxy": "", + } + ) + log = self.logs / ("{:02d}-{}.log".format(len(self.receipt["commands"]), label)) + started = time.monotonic() + record: dict[str, Any] = {"argv": command, "log": str(log), "offline": offline} + self.receipt["commands"].append(record) + self.save() + try: + with log.open("w", encoding="utf-8") as stream: + completed = subprocess.run( + command, + cwd=self.cwd, + env=environment, + stdout=stream, + stderr=subprocess.STDOUT, + timeout=self.timeout, + check=False, + ) + record["returncode"] = completed.returncode + except subprocess.TimeoutExpired: + record["returncode"] = None + record["timed_out"] = True + output = log.read_text(encoding="utf-8", errors="replace") + record["duration_seconds"] = round(time.monotonic() - started, 3) + self.save() + require( + record["returncode"] == 0, + "{} failed; inspect {}\n{}".format(label, log, output[-2500:]), + ) + return output + + def gate( + self, + name: str, + action: Callable[[], dict[str, Any]], + needs: tuple[str, ...] = (), + ) -> None: + started = time.monotonic() + previous = {gate["name"]: gate["status"] for gate in self.receipt["gates"]} + missing = [ + dependency for dependency in needs if previous.get(dependency) != "passed" + ] + record: dict[str, Any] = {"name": name} + if missing: + record.update( + status="blocked", reason="Failed prerequisite: " + ", ".join(missing) + ) + else: + try: + record.update(status="passed", evidence=action()) + except (AssertionError, OSError, ValueError, KeyError) as error: + record.update(status="failed", reason=str(error)) + record["duration_seconds"] = round(time.monotonic() - started, 3) + self.receipt["gates"].append(record) + self.save() + print("{}: {}".format(name, record["status"]), flush=True) + + def python( + self, label: str, code: str, venv: Path | None = None, offline: bool = False + ) -> dict[str, Any]: + if venv is None: + arguments = ["python@" + PYTHON_VERSION, "-I", "-c", code] + else: + arguments = [ + self.uv, + "run", + "--no-project", + "--no-sync", + "--no-python-downloads", + "--python", + str(self.venv_python(venv)), + "python", + "-I", + "-c", + code, + ] + output = self.run(label, *arguments, offline=offline) + for line in reversed(output.splitlines()): + if line.startswith("{"): + return json.loads(line) + raise AssertionError("Python command did not emit JSON: " + output[-1000:]) + + @staticmethod + def venv_python(venv: Path) -> Path: + return venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + + def check_identity(self, data: dict[str, Any], venv: Path | None = None) -> None: + require( + data["implementation"] == "cpython", + "Expected CPython, got " + data["implementation"], + ) + require( + data["version"] == [3, 7, 9], + "Interpreter version differs from the explicit request", + ) + base = self.vx_home / "store" / "python" / PYTHON_VERSION + require( + within(Path(data["base_prefix"]), base), + "Base interpreter escaped the owned VX store", + ) + require( + Path(data["prefix"]).resolve() == (venv or base).resolve(), + "Incorrect Python prefix", + ) + # On Unix uv normally symlinks venv/bin/python to the managed base. + # sys.prefix binds the environment; sys.executable must name its entry point. + executable = Path(data["executable"]).absolute() + require( + executable.is_relative_to((venv or base).absolute()), + "Interpreter executable escaped its environment", + ) + + def contract(self) -> dict[str, Any]: + self.state["artifact"] = artifact_contract(self.provider) + return {"provider": str(self.provider), **self.state["artifact"]} + + def fresh_install(self) -> dict[str, Any]: + require( + not self.vx_home.exists() and not self.uv_cache.exists(), + "Acceptance must start with empty caches", + ) + self.run("vx-version", "--version") + output = self.run("fresh-python-install", "install", "python@" + PYTHON_VERSION) + expected = self.state["artifact"]["sha256"] + require( + "Verified artifact SHA256" in output and expected in output, + "VX did not attest the expected digest during fresh acquisition", + ) + return { + "empty_vx_home": True, + "empty_uv_cache": True, + "verified_sha256": expected, + } + + def archive_integrity(self) -> dict[str, Any]: + artifact = self.state["artifact"] + # DownloadCache::file_path uses the full URL SHA256 under its two-byte shard. + key = hashlib.sha256(artifact["url"].encode()).hexdigest() + archive = self.vx_home / "cache" / "downloads" / key[:2] / key + require( + archive.is_file(), "Verified origin archive missing from VX download cache" + ) + actual = sha256(archive) + require( + actual == artifact["sha256"], + "Cached bytes differ from the trusted lifecycle SHA256", + ) + self.state["archive"] = archive + return { + "archive": str(archive), + "size": archive.stat().st_size, + "sha256": actual, + } + + def identity(self) -> dict[str, Any]: + data = self.python("python-identity", IDENTITY_CODE) + self.check_identity(data) + require( + Path(data["include"]).joinpath("Python.h").is_file(), + "Python headers required for native builds are absent", + ) + return data + + def install_uv(self) -> dict[str, Any]: + output = self.run("uv-version", self.uv, "--version") + require("uv " + self.uv.split("@", 1)[1] in output, "Wrong uv version") + return {"version_output": output.strip()} + + def create_venv(self, offline: bool = False) -> dict[str, Any]: + venv = self.directory / ("venv-offline" if offline else "venv-online") + self.run( + "venv-offline" if offline else "venv-online", + self.uv, + "venv", + "--python", + PYTHON_VERSION, + "--no-python-downloads", + str(venv), + offline=offline, + ) + data = self.python( + "venv-identity-offline" if offline else "venv-identity", + IDENTITY_CODE, + venv, + offline, + ) + self.check_identity(data, venv) + self.state["venv-offline" if offline else "venv-online"] = venv + return data + + def packages(self, offline: bool = False) -> dict[str, Any]: + venv = self.state["venv-offline" if offline else "venv-online"] + self.run( + "packages-offline" if offline else "packages-online", + self.uv, + "pip", + "install", + "--python", + str(self.venv_python(venv)), + "--only-binary", + ":all:", + *BUILD_REQUIREMENTS, + offline=offline, + ) + versions = self.python( + "backend-versions-offline" if offline else "backend-versions", + """ +import json, pkg_resources +print(json.dumps({name: pkg_resources.get_distribution(name).version + for name in ['setuptools', 'wheel', 'build']})) +""", + venv, + offline, + ) + return {"requirements": BUILD_REQUIREMENTS, "installed": versions} + + def build(self, offline: bool = False) -> dict[str, Any]: + mode = "offline" if offline else "online" + venv = self.state["venv-" + mode] + source = self.directory / ("source-" + mode) + shutil.copytree( + FIXTURE, source, ignore=shutil.ignore_patterns("__pycache__", "*.pyc") + ) + destination = self.directory / ("wheels-" + mode) + self.run( + "build-" + mode, + self.uv, + "build", + "--python", + str(self.venv_python(venv)), + "--no-python-downloads", + "--no-build-isolation", + "--wheel", + "--out-dir", + str(destination), + str(source), + offline=offline, + ) + wheels = list(destination.glob("*.whl")) + require(len(wheels) == 1, "Expected exactly one native wheel") + require("-cp37-" in wheels[0].name, "Wheel was not built for CPython 3.7") + self.state["wheel-" + mode] = wheels[0] + return { + "wheel": str(wheels[0]), + "sha256": sha256(wheels[0]), + "no_build_isolation": True, + } + + def verify_wheel(self, offline: bool = False) -> dict[str, Any]: + mode = "offline" if offline else "online" + venv = self.state["venv-" + mode] + self.run( + "wheel-install-" + mode, + self.uv, + "pip", + "install", + "--python", + str(self.venv_python(venv)), + str(self.state["wheel-" + mode]), + offline=offline, + ) + data = self.python("wheel-execution-" + mode, WHEEL_CODE, venv, offline) + require( + data["version"] == [3, 7, 9], "Wheel executed on a different Python version" + ) + require( + within(Path(data["module"]), venv), + "Imported source tree instead of the installed wheel", + ) + require( + all( + version.startswith(PYTHON_VERSION) + for version in data["native_versions"] + ), + "Native extension compile-time/runtime versions differ from CPython 3.7.9", + ) + self.check_identity(data["build"], venv) + return data + + def remove_owned_store(self) -> Path: + store = self.vx_home / "store" / "python" / PYTHON_VERSION + require( + self.marker.read_text(encoding="utf-8") == str(self.directory), + "Missing run ownership marker", + ) + require( + within(store, self.directory) and store.resolve() != self.directory, + "Unsafe removal target", + ) + require(not store.is_symlink(), "Refusing to remove a symlinked Python store") + shutil.rmtree(store) + return store + + def cached_reinstall(self) -> dict[str, Any]: + store = self.remove_owned_store() + output = self.run( + "cached-python-reinstall", + "install", + "python@" + PYTHON_VERSION, + offline=True, + ) + require( + "Served from download cache" in output, + "Reinstall did not attest archive-cache reuse", + ) + require( + "Verified artifact SHA256" in output + and self.state["artifact"]["sha256"] in output, + "Cached payload was not verified again before extraction", + ) + actual = sha256(self.state["archive"]) + require( + actual == self.state["artifact"]["sha256"], + "Reinstall changed the trusted archive bytes", + ) + data = self.python("cached-python-identity", IDENTITY_CODE, offline=True) + self.check_identity(data) + return { + "deleted_store": str(store), + "network_policy": "unreachable_proxy", + "archive_sha256": actual, + "identity": data, + } + + def partial_installation_repair(self) -> dict[str, Any]: + store = self.vx_home / "store" / "python" / PYTHON_VERSION + relative = Path("python.exe" if os.name == "nt" else "bin/python3") + backup = self.directory / "partial-python-executable" + shutil.copy2(store / relative, backup) + self.remove_owned_store() + partial = store / relative + partial.parent.mkdir(parents=True) + shutil.copy2(backup, partial) + output = self.run( + "partial-install-repair", + "install", + "python@" + PYTHON_VERSION, + offline=True, + ) + require( + "Served from download cache" in output, + "VX reused the partial executable instead of repairing the installation", + ) + require( + "Verified artifact SHA256" in output + and self.state["artifact"]["sha256"] in output, + "Repair did not verify the fixed lifecycle payload", + ) + data = self.python("repaired-python-identity", IDENTITY_CODE, offline=True) + self.check_identity(data) + require( + Path(data["include"]).joinpath("Python.h").is_file(), + "Repair did not restore development headers", + ) + return { + "simulated_partial_install": str(partial), + "identity": data, + "network_policy": "unreachable_proxy", + } + + def execute(self) -> int: + self.gate("artifact_contract", self.contract) + self.gate("fresh_acquisition", self.fresh_install, ("artifact_contract",)) + self.gate("archive_integrity", self.archive_integrity, ("fresh_acquisition",)) + self.gate("interpreter_identity", self.identity, ("fresh_acquisition",)) + self.gate("uv_installation", self.install_uv) + self.gate( + "uv_venv", self.create_venv, ("interpreter_identity", "uv_installation") + ) + self.gate("compatible_packages", self.packages, ("uv_venv",)) + self.gate("native_pep517_build", self.build, ("compatible_packages",)) + self.gate( + "installed_wheel_execution", self.verify_wheel, ("native_pep517_build",) + ) + self.gate( + "cached_archive_reinstall", + self.cached_reinstall, + ("archive_integrity", "interpreter_identity"), + ) + self.gate( + "partial_installation_repair", + self.partial_installation_repair, + ("cached_archive_reinstall",), + ) + self.gate( + "offline_venv", + lambda: self.create_venv(True), + ("partial_installation_repair", "uv_installation"), + ) + self.gate( + "offline_packages", + lambda: self.packages(True), + ("offline_venv", "compatible_packages"), + ) + self.gate( + "offline_native_build", lambda: self.build(True), ("offline_packages",) + ) + self.gate( + "offline_wheel_execution", + lambda: self.verify_wheel(True), + ("offline_native_build",), + ) + passed = all(gate["status"] == "passed" for gate in self.receipt["gates"]) + self.receipt.update( + status="passed" if passed else "failed", + finished_at=datetime.now(timezone.utc).isoformat(), + ) + self.save() + print("Receipt: " + str(self.receipt_path), flush=True) + return 0 if passed else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--vx-binary", type=Path, required=True) + parser.add_argument( + "--work-root", + type=Path, + default=Path(tempfile.gettempdir()) / "vx-legacy-python", + ) + parser.add_argument("--receipt", type=Path) + parser.add_argument( + "--source-revision", help="Source revision used to build the tested binary" + ) + parser.add_argument( + "--provider", + type=Path, + default=ROOT / "crates/vx-providers/python/provider.star", + ) + parser.add_argument("--uv-version", default=UV_VERSION) + parser.add_argument("--command-timeout", type=int, default=600) + return Acceptance(parser.parse_args()).execute() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_legacy_python_harness.py b/scripts/tests/test_legacy_python_harness.py new file mode 100644 index 000000000..137e5cb2a --- /dev/null +++ b/scripts/tests/test_legacy_python_harness.py @@ -0,0 +1,118 @@ +"""Safety and evidence tests for the live legacy Python acceptance harness.""" + +import argparse +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import test_legacy_python as harness + + +class LegacyPythonHarnessTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory(prefix="legacy-harness-test-") + self.root = Path(self.temporary.name) + with patch.object( + harness, + "existing_compiler_environment", + side_effect=lambda env: (env, None), + ): + self.acceptance = harness.Acceptance( + argparse.Namespace( + vx_binary=Path(sys.executable), + work_root=self.root, + receipt=None, + provider=self.root / "provider.star", + uv_version=harness.UV_VERSION, + command_timeout=10, + ) + ) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def identity(self) -> dict: + base = self.acceptance.vx_home / "store" / "python" / harness.PYTHON_VERSION + return { + "implementation": "cpython", + "version": [3, 7, 9], + "prefix": str(base), + "base_prefix": str(base), + "executable": str(base / "python.exe"), + } + + def test_identity_refuses_substituted_implementation(self) -> None: + data = self.identity() + data["implementation"] = "pypy" + with self.assertRaisesRegex(AssertionError, "Expected CPython"): + self.acceptance.check_identity(data) + + def test_identity_refuses_substituted_version(self) -> None: + data = self.identity() + data["version"] = [3, 7, 17] + with self.assertRaisesRegex(AssertionError, "version differs"): + self.acceptance.check_identity(data) + + def test_identity_refuses_external_base_interpreter(self) -> None: + data = self.identity() + data["base_prefix"] = str(self.root / "external-python") + with self.assertRaisesRegex(AssertionError, "escaped the owned VX store"): + self.acceptance.check_identity(data) + + def test_identity_accepts_venv_entry_point(self) -> None: + data = self.identity() + venv = self.acceptance.directory / "venv" + data.update(prefix=str(venv), executable=str(venv / "bin" / "python")) + self.acceptance.check_identity(data, venv) + + def test_catalog_refuses_unverified_archive(self) -> None: + self.acceptance.provider.write_text( + '_LEGACY_37_ASSETS = {"linux/x64": {"sha256": None}}', encoding="utf-8" + ) + with patch.object(harness, "platform_key", return_value="linux/x64"): + with self.assertRaisesRegex(AssertionError, "trusted SHA256"): + self.acceptance.contract() + + def test_catalog_reads_fixed_source_and_digest(self) -> None: + self.acceptance.provider.write_text( + '_LEGACY_37_ASSETS = {"linux/x64": {"sha256": "' + + "a" * 64 + + '", "release": "historical", "asset": "fixed.tar.zst"}}', + encoding="utf-8", + ) + with patch.object(harness, "platform_key", return_value="linux/x64"): + artifact = self.acceptance.contract() + self.assertEqual(artifact["sha256"], "a" * 64) + self.assertTrue(artifact["url"].endswith("/historical/fixed.tar.zst")) + + def test_reinstall_refuses_invalid_ownership_marker(self) -> None: + store = self.acceptance.vx_home / "store" / "python" / harness.PYTHON_VERSION + store.mkdir(parents=True) + self.acceptance.marker.write_text("another-run", encoding="utf-8") + with self.assertRaisesRegex(AssertionError, "ownership marker"): + self.acceptance.cached_reinstall() + self.assertTrue( + store.exists(), "Guard must run before deleting the installation" + ) + + def test_failed_gate_blocks_only_its_dependents(self) -> None: + def failure(): + raise AssertionError("expected acquisition failure") + + dependent = unittest.mock.Mock(return_value={}) + self.acceptance.gate("fresh", failure) + self.acceptance.gate("dependent", dependent, ("fresh",)) + self.acceptance.gate("independent", lambda: {"checked": True}) + self.assertEqual( + [gate["status"] for gate in self.acceptance.receipt["gates"]], + ["failed", "blocked", "passed"], + ) + dependent.assert_not_called() + self.assertTrue(self.acceptance.receipt_path.is_file()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/fixtures/legacy-python-build/native.c b/tests/fixtures/legacy-python-build/native.c new file mode 100644 index 000000000..d7248e390 --- /dev/null +++ b/tests/fixtures/legacy-python-build/native.c @@ -0,0 +1,19 @@ +#define PY_SSIZE_T_CLEAN +#include + +static PyObject *versions(PyObject *self, PyObject *args) { + return Py_BuildValue("(ss)", PY_VERSION, Py_GetVersion()); +} + +static PyMethodDef methods[] = { + {"versions", versions, METH_NOARGS, "Return compile-time and runtime Python versions."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, "_native", NULL, -1, methods +}; + +PyMODINIT_FUNC PyInit__native(void) { + return PyModule_Create(&module); +} diff --git a/tests/fixtures/legacy-python-build/pyproject.toml b/tests/fixtures/legacy-python-build/pyproject.toml new file mode 100644 index 000000000..aa22e85b9 --- /dev/null +++ b/tests/fixtures/legacy-python-build/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools==67.8.0", "wheel==0.41.3"] +build-backend = "setuptools.build_meta" diff --git a/tests/fixtures/legacy-python-build/setup.py b/tests/fixtures/legacy-python-build/setup.py new file mode 100644 index 000000000..7d8d3b00a --- /dev/null +++ b/tests/fixtures/legacy-python-build/setup.py @@ -0,0 +1,36 @@ +"""A native PEP 517 build that records the interpreter running its backend.""" + +import json +from pathlib import Path +import sys +import sysconfig + +from setuptools import Extension, setup +from setuptools.command.build_py import build_py + + +class BuildWithProvenance(build_py): + def run(self): + build_py.run(self) + provenance = { + "implementation": sys.implementation.name, + "version": list(sys.version_info[:3]), + "executable": sys.executable, + "prefix": sys.prefix, + "base_prefix": sys.base_prefix, + "compiler": sysconfig.get_config_var("CC"), + } + target = Path(self.build_lib) / "vx_legacy_probe" / "build_provenance.json" + target.write_text(json.dumps(provenance, indent=2), encoding="utf-8") + + +setup( + name="vx-legacy-python-probe", + version="0.0.1", + description="VX legacy Python native build acceptance fixture", + python_requires=">=3.7", + packages=["vx_legacy_probe"], + package_data={"vx_legacy_probe": ["build_provenance.json"]}, + ext_modules=[Extension("vx_legacy_probe._native", ["native.c"])], + cmdclass={"build_py": BuildWithProvenance}, +) diff --git a/tests/fixtures/legacy-python-build/vx_legacy_probe/__init__.py b/tests/fixtures/legacy-python-build/vx_legacy_probe/__init__.py new file mode 100644 index 000000000..fcc6bf487 --- /dev/null +++ b/tests/fixtures/legacy-python-build/vx_legacy_probe/__init__.py @@ -0,0 +1,12 @@ +"""Installed-wheel probes for legacy Python acceptance.""" + +import json +from pathlib import Path + +from ._native import versions as versions + + +def build_provenance(): + return json.loads( + (Path(__file__).parent / "build_provenance.json").read_text(encoding="utf-8") + ) From ed289c7b7bbab6335f5681c1aa302bd87606a1cb Mon Sep 17 00:00:00 2001 From: loonghao Date: Fri, 9 Oct 2026 01:49:12 +0800 Subject: [PATCH 2/2] test: attest offline legacy Python acceptance --- scripts/test_legacy_python.py | 2 ++ scripts/tests/test_legacy_python_harness.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/scripts/test_legacy_python.py b/scripts/test_legacy_python.py index d7edfdb76..693ade5c3 100644 --- a/scripts/test_legacy_python.py +++ b/scripts/test_legacy_python.py @@ -267,6 +267,8 @@ def save(self) -> None: def run(self, label: str, *arguments: str, offline: bool = False) -> str: command = [str(self.vx), *arguments] + if offline and arguments and arguments[0] == self.uv: + command.insert(2, "--offline") environment = dict(self.environment) if offline: environment.update( diff --git a/scripts/tests/test_legacy_python_harness.py b/scripts/tests/test_legacy_python_harness.py index 137e5cb2a..91ea24279 100644 --- a/scripts/tests/test_legacy_python_harness.py +++ b/scripts/tests/test_legacy_python_harness.py @@ -3,6 +3,7 @@ import argparse from pathlib import Path import sys +import subprocess import tempfile import unittest from unittest.mock import patch @@ -113,6 +114,21 @@ def failure(): dependent.assert_not_called() self.assertTrue(self.acceptance.receipt_path.is_file()) + def test_offline_uv_disables_network_and_records_command(self) -> None: + def completed(command, **kwargs): + self.assertEqual(command[2], "--offline") + self.assertEqual(kwargs["env"]["UV_OFFLINE"], "1") + self.assertEqual(kwargs["env"]["HTTPS_PROXY"], "http://127.0.0.1:9") + kwargs["stdout"].write("offline invocation\n") + return subprocess.CompletedProcess(command, 0) + + with patch.object(harness.subprocess, "run", side_effect=completed): + output = self.acceptance.run( + "offline", self.acceptance.uv, "--version", offline=True + ) + self.assertIn("offline invocation", output) + self.assertTrue(self.acceptance.receipt["commands"][-1]["offline"]) + if __name__ == "__main__": unittest.main()