Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 130 additions & 0 deletions .github/workflows/test-legacy-python.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
name: Legacy Python Acceptance

on:
pull_request:
paths:
- "crates/vx-providers/python/**"
- "crates/vx-providers/uv/**"
- "crates/vx-runtime/**"
- "crates/vx-runtime-http/**"
- "crates/vx-runtime-archive/**"
- "crates/vx-cache/**"
- "crates/vx-paths/**"
- "crates/vx-cli/**"
- "crates/vx-resolver/**"
- "crates/vx-starlark/**"
- "scripts/test_legacy_python.py"
- "scripts/tests/test_legacy_python_harness.py"
- "tests/fixtures/legacy-python-build/**"
- ".github/workflows/test-legacy-python.yml"
workflow_dispatch:

permissions:
contents: read

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
build:
name: Build legacy acceptance VX (${{ matrix.platform }})
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- platform: linux-x64
os: ubuntu-24.04
binary: vx
- platform: windows-x64
os: windows-2025
binary: vx.exe
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: sccache
SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache
steps:
- uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: ./.github/actions/setup-rust-sccache
with:
shared-key: legacy-python-build
- name: Bootstrap VX for build commands
uses: ./
with:
cache: "false"
setup: "false"
- name: Build this revision
run: vx cargo build --locked --profile dev-fast --bin vx
- uses: actions/upload-artifact@v7
with:
name: legacy-python-vx-${{ matrix.platform }}
path: target/dev-fast/${{ matrix.binary }}
if-no-files-found: error
retention-days: 3

acceptance:
name: Legacy Python 3.7 (${{ matrix.platform }})
needs: build
runs-on: ${{ matrix.os }}
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
- platform: linux-x64
os: ubuntu-24.04
binary: vx
- platform: windows-x64
os: windows-2025
binary: vx.exe
steps:
- uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: actions/download-artifact@v8
with:
name: legacy-python-vx-${{ matrix.platform }}
path: bin
- name: Make VX executable
if: runner.os != 'Windows'
run: chmod +x bin/vx
- name: Bootstrap modern harness interpreter
uses: ./
with:
tools: "python@3.12"
cache: "false"
setup: "false"
- name: Bind existing Windows compiler
if: runner.os == 'Windows'
uses: ilammy/msvc-dev-cmd@v1
with:
arch: x64
- name: Check acceptance guards
run: vx python@3.12 -m unittest discover -s scripts/tests -p test_legacy_python_harness.py
- name: Run isolated legacy acceptance
shell: bash
env:
VX_ACCEPTANCE_BINARY: ${{ github.workspace }}/bin/${{ matrix.binary }}
VX_ACCEPTANCE_ROOT: ${{ runner.temp }}/vx-legacy-python
VX_ACCEPTANCE_RECEIPT: ${{ github.workspace }}/legacy-python-receipt.json
VX_ACCEPTANCE_SOURCE_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: |
vx python@3.12 scripts/test_legacy_python.py \
--vx-binary "$VX_ACCEPTANCE_BINARY" \
--work-root "$VX_ACCEPTANCE_ROOT" \
--receipt "$VX_ACCEPTANCE_RECEIPT" \
--source-revision "$VX_ACCEPTANCE_SOURCE_SHA"
- name: Upload acceptance evidence
if: always()
uses: actions/upload-artifact@v7
with:
name: legacy-python-evidence-${{ matrix.platform }}
path: |
legacy-python-receipt.json
${{ runner.temp }}/vx-legacy-python/run-*/logs
if-no-files-found: warn
retention-days: 14
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

63 changes: 47 additions & 16 deletions crates/vx-providers/python/provider.star
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,35 @@ _PBS_TRIPLES = {
}

_LEGACY_37_ASSETS = {
# python-build-standalone's 20200822 release predates the modern
# cpython-{version}+{build_tag}-{triple}-install_only_stripped naming.
"windows/x64": "cpython-3.7.9-x86_64-pc-windows-msvc-shared-pgo-20200823T0118.tar.zst",
"macos/x64": "cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T0123.tar.zst",
"linux/x64": "cpython-3.7.9-x86_64-unknown-linux-gnu-pgo-20200823T0036.tar.zst",
# Fixed lifecycle catalog: release metadata must not select a different
# artifact after upstream stops publishing this Python line.
# Digests are VX intake pins of official HTTPS release assets; these old
# releases publish no checksums or signatures. They are not publisher hashes.
"windows/x64": {
"release": "20200822",
"asset": "cpython-3.7.9-x86_64-pc-windows-msvc-shared-pgo-20200823T0118.tar.zst",
"sha256": "8769a244cfe54c32c5253b78897a0fe82fe419dfde653b1afc3f5f20594cca89",
"asset_id": 24195911,
"size": 31429615,
"required_paths": ["Lib/site.py", "Lib/venv/__init__.py", "include/Python.h", "libs/python37.lib"],
},
"macos/x64": {
# The 20200822 build has an unwanted libintl.dylib dependency.
"release": "20200823",
"asset": "cpython-3.7.9-x86_64-apple-darwin-pgo-20200823T2228.tar.zst",
"sha256": "53657e7712cc7b24491fb1fc66dcc8f47a577fc77df137178746987ba4c5afb8",
"asset_id": 24211674,
"size": 26136896,
"required_paths": ["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.dylib"],
},
"linux/x64": {
"release": "20200822",
"asset": "cpython-3.7.9-x86_64-unknown-linux-gnu-pgo-20200823T0036.tar.zst",
"sha256": "c6d6256d13e929e77e7ee6e53470fe63ad19d173fee6d56bb1b2dbda67081543",
"asset_id": 24195897,
"size": 28357634,
"required_paths": ["lib/python3.7/site.py", "lib/python3.7/venv/__init__.py", "include/python3.7m/Python.h", "lib/libpython3.7m.a", "lib/libpython3.7m.so.1.0"],
},
}

_PYPY27_VERSION = "2.7.18"
Expand All @@ -102,12 +126,18 @@ def _pbs_triple(ctx):
def _platform_key(ctx):
return "{}/{}".format(ctx.platform.os, ctx.platform.arch)

def _is_legacy_37(version, build_tag):
return version == "3.7.9" and build_tag == "20200822"

def _is_pypy27(version):
return version == _PYPY27_VERSION

def _legacy37_artifact(ctx, version):
if version != "3.7.9":
fail("Python {} has no pinned portable build; request Python 3.7.9 explicitly".format(version))
platform = _platform_key(ctx)
artifact = _LEGACY_37_ASSETS.get(platform)
if artifact == None:
fail("Python 3.7.9 portable build unavailable for {}; supported platforms: Windows x64, Linux x64, macOS x64".format(platform))
return artifact

# ---------------------------------------------------------------------------
# download_url — python-build-standalone asset
# ---------------------------------------------------------------------------
Expand All @@ -119,17 +149,16 @@ def download_url(ctx, version):
return None
return "https://downloads.python.org/pypy/{}".format(asset)

if version.startswith("3.7."):
artifact = _legacy37_artifact(ctx, version)
return github_asset_url("astral-sh", "python-build-standalone", artifact["release"], artifact["asset"])

triple = _pbs_triple(ctx)
if not triple:
return None
build_tag = ctx.version_date
if not build_tag:
return None
if _is_legacy_37(version, build_tag):
asset = _LEGACY_37_ASSETS.get(_platform_key(ctx))
if not asset:
return None
return github_asset_url("astral-sh", "python-build-standalone", build_tag, asset)
asset = "cpython-{}+{}-{}-install_only_stripped.tar.gz".format(version, build_tag, triple)
return github_asset_url("astral-sh", "python-build-standalone", build_tag, asset)

Expand Down Expand Up @@ -157,9 +186,8 @@ def install_layout(ctx, version):
"executable_paths": exe_paths,
}

if version == "3.7.9":
if _LEGACY_37_ASSETS.get(_platform_key(ctx)) == None:
return None
if version.startswith("3.7."):
artifact = _legacy37_artifact(ctx, version)
if ctx.platform.os == "windows":
exe_paths = ["python.exe"]
else:
Expand All @@ -168,6 +196,9 @@ def install_layout(ctx, version):
"type": "archive",
"strip_prefix": "python/install",
"executable_paths": exe_paths,
"required_paths": artifact["required_paths"],
"sha256": artifact["sha256"],
"mirror_urls": [],
}

# The python-build-standalone tarball has a top-level "python/" directory.
Expand Down
2 changes: 2 additions & 0 deletions crates/vx-runtime-http/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ repository.workspace = true
async-trait = { workspace = true }
anyhow = { workspace = true }
tracing = { workspace = true }
sha2 = { workspace = true }
hex = "0.4"

# Async runtime
tokio = { workspace = true, features = ["rt", "fs", "io-util", "process"] }
Expand Down
61 changes: 57 additions & 4 deletions crates/vx-runtime-http/src/installer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,12 @@ impl RealInstaller {
/// the GET response's Content-Disposition header or final redirected URL.
/// For APIs like Adoptium that use redirect chains (307 → 302 → CDN),
/// this saves ~3-5 seconds by eliminating the redundant HEAD round-trip.
async fn download_and_detect_filename(&self, url: &str, dest: &Path) -> Result<Option<String>> {
async fn download_and_detect_filename(
&self,
url: &str,
dest: &Path,
expected_sha256: Option<&str>,
) -> Result<Option<String>> {
use futures_util::StreamExt;
use indicatif::{ProgressBar, ProgressStyle};
use tokio::io::AsyncWriteExt;
Expand All @@ -97,6 +102,7 @@ impl RealInstaller {
}
std::fs::copy(&path, dest)?;
pb.finish_and_clear();
self.verify_download(url, dest, expected_sha256)?;
tracing::debug!(url = url, cached_path = ?path, "Served from download cache");
// Return cached filename from metadata
let cached_filename = if !metadata.filename.is_empty()
Expand Down Expand Up @@ -125,6 +131,7 @@ impl RealInstaller {
}
std::fs::copy(&path, dest)?;
pb.finish_and_clear();
self.verify_download(url, dest, expected_sha256)?;
tracing::debug!(url = url, cached_path = ?path, "Served from download cache (with ETag)");
return Ok(None);
}
Expand Down Expand Up @@ -268,6 +275,11 @@ impl RealInstaller {

progress_bar.finish_and_clear();
file.flush().await?;
drop(file);

// The catalog digest is authoritative for every transport, including
// mirrors and cache hits. Never admit an unverified payload to cache.
self.verify_download(url, dest, expected_sha256)?;

// Store in download cache if enabled
if let Some(cache) = &self.http.download_cache
Expand All @@ -279,6 +291,36 @@ impl RealInstaller {
Ok(detected_filename)
}

fn verify_download(&self, url: &str, path: &Path, expected: Option<&str>) -> Result<()> {
use sha2::{Digest, Sha256};
use std::io::Read;

let Some(expected) = expected else {
return Ok(());
};
let mut file = std::fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0; 64 * 1024];
loop {
let count = file.read(&mut buffer)?;
if count == 0 {
break;
}
hasher.update(&buffer[..count]);
}
let actual = hex::encode(hasher.finalize());
if !actual.eq_ignore_ascii_case(expected) {
if let Some(cache) = &self.http.download_cache {
let _ = cache.remove(url);
}
anyhow::bail!(
"SHA256 mismatch for {url}: expected {expected}, got {actual}; refusing installation"
);
}
tracing::info!(url, sha256 = %actual, "Verified artifact SHA256");
Ok(())
}

/// Parse filename from Content-Disposition header value
fn parse_content_disposition(value: &str) -> Option<String> {
// Handle: attachment; filename=xxx.zip
Expand Down Expand Up @@ -526,7 +568,7 @@ impl Installer for RealInstaller {
}

async fn download_and_extract(&self, url: &str, dest: &Path) -> Result<()> {
self.download_and_extract_impl(url, dest, false).await
self.download_and_extract_impl(url, dest, false, None).await
}

async fn download_with_layout(
Expand All @@ -539,7 +581,13 @@ impl Installer for RealInstaller {
// example, NSIS). Explicit archive extensions still take precedence.
let skip_embedded_archive_detection =
metadata.contains_key("target_name") && metadata.contains_key("target_dir");
self.download_and_extract_impl(url, dest, skip_embedded_archive_detection)
let expected_sha256 = metadata.get("sha256").map(String::as_str);
if let Some(digest) = expected_sha256
&& (digest.len() != 64 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()))
{
anyhow::bail!("Invalid artifact SHA256: expected exactly 64 hexadecimal characters");
}
self.download_and_extract_impl(url, dest, skip_embedded_archive_detection, expected_sha256)
.await?;

// Debug: log metadata and dest contents
Expand Down Expand Up @@ -749,6 +797,7 @@ impl RealInstaller {
url: &str,
dest: &Path,
skip_embedded_archive_detection: bool,
expected_sha256: Option<&str>,
) -> Result<()> {
// Create temp file for download
let temp_dir = tempfile::tempdir()?;
Expand All @@ -759,7 +808,11 @@ impl RealInstaller {
// Download and detect filename in a single GET request (no separate HEAD).
let temp_download_path = temp_dir.path().join("download_temp");
let detected_filename = self
.download_and_detect_filename(url_without_fragment, &temp_download_path)
.download_and_detect_filename(
url_without_fragment,
&temp_download_path,
expected_sha256,
)
.await?;

let archive_name = detected_filename.unwrap_or_else(|| {
Expand Down
Loading
Loading