Share a pinned FFmpeg dependency bundle across gallery CI shards

.github/actions/restore-gallery-dependencies/action.yml

@@ -0,0 +1,11 @@
+name: Restore gallery system dependencies
+description: Verify and install this run's pinned Ubuntu FFmpeg packages without package-server access.
+runs:
+  using: composite
+  steps:
+    - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ci-gallery-ffmpeg
+        path: dist/gallery-ffmpeg
+    - shell: bash
+      run: python3 scripts/ci_gallery_dependencies.py install

.github/workflows/ci.yml

@@ -169,12 +169,41 @@ jobs:
           .venv/bin/python -c "import matplotlib; assert matplotlib.__version__ == '3.11.1'"
           .venv/bin/pytest -q tests/pyplot/test_modes.py tests/pyplot/test_public_protocols.py
 
+  # Resolve package-server traffic once, not independently in eight shards.
+  gallery_dependencies:
+    name: Shared gallery FFmpeg dependencies
+    needs: changes
+    if: needs.changes.outputs.pyplot == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Cache pinned FFmpeg package bundle
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: dist/gallery-ffmpeg
+          key: gallery-ffmpeg-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('scripts/ci_gallery_ffmpeg.lock.json') }}
+      - name: Fetch and verify pinned FFmpeg packages once
+        timeout-minutes: 5
+        run: python3 scripts/ci_gallery_dependencies.py download
+      - name: Publish verified FFmpeg package bundle
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: ci-gallery-ffmpeg
+          path: dist/gallery-ffmpeg
+          compression-level: 0
+          retention-days: 3
+
   # The compat canvas renders PNG through XY's native display-list rasterizer
   # and reports any forbidden fallback. The completed zero-waiver baseline
   # makes every accepted execution/render/behavior a required gate.
   pyplot_gallery:
     name: Pyplot gallery ratchet (${{ matrix.shard }}/8)
-    needs: [changes, build]
+    needs: [changes, build, gallery_dependencies]
     if: needs.changes.outputs.pyplot == 'true'
     runs-on: ubuntu-24.04
     timeout-minutes: 45
@@ -203,10 +232,9 @@ jobs:
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: "22.23.3"
-      - name: Install standard gallery system dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends -y ffmpeg
+      - name: Restore verified shared FFmpeg packages
+        timeout-minutes: 5
+        uses: ./.github/actions/restore-gallery-dependencies
       - name: Install xy and Matplotlib 3.11
         env:
           XY_REQUIRE_CARGO: "1"
@@ -1313,7 +1341,7 @@ jobs:
     name: CI result
     runs-on: ubuntu-24.04
     if: always()
-    needs: [changes, build, test, rust_checks, browser_smokes, regression_checks, python_floor, matplotlib_reference, matplotlib_patch_smoke, pyplot_gallery, pyplot_gallery_extended, browser_conformance, sdist, wheels, install_without_rust]
+    needs: [changes, build, test, rust_checks, browser_smokes, regression_checks, python_floor, matplotlib_reference, matplotlib_patch_smoke, pyplot_gallery, pyplot_gallery_extended, browser_conformance, sdist, wheels, install_without_rust, gallery_dependencies]
     steps:
       - name: Require selected checks to succeed
         env:

news/541.misc.md

@@ -0,0 +1 @@
+Download and cache a pinned, hash-verified Ubuntu FFmpeg package bundle once per CI run and share it across all eight standard gallery shards, replacing repeated package-server installs with offline restoration.

scripts/ci_gallery_dependencies.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""Share pinned Ubuntu FFmpeg packages across gallery shards (readiness § CI)."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import re
+import shlex
+import subprocess
+import tempfile
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import urlopen
+
+LOCK = Path(__file__).with_name("ci_gallery_ffmpeg.lock.json")
+HOSTS = {"archive.ubuntu.com", "security.ubuntu.com"}
+
+
+def load_lock(path: Path = LOCK) -> list[dict]:
+    data = json.loads(path.read_text())
+    if data.get("schema_version") != 1 or data.get("platform") != "ubuntu-24.04-amd64":
+        raise ValueError("unsupported gallery dependency lock")
+    packages = data["packages"]
+    seen: set[str] = set()
+    names: set[str] = set()
+    for package in packages:
+        filename, url = package["filename"], urlsplit(package["url"])
+        if (
+            not re.fullmatch(r"[a-z0-9][A-Za-z0-9._+%:~-]*\.deb", filename)
+            or filename in seen
+            or package["name"] in names
+            or package["architecture"] not in {"amd64", "all"}
+            or not re.fullmatch(r"[0-9a-f]{128}", package["sha512"])
+            or not isinstance(package["size"], int)
+            or package["size"] <= 0
+            or url.scheme != "https"
+            or url.netloc not in HOSTS
+            or not url.path.startswith("/ubuntu/pool/")
+            or url.query
+            or url.fragment
+        ):
+            raise ValueError(f"unsafe or duplicate gallery dependency: {filename}")
+        seen.add(filename)
+        names.add(package["name"])
+    if "ffmpeg" not in names:
+        raise ValueError("gallery dependency lock must include ffmpeg")
+    return packages
+
+
+def verify_file(path: Path, package: dict) -> None:
+    if path.is_symlink() or not path.is_file() or path.stat().st_size != package["size"]:
+        raise ValueError(f"missing, unsafe or wrong-size gallery package: {path.name}")
+    with path.open("rb") as stream:
+        digest = hashlib.file_digest(stream, "sha512").hexdigest()
+    if digest != package["sha512"]:
+        raise ValueError(f"gallery package digest mismatch: {path.name}")
+
+
+def verify(directory: Path, packages: list[dict]) -> None:
+    if directory.is_symlink() or not directory.is_dir():
+        raise ValueError("gallery bundle must be a real directory")
+    if {p.name for p in directory.iterdir()} != {p["filename"] for p in packages}:
+        raise ValueError("gallery bundle has missing or unexpected files")
+    for package in packages:
+        verify_file(directory / package["filename"], package)
+
+
+def download(directory: Path, packages: list[dict]) -> None:
+    directory.mkdir(parents=True, exist_ok=True)
+    if directory.is_symlink():
+        raise ValueError("gallery bundle must be a real directory")
+
+    def fetch(package: dict) -> None:
+        path = directory / package["filename"]
+        if path.exists() or path.is_symlink():
+            verify_file(path, package)
+            return
+        original = urlsplit(package["url"])
+        alternate = (
+            "security.ubuntu.com"
+            if original.netloc == "archive.ubuntu.com"
+            else "archive.ubuntu.com"
+        )
+        urls = [package["url"], urlunsplit(original._replace(netloc=alternate))]
+        last_error: Exception | None = None
+        for url in urls:
+            # Partial downloads are never promoted into the cache/artifact.
+            fd, name = tempfile.mkstemp(dir=directory, suffix=".partial")
+            partial = Path(name)
+            try:
+                with os.fdopen(fd, "wb") as output, urlopen(url, timeout=20) as response:
+                    final = urlsplit(response.geturl())
+                    if final.scheme != "https" or final.netloc not in HOSTS:
+                        raise ValueError("gallery package download redirected outside Ubuntu HTTPS")
+                    remaining = package["size"]
+                    deadline = time.monotonic() + 40
+                    while block := response.read1(min(1024 * 1024, remaining + 1)):
+                        if time.monotonic() > deadline:
+                            raise TimeoutError("gallery package transfer exceeded 40 seconds")
+                        remaining -= len(block)
+                        if remaining < 0:
+                            raise ValueError(f"oversize gallery package: {path.name}")
+                        output.write(block)
+                verify_file(partial, package)
+                partial.chmod(0o644)
+                partial.replace(path)
+                return
+            except (HTTPError, URLError, TimeoutError, ValueError) as error:
+                last_error = error
+            finally:
+                partial.unlink(missing_ok=True)
+        raise RuntimeError(f"cannot download pinned package {path.name}") from last_error
+
+    started = time.monotonic()
+    with ThreadPoolExecutor(max_workers=8) as pool:
+        list(pool.map(fetch, packages))
+    verify(directory, packages)
+    print(f"Verified {len(packages)} pinned packages in {time.monotonic() - started:.2f}s")
+
+
+def install(directory: Path, packages: list[dict]) -> None:
+    # Integrity checks precede every subprocess, especially privileged dpkg.
+    verify(directory, packages)
+    release = Path("/etc/os-release").read_text().splitlines()
+    if (
+        'VERSION_ID="24.04"' not in release
+        or "ID=ubuntu" not in release
+        or platform.machine() != "x86_64"
+    ):
+        raise ValueError("gallery packages require Ubuntu 24.04 x86_64")
+    subprocess.run(
+        [
+            "sudo",
+            "env",
+            "DEBIAN_FRONTEND=noninteractive",
+            "dpkg",
+            "--install",
+            *[str(directory / p["filename"]) for p in packages],
+        ],
+        check=True,
+        timeout=180,
+    )
+    version = next(p["version"] for p in packages if p["name"] == "ffmpeg")
+    installed = subprocess.check_output(["dpkg-query", "-W", "-f=${Version}", "ffmpeg"], text=True)
+    if installed != version:
+        raise ValueError(f"unexpected installed FFmpeg version: {installed}")
+    # The gallery needs a real encoder, not merely an executable on PATH.
+    with tempfile.TemporaryDirectory() as temporary:
+        subprocess.run(
+            [
+                "ffmpeg",
+                "-nostdin",
+                "-v",
+                "error",
+                "-f",
+                "lavfi",
+                "-i",
+                "color=size=16x16:rate=1",
+                "-frames:v",
+                "1",
+                "-c:v",
+                "libx264",
+                str(Path(temporary) / "probe.mp4"),
+            ],
+            check=True,
+            timeout=20,
+        )
+
+
+def refresh_lock(path: Path) -> None:
+    """Resolve reviewed runtime package names from authenticated Ubuntu indexes.
+
+    This maintenance command does not install packages or change host APT state.
+    CI only consumes the committed lock; it never resolves floating dependencies.
+    """
+    names = [p["name"] for p in load_lock(path)]
+    with tempfile.TemporaryDirectory(prefix="xy-gallery-apt-") as temporary:
+        root = Path(temporary)
+        for child in ("lists/partial", "archives/partial", "sources.list.d"):
+            (root / child).mkdir(parents=True, exist_ok=True)
+        (root / "status").touch()
+        sources = [
+            ("archive.ubuntu.com", "noble"),
+            ("archive.ubuntu.com", "noble-updates"),
+            ("security.ubuntu.com", "noble-security"),
+        ]
+        (root / "sources.list").write_text(
+            "".join(
+                f"deb [arch=amd64 signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] https://{host}/ubuntu {suite} main universe\n"
+                for host, suite in sources
+            )
+        )
+        config = root / "apt.conf"
+        config.write_text(f'''Dir::Etc::sourcelist "{root}/sources.list";
+Dir::Etc::sourceparts "{root}/sources.list.d";
+Dir::State::lists "{root}/lists";
+Dir::State::status "{root}/status";
+Dir::Cache::archives "{root}/archives";
+Dir::Cache::pkgcache "";
+Dir::Cache::srcpkgcache "";
+Acquire::Languages "none";
+APT::Architecture "amd64";
+APT::Architectures {{ "amd64"; }};
+Acquire::Retries "2";
+Acquire::http::Timeout "20";
+Acquire::https::Timeout "20";
+Acquire::https::Verify-Peer "true";
+Acquire::https::Verify-Host "true";
+Acquire::Check-Valid-Until "true";
+Acquire::AllowInsecureRepositories "false";
+Acquire::AllowDowngradeToInsecureRepositories "false";
+APT::Get::AllowUnauthenticated "false";
+APT::Update::Error-Mode "any";
+''')
+        environment = {**os.environ, "LC_ALL": "C"}
+        subprocess.run(
+            ["apt-get", "-c", str(config), "update"], env=environment, check=True, timeout=300
+        )
+        policy = subprocess.check_output(
+            ["apt-cache", "-c", str(config), "policy", *names], env=environment, text=True
+        )
+        versions: dict[str, str] = {}
+        current = ""
+        for line in policy.splitlines():
+            if line and not line[0].isspace() and line.endswith(":"):
+                current = line[:-1]
+            if line.strip().startswith("Candidate:"):
+                versions[current] = line.split("Candidate:", 1)[1].strip()
+        if set(versions) != set(names) or "(none)" in versions.values():
+            raise ValueError("Ubuntu indexes cannot resolve every locked package")
+        uris = subprocess.check_output(
+            [
+                "apt-get",
+                "-c",
+                str(config),
+                "--print-uris",
+                "download",
+                *[f"{name}={versions[name]}" for name in names],
+            ],
+            env=environment,
+            text=True,
+        )
+        packages = []
+        for line in uris.splitlines():
+            if not line.startswith("'https://"):
+                continue
+            url, filename, size, checksum = shlex.split(line)
+            algorithm, digest = checksum.split(":", 1)
+            if algorithm != "SHA512":
+                raise ValueError("Ubuntu package index must provide a SHA-512 digest")
+            name = filename.split("_", 1)[0]
+            packages.append(
+                dict(
+                    name=name,
+                    version=versions[name],
+                    architecture=filename.rsplit("_", 1)[1][:-4],
+                    filename=filename,
+                    size=int(size),
+                    sha512=digest,
+                    url=url,
+                )
+            )
+        if {p["name"] for p in packages} != set(names):
+            raise ValueError("Ubuntu indexes omitted a locked package")
+        updated = {
+            "schema_version": 1,
+            "platform": "ubuntu-24.04-amd64",
+            "source": "Ubuntu noble, noble-updates and noble-security; authenticated with ubuntu-archive-keyring",
+            "packages": sorted(packages, key=lambda p: p["name"]),
+        }
+        staged = root / "lock.json"
+        staged.write_text(json.dumps(updated, indent=2) + "\n")
+        load_lock(staged)
+        path.write_text(staged.read_text())
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=("download", "verify", "install", "refresh-lock"))
+    parser.add_argument("--lock", type=Path, default=LOCK)
+    parser.add_argument("--directory", type=Path, default=Path("dist/gallery-ffmpeg"))
+    args = parser.parse_args()
+    if args.command == "refresh-lock":
+        refresh_lock(args.lock)
+    else:
+        {"download": download, "verify": verify, "install": install}[args.command](
+            args.directory, load_lock(args.lock)
+        )
+
+
+if __name__ == "__main__":
+    main()