From 0e5708955b6a9b6c43f89bea198ad00935eca2b8 Mon Sep 17 00:00:00 2001 From: Hal <13111745+loonghao@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:24:14 -0700 Subject: [PATCH] ci: gate installer extended formats on native platforms --- .github/workflows/ci.yml | 76 +++++++++++++ ...est_installer_extended_formats_workflow.py | 102 ++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 scripts/tests/test_installer_extended_formats_workflow.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0663bef1c..1e76f8e9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,9 @@ jobs: with: fetch-depth: 0 + - name: Validate installer CI contract + run: python3 -m unittest discover -s scripts/tests -p "test_installer_extended_formats_workflow.py" -v + - name: Detect file changes id: filter shell: bash @@ -941,6 +944,68 @@ jobs: if command -v python3 >/dev/null 2>&1; then PY=python3; else PY=python; fi "$PY" scripts/summarize_nextest_flaky.py --junit target/nextest/default/junit.xml + # The default workspace suite does not enable extended-formats. Keep the + # archive safety/functional regressions independent of the network-heavy CLI + # suite and coverage, so a timeout there cannot hide an untested installer. + test-installer-extended-formats: + name: Installer extended formats (${{ matrix.platform.name }}) + runs-on: ${{ matrix.platform.os }} + needs: [detect-changes, build-vx] + if: needs.detect-changes.outputs.should_test == 'true' + timeout-minutes: 25 + env: + RUSTC_WRAPPER: sccache + SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache + + strategy: + fail-fast: false + matrix: + platform: + - name: Linux + os: ubuntu-latest + artifact: vx-linux-x64 + binary: vx + - name: Windows + os: windows-latest + artifact: vx-windows-x64 + binary: vx.exe + - name: macOS + os: macos-latest + artifact: vx-macos-x64 + binary: vx + + steps: + - uses: actions/checkout@v7 + + - name: Download vx binary + uses: actions/download-artifact@v8 + with: + name: ${{ matrix.platform.artifact }} + path: ./bin + + - name: Setup vx + shell: bash + run: | + chmod +x ./bin/${{ matrix.platform.binary }} 2>/dev/null || true + echo "${{ github.workspace }}/bin" >> $GITHUB_PATH + + - name: Setup Rust + sccache + uses: ./.github/actions/setup-rust-sccache + with: + shared-key: installer-extended-formats + install-nextest: "true" + + - name: Run native 7z and SFX regressions + shell: bash + # This runs the real extractor against generated archives, including + # traversal rejection on all three OSes and symlink rejection on Unix. + # Fail if feature/configuration drift leaves zero tests selected, and do + # not turn a filesystem-safety regression into a pass through retries. + run: >- + vx cargo nextest run --locked -p vx-installer + --features extended-formats --test sevenz_sfx_tests + --no-fail-fast --no-tests fail --retries 0 + # ============================================================================ # Stage 5: Cross-compilation (PR only, skip on main since release builds cover it) # ============================================================================ @@ -1168,6 +1233,7 @@ jobs: - code-quality - test-targeted - test-full + - test-installer-extended-formats - provider-static - cross-build - security-audit @@ -1231,6 +1297,16 @@ jobs: check_job "build-vx" "${{ needs.build-vx.result }}" check_job "code-quality" "${{ needs.code-quality.result }}" check_job "test-targeted" "${{ needs.test-targeted.result }}" + + # Once Rust/CI changes select installer regressions, skipped is not a + # pass either: all three native matrix legs must have actually run. + if [[ "${{ needs.detect-changes.outputs.should_test }}" == "true" && + "${{ needs.test-installer-extended-formats.result }}" != "success" ]]; then + echo "- ❌ test-installer-extended-formats (required but not successful)" >> $GITHUB_STEP_SUMMARY + failed=1 + else + check_job "test-installer-extended-formats" "${{ needs.test-installer-extended-formats.result }}" + fi check_job "provider-static" "${{ needs.provider-static.result }}" check_job "security-audit" "${{ needs.security-audit.result }}" diff --git a/scripts/tests/test_installer_extended_formats_workflow.py b/scripts/tests/test_installer_extended_formats_workflow.py new file mode 100644 index 000000000..e13287a80 --- /dev/null +++ b/scripts/tests/test_installer_extended_formats_workflow.py @@ -0,0 +1,102 @@ +"""Keep native installer regression coverage connected to the required CI gate.""" + +import os +from pathlib import Path +import re +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") +INSTALLER_JOB = "test-installer-extended-formats" + + +def job_body(name: str) -> str: + """Read a top-level job without adding a YAML dependency to script tests.""" + match = re.search(rf"^ {re.escape(name)}:\n(.*?)(?=^ [\w-]+:|\Z)", + WORKFLOW, re.MULTILINE | re.DOTALL) + if match is None: + raise AssertionError(f"Missing CI job: {name}") + return match.group(1) + + +class InstallerExtendedFormatsWorkflowTests(unittest.TestCase): + def test_native_matrix_covers_all_supported_operating_systems(self) -> None: + job = job_body(INSTALLER_JOB) + self.assertEqual(re.findall(r"^ os: (.+)$", job, re.MULTILINE), + ["ubuntu-latest", "windows-latest", "macos-latest"]) + self.assertIn("runs-on: ${{ matrix.platform.os }}", job) + self.assertIn("fail-fast: false", job) + self.assertIn("needs: [detect-changes, build-vx]", job) + self.assertIn("if: needs.detect-changes.outputs.should_test == 'true'", job) + self.assertIn("name: ${{ matrix.platform.artifact }}", job) + + def test_real_regressions_are_feature_enabled_and_cannot_silently_skip(self) -> None: + job = job_body(INSTALLER_JOB) + command = " ".join(job.split(" run: >-\n", 1)[1].split()) + self.assertTrue(command.startswith("vx cargo nextest run --locked -p vx-installer ")) + for required in ("--features extended-formats", "--test sevenz_sfx_tests", + "--no-fail-fast", "--no-tests fail", "--retries 0"): + self.assertIn(required, command) + for forbidden in ("--no-run", "--target", "--workspace", "|| true", + "continue-on-error", "--run-ignored"): + self.assertNotIn(forbidden, command) + self.assertNotIn("continue-on-error", job) + self.assertIn('install-nextest: "true"', job) + + def test_contract_and_success_gate_are_wired_into_ci(self) -> None: + self.assertIn(Path(__file__).name, job_body("detect-changes")) + self.assertIn(f" - {INSTALLER_JOB}\n", job_body("ci-success")) + self.assertIn("if: always()", job_body("ci-success")) + + def test_whole_workspace_coverage_is_preserved(self) -> None: + coverage = job_body("coverage") + self.assertIn("github.event.inputs.force_full_ci == 'true'", coverage) + self.assertIn("run: vx just coverage-ci", coverage) + self.assertIn("timeout-minutes: 45", coverage) + self.assertIn("cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info", + (ROOT / "justfile").read_text(encoding="utf-8")) + self.assertIn('check_optional_job "coverage" "${{ needs.coverage.result }}"', + job_body("ci-success")) + + def run_success_gate(self, selected: bool, result: str, + coverage: str = "skipped") -> subprocess.CompletedProcess: + # Execute the workflow's actual shell gate, substituting Actions values. + # This tests orchestration only; extraction itself runs natively in CI. + script = textwrap.dedent(job_body("ci-success").split(" run: |\n", 1)[1]) + values = { + "needs.detect-changes.outputs.should_test": str(selected).lower(), + f"needs.{INSTALLER_JOB}.result": result, + "needs.coverage.result": coverage, + } + script = re.sub(r"\$\{\{\s*(.*?)\s*\}\}", + lambda match: values.get(match.group(1), "success"), script) + with tempfile.TemporaryDirectory() as directory: + return subprocess.run( + ["bash", "-e", "-c", script], check=False, capture_output=True, text=True, + env={**os.environ, "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary")}, + ) + + def test_selected_installer_regressions_require_success(self) -> None: + for result in ("failure", "cancelled", "skipped", ""): + with self.subTest(result=result): + self.assertNotEqual(self.run_success_gate(True, result).returncode, 0) + self.assertEqual(self.run_success_gate(True, "success").returncode, 0) + + def test_unselected_installer_regressions_may_skip_but_not_fail(self) -> None: + self.assertEqual(self.run_success_gate(False, "skipped").returncode, 0) + for result in ("failure", "cancelled"): + with self.subTest(result=result): + self.assertNotEqual(self.run_success_gate(False, result).returncode, 0) + + def test_coverage_failure_or_cancellation_still_fails_the_gate(self) -> None: + for result in ("failure", "cancelled"): + with self.subTest(result=result): + self.assertNotEqual(self.run_success_gate(True, "success", result).returncode, 0) + + +if __name__ == "__main__": + unittest.main()