Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ jobs:
with:
fetch-depth: 0

- name: Validate installer CI contract
run: python3 -m unittest discover -s scripts/tests -p "test_installer_extended_formats_workflow.py" -v

- name: Detect file changes
id: filter
shell: bash
Expand Down Expand Up @@ -941,6 +944,68 @@ jobs:
if command -v python3 >/dev/null 2>&1; then PY=python3; else PY=python; fi
"$PY" scripts/summarize_nextest_flaky.py --junit target/nextest/default/junit.xml

# The default workspace suite does not enable extended-formats. Keep the
# archive safety/functional regressions independent of the network-heavy CLI
# suite and coverage, so a timeout there cannot hide an untested installer.
test-installer-extended-formats:
name: Installer extended formats (${{ matrix.platform.name }})
runs-on: ${{ matrix.platform.os }}
needs: [detect-changes, build-vx]
if: needs.detect-changes.outputs.should_test == 'true'
timeout-minutes: 25
env:
RUSTC_WRAPPER: sccache
SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache

strategy:
fail-fast: false
matrix:
platform:
- name: Linux
os: ubuntu-latest
artifact: vx-linux-x64
binary: vx
- name: Windows
os: windows-latest
artifact: vx-windows-x64
binary: vx.exe
- name: macOS
os: macos-latest
artifact: vx-macos-x64
binary: vx

steps:
- uses: actions/checkout@v7

- name: Download vx binary
uses: actions/download-artifact@v8
with:
name: ${{ matrix.platform.artifact }}
path: ./bin

- name: Setup vx
shell: bash
run: |
chmod +x ./bin/${{ matrix.platform.binary }} 2>/dev/null || true
echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

- name: Setup Rust + sccache
uses: ./.github/actions/setup-rust-sccache
with:
shared-key: installer-extended-formats
install-nextest: "true"

- name: Run native 7z and SFX regressions
shell: bash
# This runs the real extractor against generated archives, including
# traversal rejection on all three OSes and symlink rejection on Unix.
# Fail if feature/configuration drift leaves zero tests selected, and do
# not turn a filesystem-safety regression into a pass through retries.
run: >-
vx cargo nextest run --locked -p vx-installer
--features extended-formats --test sevenz_sfx_tests
--no-fail-fast --no-tests fail --retries 0

# ============================================================================
# Stage 5: Cross-compilation (PR only, skip on main since release builds cover it)
# ============================================================================
Expand Down Expand Up @@ -1168,6 +1233,7 @@ jobs:
- code-quality
- test-targeted
- test-full
- test-installer-extended-formats
- provider-static
- cross-build
- security-audit
Expand Down Expand Up @@ -1231,6 +1297,16 @@ jobs:
check_job "build-vx" "${{ needs.build-vx.result }}"
check_job "code-quality" "${{ needs.code-quality.result }}"
check_job "test-targeted" "${{ needs.test-targeted.result }}"

# Once Rust/CI changes select installer regressions, skipped is not a
# pass either: all three native matrix legs must have actually run.
if [[ "${{ needs.detect-changes.outputs.should_test }}" == "true" &&
"${{ needs.test-installer-extended-formats.result }}" != "success" ]]; then
echo "- ❌ test-installer-extended-formats (required but not successful)" >> $GITHUB_STEP_SUMMARY
failed=1
else
check_job "test-installer-extended-formats" "${{ needs.test-installer-extended-formats.result }}"
fi
check_job "provider-static" "${{ needs.provider-static.result }}"
check_job "security-audit" "${{ needs.security-audit.result }}"

Expand Down
102 changes: 102 additions & 0 deletions scripts/tests/test_installer_extended_formats_workflow.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
"""Keep native installer regression coverage connected to the required CI gate."""

import os
from pathlib import Path
import re
import subprocess
import tempfile
import textwrap
import unittest


ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8")
INSTALLER_JOB = "test-installer-extended-formats"


def job_body(name: str) -> str:
"""Read a top-level job without adding a YAML dependency to script tests."""
match = re.search(rf"^ {re.escape(name)}:\n(.*?)(?=^ [\w-]+:|\Z)",
WORKFLOW, re.MULTILINE | re.DOTALL)
if match is None:
raise AssertionError(f"Missing CI job: {name}")
return match.group(1)


class InstallerExtendedFormatsWorkflowTests(unittest.TestCase):
def test_native_matrix_covers_all_supported_operating_systems(self) -> None:
job = job_body(INSTALLER_JOB)
self.assertEqual(re.findall(r"^ os: (.+)$", job, re.MULTILINE),
["ubuntu-latest", "windows-latest", "macos-latest"])
self.assertIn("runs-on: ${{ matrix.platform.os }}", job)
self.assertIn("fail-fast: false", job)
self.assertIn("needs: [detect-changes, build-vx]", job)
self.assertIn("if: needs.detect-changes.outputs.should_test == 'true'", job)
self.assertIn("name: ${{ matrix.platform.artifact }}", job)

def test_real_regressions_are_feature_enabled_and_cannot_silently_skip(self) -> None:
job = job_body(INSTALLER_JOB)
command = " ".join(job.split(" run: >-\n", 1)[1].split())
self.assertTrue(command.startswith("vx cargo nextest run --locked -p vx-installer "))
for required in ("--features extended-formats", "--test sevenz_sfx_tests",
"--no-fail-fast", "--no-tests fail", "--retries 0"):
self.assertIn(required, command)
for forbidden in ("--no-run", "--target", "--workspace", "|| true",
"continue-on-error", "--run-ignored"):
self.assertNotIn(forbidden, command)
self.assertNotIn("continue-on-error", job)
self.assertIn('install-nextest: "true"', job)

def test_contract_and_success_gate_are_wired_into_ci(self) -> None:
self.assertIn(Path(__file__).name, job_body("detect-changes"))
self.assertIn(f" - {INSTALLER_JOB}\n", job_body("ci-success"))
self.assertIn("if: always()", job_body("ci-success"))

def test_whole_workspace_coverage_is_preserved(self) -> None:
coverage = job_body("coverage")
self.assertIn("github.event.inputs.force_full_ci == 'true'", coverage)
self.assertIn("run: vx just coverage-ci", coverage)
self.assertIn("timeout-minutes: 45", coverage)
self.assertIn("cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info",
(ROOT / "justfile").read_text(encoding="utf-8"))
self.assertIn('check_optional_job "coverage" "${{ needs.coverage.result }}"',
job_body("ci-success"))

def run_success_gate(self, selected: bool, result: str,
coverage: str = "skipped") -> subprocess.CompletedProcess:
# Execute the workflow's actual shell gate, substituting Actions values.
# This tests orchestration only; extraction itself runs natively in CI.
script = textwrap.dedent(job_body("ci-success").split(" run: |\n", 1)[1])
values = {
"needs.detect-changes.outputs.should_test": str(selected).lower(),
f"needs.{INSTALLER_JOB}.result": result,
"needs.coverage.result": coverage,
}
script = re.sub(r"\$\{\{\s*(.*?)\s*\}\}",
lambda match: values.get(match.group(1), "success"), script)
with tempfile.TemporaryDirectory() as directory:
return subprocess.run(
["bash", "-e", "-c", script], check=False, capture_output=True, text=True,
env={**os.environ, "GITHUB_STEP_SUMMARY": str(Path(directory) / "summary")},
)

def test_selected_installer_regressions_require_success(self) -> None:
for result in ("failure", "cancelled", "skipped", ""):
with self.subTest(result=result):
self.assertNotEqual(self.run_success_gate(True, result).returncode, 0)
self.assertEqual(self.run_success_gate(True, "success").returncode, 0)

def test_unselected_installer_regressions_may_skip_but_not_fail(self) -> None:
self.assertEqual(self.run_success_gate(False, "skipped").returncode, 0)
for result in ("failure", "cancelled"):
with self.subTest(result=result):
self.assertNotEqual(self.run_success_gate(False, result).returncode, 0)

def test_coverage_failure_or_cancellation_still_fails_the_gate(self) -> None:
for result in ("failure", "cancelled"):
with self.subTest(result=result):
self.assertNotEqual(self.run_success_gate(True, "success", result).returncode, 0)


if __name__ == "__main__":
unittest.main()
Loading