diff --git a/.editorconfig b/.editorconfig index 30de00caa..03d94e8dc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,17 +1,14 @@ root = true -[*.py] +[*.{py,md,yml,yaml,json,toml}] charset = utf-8 end_of_line = lf indent_style = space -indent_size = 4 insert_final_newline = true trim_trailing_whitespace = true +[*.py] +indent_size = 4 + [*.{md,yml,yaml,json,toml}] -charset = utf-8 -end_of_line = lf -indent_style = space indent_size = 2 -insert_final_newline = true -trim_trailing_whitespace = true diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6ede215d9..d5e15dfbf 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,12 +1,10 @@ name: CodeQL Security Analysis on: - push: - branches: - - main - pull_request: + push: &main-branch-trigger branches: - main + pull_request: *main-branch-trigger schedule: # Run weekly on Monday at 06:00 UTC - cron: "0 6 * * 1" diff --git a/.github/workflows/docker-image-test.yml b/.github/workflows/docker-image-test.yml index 8fd05972a..c06f04844 100644 --- a/.github/workflows/docker-image-test.yml +++ b/.github/workflows/docker-image-test.yml @@ -4,6 +4,7 @@ on: pull_request: paths: - "Dockerfile*" + - "docker-install-rust.sh" - ".dockerignore" - "modelaudit/**" - "packages/modelaudit-picklescan/**" @@ -37,6 +38,7 @@ jobs: filters: | docker: - 'Dockerfile*' + - 'docker-install-rust.sh' - '.dockerignore' - 'modelaudit/**' - 'packages/modelaudit-picklescan/**' @@ -45,10 +47,12 @@ jobs: - '.github/workflows/docker-image-test.yml' full-image: - 'Dockerfile.full' + - 'docker-install-rust.sh' - 'packages/modelaudit-picklescan/**' - '.github/workflows/docker-image-test.yml' tensorflow-image: - 'Dockerfile.tensorflow' + - 'docker-install-rust.sh' - 'requirements-tensorflow.txt' - 'modelaudit/**' - 'packages/modelaudit-picklescan/**' @@ -65,7 +69,8 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Set up Docker Buildx + - &shared-set-up-docker-buildx + name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 - name: Build lightweight image @@ -77,7 +82,7 @@ jobs: load: true cache-from: type=gha,scope=lightweight cache-to: type=gha,mode=max,scope=lightweight - build-args: | + build-args: &inline-cache-build-args | BUILDKIT_INLINE_CACHE=1 - name: Test lightweight container help command @@ -130,8 +135,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - *shared-set-up-docker-buildx - name: Build full image uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 @@ -142,8 +146,7 @@ jobs: load: true cache-from: type=gha,scope=full cache-to: type=gha,mode=max,scope=full - build-args: | - BUILDKIT_INLINE_CACHE=1 + build-args: *inline-cache-build-args timeout-minutes: 60 # Increased timeout for ML dependency build - name: Test full container help command @@ -195,8 +198,7 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - *shared-set-up-docker-buildx - name: Build TensorFlow image uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 09db494e5..487402482 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -24,16 +24,17 @@ jobs: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 5 - outputs: + outputs: &validated-image-outputs image_tag: ${{ steps.validate.outputs.image_tag }} source_ref: ${{ steps.validate.outputs.source_ref }} source_sha: ${{ steps.validate.outputs.source_sha }} environment: name: ghcr-manual-publish - permissions: + permissions: &read-contents-permissions contents: read steps: - - name: Checkout repo + - &shared-checkout-repo + name: Checkout repo uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: ${{ github.event.repository.default_branch }} @@ -56,17 +57,10 @@ jobs: if: github.event_name == 'release' && startsWith(github.event.release.tag_name, 'v') runs-on: ubuntu-latest timeout-minutes: 5 - outputs: - image_tag: ${{ steps.validate.outputs.image_tag }} - source_ref: ${{ steps.validate.outputs.source_ref }} - source_sha: ${{ steps.validate.outputs.source_sha }} - permissions: - contents: read + outputs: *validated-image-outputs + permissions: *read-contents-permissions steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - ref: ${{ github.event.repository.default_branch }} + - *shared-checkout-repo - name: Validate release image tag id: validate diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fc3bf9625..9fde9fd49 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,10 +25,12 @@ jobs: - { os: windows-latest, python-version: "3.12" } - { os: windows-latest, python-version: "3.13" } steps: - - name: Checkout repo + - &checkout-repo + name: Checkout repo uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Install uv + - &install-uv + name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: enable-cache: true @@ -37,7 +39,8 @@ jobs: run: | uv python pin ${{ matrix.python-version }} - - name: Install Rust toolchain + - &install-rust-toolchain + name: Install Rust toolchain run: | rustup toolchain install stable --profile minimal rustup default stable @@ -55,22 +58,15 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - name: Pin Python version run: | uv python pin 3.11 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Sync dependencies run: | @@ -85,13 +81,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Run standalone picklescan Rust tests once run: | diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 617a66579..edd00c713 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -2,7 +2,7 @@ name: Performance Benchmarks on: pull_request: - paths: + paths: &benchmark-trigger-paths - "modelaudit/**" - "packages/modelaudit-picklescan/**" - "tests/benchmarks/**" @@ -17,18 +17,7 @@ on: push: branches: - main - paths: - - "modelaudit/**" - - "packages/modelaudit-picklescan/**" - - "tests/benchmarks/**" - - "tests/helpers/**" - - "tests/conftest.py" - - "tests/test_benchmark_report.py" - - "tests/test_performance_benchmarks.py" - - "scripts/benchmark_report.py" - - "pyproject.toml" - - "uv.lock" - - ".github/workflows/perf.yml" + paths: *benchmark-trigger-paths workflow_dispatch: permissions: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index ca1b0209a..d48bcf06a 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -175,12 +175,13 @@ jobs: if: needs.release-please.outputs.pr_branch != '' runs-on: ubuntu-latest needs: release-please - permissions: + permissions: &read-contents-permissions contents: read outputs: source_sha: ${{ steps.source.outputs.sha }} steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - &checkout-release-metadata + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: ${{ needs.release-please.outputs.pr_branch }} fetch-depth: 1 @@ -194,7 +195,7 @@ jobs: uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - name: Install Rust toolchain for standalone lock refresh - run: | + run: &install-rust-toolchain | rustup toolchain install stable --profile minimal rustup default stable @@ -240,11 +241,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - ref: ${{ needs.release-please.outputs.pr_branch }} - fetch-depth: 1 - persist-credentials: false + - *checkout-release-metadata - name: Download release metadata id: download-metadata @@ -306,22 +303,20 @@ jobs: if: needs.release-please.outputs.release_created == 'true' runs-on: ubuntu-latest needs: release-please - permissions: - contents: read + permissions: *read-contents-permissions outputs: artifact-name: ${{ steps.upload.outputs.artifact-id }} steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Install uv + - &install-uv + name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: enable-cache: true - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + run: *install-rust-toolchain - name: Type check root package with mypy run: | @@ -593,7 +588,7 @@ jobs: - name: Upload build artifacts id: upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: + with: &root-dist-artifact name: dist path: dist/ @@ -602,8 +597,7 @@ jobs: name: Build standalone pickle package (${{ matrix.artifact-suffix }}) runs-on: ${{ matrix.os }} needs: release-please - permissions: - contents: read + permissions: *read-contents-permissions strategy: fail-fast: false matrix: @@ -630,10 +624,7 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - name: Pin Python version run: | @@ -825,15 +816,14 @@ jobs: environment: name: pypi url: https://pypi.org/project/modelaudit/ - permissions: + permissions: &pypi-publish-permissions contents: read id-token: write steps: - - name: Download build artifacts + - &shared-download-build-artifacts + name: Download build artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: dist - path: dist/ + with: *root-dist-artifact - name: Verify the root picklescan dependency is available on PyPI run: | @@ -935,7 +925,8 @@ jobs: raise SystemExit(f"Refusing to publish modelaudit with an unavailable picklescan dependency: {last_status}") PY - - name: Publish to PyPI + - &shared-publish-to-pypi + name: Publish to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: print-hash: true @@ -948,34 +939,30 @@ jobs: environment: name: pypi url: https://pypi.org/project/modelaudit-picklescan/ - permissions: - contents: read - id-token: write + permissions: *pypi-publish-permissions steps: - - name: Download standalone pickle package artifacts + - &shared-download-standalone-pickle-package-artifacts + name: Download standalone pickle package artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: modelaudit-picklescan-dist-* path: dist/ merge-multiple: true - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 - with: - print-hash: true - attestations: true + - *shared-publish-to-pypi verify-picklescan-pypi: if: needs.release-please.outputs.picklescan_release_created == 'true' needs: [publish-picklescan-pypi, release-please] runs-on: ubuntu-latest - permissions: - contents: read + permissions: *read-contents-permissions env: EXPECTED_VERSION: ${{ needs.release-please.outputs.picklescan_version }} steps: - name: Wait for modelaudit-picklescan files on PyPI - run: | + env: + PYPI_PROJECT: modelaudit-picklescan + run: &wait-for-pypi-files | python - <<'PY' import gzip import json @@ -985,16 +972,27 @@ jobs: import zlib version = os.environ["EXPECTED_VERSION"] + project = os.environ["PYPI_PROJECT"] expected_files = { - f"modelaudit_picklescan-{version}-cp310-abi3-macosx_10_12_x86_64.whl", - f"modelaudit_picklescan-{version}-cp310-abi3-macosx_11_0_arm64.whl", - f"modelaudit_picklescan-{version}-cp310-abi3-manylinux_2_28_aarch64.whl", - f"modelaudit_picklescan-{version}-cp310-abi3-manylinux_2_28_x86_64.whl", - f"modelaudit_picklescan-{version}-cp310-abi3-win_amd64.whl", - f"modelaudit_picklescan-{version}.tar.gz", + 'modelaudit-picklescan': { + f"modelaudit_picklescan-{version}-cp310-abi3-macosx_10_12_x86_64.whl", + f"modelaudit_picklescan-{version}-cp310-abi3-macosx_11_0_arm64.whl", + f"modelaudit_picklescan-{version}-cp310-abi3-manylinux_2_28_aarch64.whl", + f"modelaudit_picklescan-{version}-cp310-abi3-manylinux_2_28_x86_64.whl", + f"modelaudit_picklescan-{version}-cp310-abi3-win_amd64.whl", + f"modelaudit_picklescan-{version}.tar.gz", + }, + 'modelaudit': { + f"modelaudit-{version}-py3-none-any.whl", + f"modelaudit-{version}.tar.gz", + }, + }[project] + url_templates = { + "modelaudit-picklescan": ("https://pypi.org/pypi/modelaudit-picklescan/{version}/json", "https://pypi.org/simple/modelaudit-picklescan/"), + "modelaudit": ("https://pypi.org/pypi/modelaudit/{version}/json", "https://pypi.org/simple/modelaudit/"), } - url = f"https://pypi.org/pypi/modelaudit-picklescan/{version}/json" - simple_url = "https://pypi.org/simple/modelaudit-picklescan/" + url, simple_url = url_templates[project] + url = url.format(version=version) deadline = time.monotonic() + 600 last_status = "not checked" @@ -1028,14 +1026,14 @@ jobs: missing_simple = sorted(expected_files - simple_filenames) info_version = payload.get("info", {}).get("version") if info_version == version and not missing and not missing_simple: - print(f"PyPI has modelaudit-picklescan {version}: {sorted(filenames)}") + print(f"PyPI has {project} {version}: {sorted(filenames)}") break last_status = f"version={info_version!r}, missing={missing}, missing_simple={missing_simple}" except Exception as exc: last_status = repr(exc) time.sleep(10) else: - raise SystemExit(f"Timed out waiting for modelaudit-picklescan {version} on PyPI: {last_status}") + raise SystemExit(f"Timed out waiting for {project} {version} on PyPI: {last_status}") PY - name: Install published modelaudit-picklescan and smoke test API @@ -1103,8 +1101,7 @@ jobs: verify-picklescan-pypi, ] runs-on: ubuntu-latest - permissions: - contents: read + permissions: *read-contents-permissions env: EXPECTED_VERSION: ${{ needs.release-please.outputs.version }} EXPECTED_PICKLESCAN_VERSION: ${{ needs.release-please.outputs.picklescan_version }} @@ -1112,64 +1109,9 @@ jobs: PROMPTFOO_DISABLE_TELEMETRY: "1" steps: - name: Wait for modelaudit files on PyPI - run: | - python - <<'PY' - import gzip - import json - import os - import time - import urllib.request - import zlib - - version = os.environ["EXPECTED_VERSION"] - expected_files = { - f"modelaudit-{version}-py3-none-any.whl", - f"modelaudit-{version}.tar.gz", - } - url = f"https://pypi.org/pypi/modelaudit/{version}/json" - simple_url = "https://pypi.org/simple/modelaudit/" - deadline = time.monotonic() + 600 - last_status = "not checked" - - while time.monotonic() < deadline: - try: - with urllib.request.urlopen(url, timeout=20) as response: - payload = json.load(response) - request = urllib.request.Request( - simple_url, - headers={ - "Accept": "application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html; q=0.1, text/html; q=0.01", - "Accept-Encoding": "gzip, deflate", - "Cache-Control": "max-age=0", - }, - ) - with urllib.request.urlopen(request, timeout=20) as response: - simple_body = response.read() - content_encoding = response.headers.get("Content-Encoding", "identity").lower() - if content_encoding == "gzip": - simple_body = gzip.decompress(simple_body) - elif content_encoding == "deflate": - simple_body = zlib.decompress(simple_body) - elif content_encoding not in ("", "identity"): - raise ValueError(f"Unsupported Simple API Content-Encoding: {content_encoding!r}") - simple_payload = json.loads(simple_body) - filenames = {entry["filename"] for entry in payload.get("urls", [])} - simple_filenames = { - entry["filename"] for entry in simple_payload.get("files", []) if not entry.get("yanked", False) - } - missing = sorted(expected_files - filenames) - missing_simple = sorted(expected_files - simple_filenames) - info_version = payload.get("info", {}).get("version") - if info_version == version and not missing and not missing_simple: - print(f"PyPI has modelaudit {version}: {sorted(filenames)}") - break - last_status = f"version={info_version!r}, missing={missing}, missing_simple={missing_simple}" - except Exception as exc: - last_status = repr(exc) - time.sleep(10) - else: - raise SystemExit(f"Timed out waiting for modelaudit {version} on PyPI: {last_status}") - PY + env: + PYPI_PROJECT: modelaudit + run: *wait-for-pypi-files - name: Install published modelaudit and run end-to-end smoke tests run: | @@ -1320,29 +1262,23 @@ jobs: }} needs: [build, publish-pypi, verify-pypi, release-please] runs-on: ubuntu-latest - permissions: + permissions: &provenance-permissions contents: write id-token: write attestations: write steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: - sparse-checkout: | + sparse-checkout: &root-lock-inputs | pyproject.toml uv.lock - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: dist - path: dist/ + - *shared-download-build-artifacts - - name: Generate artifact attestations + - &shared-generate-artifact-attestations + name: Generate artifact attestations uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 with: subject-path: "dist/*" @@ -1362,7 +1298,7 @@ jobs: # Upload wheel/sdist and SBOM to the GitHub Release as an alternative # download location alongside PyPI. --clobber ensures idempotency on re-runs. - name: Upload build artifacts to GitHub Release - env: + env: &github-token-env GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh release upload "${{ needs.release-please.outputs.tag_name }}" dist/* \ @@ -1388,9 +1324,7 @@ jobs: uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: ref: refs/tags/${{ needs.release-please.outputs.tag_name }} - sparse-checkout: | - pyproject.toml - uv.lock + sparse-checkout: *root-lock-inputs persist-credentials: false - name: Verify original root publish run @@ -1548,7 +1482,7 @@ jobs: run-id: ${{ needs.release-please.outputs.root_provenance_run_id }} - name: Verify recovered root artifacts match PyPI - env: + env: &root-version-env EXPECTED_VERSION: ${{ needs.release-please.outputs.version }} run: | python - <<'PY' @@ -1734,14 +1668,10 @@ jobs: predicate: >- {"package":"modelaudit","version":"${{ needs.release-please.outputs.version }}","tag":"${{ needs.release-please.outputs.tag_name }}","source_run_id":"${{ needs.release-please.outputs.root_provenance_run_id }}","source_commit":"${{ steps.verify-source.outputs.source_head_sha }}","tag_commit":"${{ steps.verify-source.outputs.tag_commit_sha }}","pypi_json_url":"https://pypi.org/pypi/modelaudit/${{ needs.release-please.outputs.version }}/json"} - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - name: Generate SBOM from tagged root lockfile - env: - EXPECTED_VERSION: ${{ needs.release-please.outputs.version }} + env: *root-version-env run: | uv export \ --frozen \ @@ -1767,10 +1697,7 @@ jobs: if: needs.release-please.outputs.picklescan_release_created == 'true' needs: [build-picklescan-package, publish-picklescan-pypi, release-please] runs-on: ubuntu-latest - permissions: - contents: write - id-token: write - attestations: write + permissions: *provenance-permissions steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -1778,22 +1705,11 @@ jobs: packages/modelaudit-picklescan/pyproject.toml packages/modelaudit-picklescan/uv.lock - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Download standalone pickle package artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - pattern: modelaudit-picklescan-dist-* - path: dist/ - merge-multiple: true + - *shared-download-standalone-pickle-package-artifacts - - name: Generate artifact attestations - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4 - with: - subject-path: "dist/*" + - *shared-generate-artifact-attestations - name: Generate standalone package SBOM working-directory: packages/modelaudit-picklescan @@ -1809,8 +1725,7 @@ jobs: ls -la ../../dist/*.cdx.json - name: Upload standalone artifacts to GitHub Release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + env: *github-token-env run: | gh release upload "${{ needs.release-please.outputs.picklescan_tag_name }}" dist/* \ --repo "${{ github.repository }}" \ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7506a1290..ef7583b1f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,6 +59,7 @@ jobs: - 'packages/modelaudit-picklescan/**' docker: - 'Dockerfile*' + - 'docker-install-rust.sh' - '.dockerignore' workflows: - '.github/workflows/**' @@ -76,27 +77,32 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo + - &checkout-repo + name: Checkout repo uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - name: Install uv + - &install-uv + name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 with: enable-cache: true - - name: Pin Python version + - &pin-python-312 + name: Pin Python version run: | uv python pin 3.12 - - name: Install Rust toolchain + - &install-rust-toolchain + name: Install Rust toolchain run: | rustup toolchain install stable --profile minimal rustup default stable - - name: Cache Python dependencies + - &cache-python-312-dependencies + name: Cache Python dependencies uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | + path: &python-cache-paths | .venv ~/.cache/pip ~/.cache/uv @@ -105,7 +111,8 @@ jobs: ${{ runner.os }}-uv-py3.12- ${{ runner.os }}-uv- - - name: Sync dependencies + - &sync-all-ci-dependencies + name: Sync dependencies run: | uv sync --extra all-ci @@ -113,10 +120,6 @@ jobs: run: | uv run ruff check modelaudit/ tests/ - - name: Check import organization with Ruff - run: | - uv run ruff check --select I modelaudit/ tests/ - - name: Check formatting with Ruff run: | uv run ruff format --check modelaudit/ tests/ @@ -128,17 +131,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - name: Audit dependencies for vulnerabilities run: | @@ -166,26 +163,15 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Check dependency licenses run: | @@ -312,22 +298,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - name: Check uv.lock is in sync with pyproject.toml - run: | + run: &check-uv-lock | uv lock --check - name: Check standalone picklescan uv.lock is in sync working-directory: packages/modelaudit-picklescan - run: | - uv lock --check + run: *check-uv-lock type-check: name: Type Check @@ -337,13 +318,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - name: Pin Python version run: | @@ -351,26 +328,18 @@ jobs: # syntax valid for the minimum supported Python target. uv python pin 3.10 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Cache Python dependencies uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv + path: *python-cache-paths key: ${{ runner.os }}-uv-py3.10-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-uv-py3.10- ${{ runner.os }}-uv- - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Type checking run: | @@ -392,8 +361,7 @@ jobs: matrix: include: ${{ github.event_name == 'pull_request' && needs.changes.outputs.workflows != 'true' && fromJSON('[{"shard-count":1,"shard-index":0,"shard-name":"1/1"}]') || fromJSON('[{"shard-count":2,"shard-index":0,"shard-name":"1/2"},{"shard-count":2,"shard-index":1,"shard-name":"2/2"}]') }} steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 @@ -402,10 +370,7 @@ jobs: run: | uv python pin 3.11 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Sync dependencies run: | @@ -435,38 +400,27 @@ jobs: matrix: include: ${{ github.event_name == 'pull_request' && needs.changes.outputs.workflows != 'true' && fromJSON('[{"python-version":"3.10","shard-count":1,"shard-index":0,"shard-name":"1/1"},{"python-version":"3.12","shard-count":1,"shard-index":0,"shard-name":"1/1"},{"python-version":"3.13","shard-count":1,"shard-index":0,"shard-name":"1/1"}]') || fromJSON('[{"python-version":"3.10","shard-count":2,"shard-index":0,"shard-name":"1/2"},{"python-version":"3.10","shard-count":2,"shard-index":1,"shard-name":"2/2"},{"python-version":"3.11","shard-count":2,"shard-index":0,"shard-name":"1/2"},{"python-version":"3.11","shard-count":2,"shard-index":1,"shard-name":"2/2"},{"python-version":"3.12","shard-count":2,"shard-index":0,"shard-name":"1/2"},{"python-version":"3.12","shard-count":2,"shard-index":1,"shard-name":"2/2"},{"python-version":"3.13","shard-count":2,"shard-index":0,"shard-name":"1/2"},{"python-version":"3.13","shard-count":2,"shard-index":1,"shard-name":"2/2"}]') }} steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version + - &pin-matrix-python + name: Pin Python version run: | uv python pin ${{ matrix.python-version }} - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Cache Python dependencies uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv + path: *python-cache-paths key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-uv-py${{ matrix.python-version }}- ${{ runner.os }}-uv- - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Run fast tests with fail-fast if: github.event_name == 'pull_request' && needs.changes.outputs.workflows != 'true' @@ -493,38 +447,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - - name: Cache Python dependencies - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 - with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv - key: ${{ runner.os }}-uv-py3.12-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-uv-py3.12- - ${{ runner.os }}-uv- + - *cache-python-312-dependencies - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Run slow and integration tests run: | @@ -548,38 +481,17 @@ jobs: matrix: shard: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - - name: Cache Python dependencies - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 - with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv - key: ${{ runner.os }}-uv-py3.12-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-uv-py3.12- - ${{ runner.os }}-uv- + - *cache-python-312-dependencies - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Run branch coverage shard run: | @@ -615,30 +527,18 @@ jobs: # Reduced matrix for PRs, full matrix for main and merge_group runs include: ${{ github.event_name == 'pull_request' && fromJSON('[{"python-version":"3.10","numpy-mode":"1.x"},{"python-version":"3.11","numpy-mode":"2.x"},{"python-version":"3.12","numpy-mode":"2.x"}]') || fromJSON('[{"python-version":"3.10","numpy-mode":"1.x"},{"python-version":"3.11","numpy-mode":"2.x"},{"python-version":"3.12","numpy-mode":"2.x"},{"python-version":"3.13","numpy-mode":"2.x"}]') }} steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin ${{ matrix.python-version }} + - *pin-matrix-python - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Cache Python dependencies uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv + path: *python-cache-paths key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-numpy${{ matrix.numpy-mode }}-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} restore-keys: | ${{ runner.os }}-uv-py${{ matrix.python-version }}-numpy${{ matrix.numpy-mode }}- @@ -696,22 +596,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Install WITHOUT TensorFlow run: | @@ -757,8 +648,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - name: Install protoc 33.5 run: | @@ -820,22 +710,13 @@ jobs: sevenzip, ] steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - name: Install with ${{ matrix.extra }} extra run: | @@ -861,38 +742,17 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin 3.12 + - *pin-python-312 - - name: Install Rust toolchain - run: | - rustup toolchain install stable --profile minimal - rustup default stable + - *install-rust-toolchain - - name: Cache Python dependencies - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 - with: - path: | - .venv - ~/.cache/pip - ~/.cache/uv - key: ${{ runner.os }}-uv-py3.12-${{ hashFiles('**/uv.lock', '**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-uv-py3.12- - ${{ runner.os }}-uv- + - *cache-python-312-dependencies - - name: Sync dependencies - run: | - uv sync --extra all-ci + - *sync-all-ci-dependencies - name: Build standalone pickle package run: | @@ -952,17 +812,11 @@ jobs: run: working-directory: packages/modelaudit-picklescan steps: - - name: Checkout repo - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + - *checkout-repo - - name: Install uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - enable-cache: true + - *install-uv - - name: Pin Python version - run: | - uv python pin ${{ matrix.python-version }} + - *pin-matrix-python - name: Install Rust toolchain run: | @@ -982,8 +836,7 @@ jobs: ${{ runner.os }}-cargo-picklescan- - name: Check standalone package lock is in sync - run: | - uv lock --check + run: *check-uv-lock - name: Check Rust scanner formatting run: | @@ -995,7 +848,6 @@ jobs: - name: Check Rust scanner MSRV run: | - rustup toolchain install 1.83.0 --profile minimal cargo +1.83.0 check --manifest-path Cargo.toml --locked - name: Lint Rust scanner crate @@ -1010,10 +862,6 @@ jobs: run: | uv run --with 'ruff==0.16.9' ruff check src tests - - name: Check standalone package import organization with Ruff - run: | - uv run --with 'ruff==0.16.9' ruff check --select I src tests - - name: Check standalone package formatting with Ruff run: | uv run --with 'ruff==0.16.9' ruff format --check src tests diff --git a/Dockerfile b/Dockerfile index f13e40fa6..618f914f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,23 +17,12 @@ WORKDIR /build COPY pyproject.toml README.md ./ COPY packages/modelaudit-picklescan ./packages/modelaudit-picklescan COPY modelaudit ./modelaudit +COPY docker-install-rust.sh ./ RUN apt-get update \ && apt-get install --yes --no-install-recommends --only-upgrade libc-bin libc6 libcap2 libssl3t64 libsystemd0 libudev1 \ && apt-get install --yes --no-install-recommends build-essential ca-certificates curl \ - && case "${TARGETARCH:=$(dpkg --print-architecture)}" in \ - amd64) rustup_target="x86_64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_X86_64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - arm64) rustup_target="aarch64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_AARCH64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - *) echo "Unsupported Docker build architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac \ - && curl --proto '=https' --tlsv1.2 -fsSL \ - "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${rustup_target}/rustup-init" \ - -o /tmp/rustup-init \ - && printf '%s %s\n' "${rustup_sha256}" /tmp/rustup-init > /tmp/rustup-init.sha256 \ - && sha256sum -c /tmp/rustup-init.sha256 \ - && chmod +x /tmp/rustup-init \ - && /tmp/rustup-init -y --profile minimal --default-toolchain "${PICKLESCAN_RUST_TOOLCHAIN}" \ - && rm -f /tmp/rustup-init /tmp/rustup-init.sha256 \ + && . ./docker-install-rust.sh \ && PATH="/root/.cargo/bin:${PATH}" pip wheel --no-cache-dir --wheel-dir /wheels \ ./packages/modelaudit-picklescan \ . \ diff --git a/Dockerfile.full b/Dockerfile.full index c00c5f36c..7b76390c0 100644 --- a/Dockerfile.full +++ b/Dockerfile.full @@ -19,19 +19,7 @@ COPY . . RUN apt-get update \ && apt-get install --yes --no-install-recommends --only-upgrade libc-bin libc6 libcap2 libssl3t64 libsystemd0 libudev1 \ && apt-get install --yes --no-install-recommends build-essential ca-certificates curl \ - && case "${TARGETARCH:=$(dpkg --print-architecture)}" in \ - amd64) rustup_target="x86_64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_X86_64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - arm64) rustup_target="aarch64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_AARCH64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - *) echo "Unsupported Docker build architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac \ - && curl --proto '=https' --tlsv1.2 -fsSL \ - "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${rustup_target}/rustup-init" \ - -o /tmp/rustup-init \ - && printf '%s %s\n' "${rustup_sha256}" /tmp/rustup-init > /tmp/rustup-init.sha256 \ - && sha256sum -c /tmp/rustup-init.sha256 \ - && chmod +x /tmp/rustup-init \ - && /tmp/rustup-init -y --profile minimal --default-toolchain "${PICKLESCAN_RUST_TOOLCHAIN}" \ - && rm -f /tmp/rustup-init /tmp/rustup-init.sha256 \ + && . ./docker-install-rust.sh \ && PATH="/root/.cargo/bin:${PATH}" pip wheel --no-cache-dir --wheel-dir /wheels \ ./packages/modelaudit-picklescan \ . \ diff --git a/Dockerfile.tensorflow b/Dockerfile.tensorflow index 2678cb926..f61f8bc25 100644 --- a/Dockerfile.tensorflow +++ b/Dockerfile.tensorflow @@ -18,22 +18,11 @@ COPY pyproject.toml README.md ./ COPY requirements-tensorflow.txt ./ COPY packages/modelaudit-picklescan ./packages/modelaudit-picklescan COPY modelaudit ./modelaudit +COPY docker-install-rust.sh ./ RUN apt-get update \ && apt-get install --yes --no-install-recommends build-essential ca-certificates curl \ - && case "${TARGETARCH:=$(dpkg --print-architecture)}" in \ - amd64) rustup_target="x86_64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_X86_64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - arm64) rustup_target="aarch64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_AARCH64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ - *) echo "Unsupported Docker build architecture: ${TARGETARCH}" >&2; exit 1 ;; \ - esac \ - && curl --proto '=https' --tlsv1.2 -fsSL \ - "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${rustup_target}/rustup-init" \ - -o /tmp/rustup-init \ - && printf '%s %s\n' "${rustup_sha256}" /tmp/rustup-init > /tmp/rustup-init.sha256 \ - && sha256sum -c /tmp/rustup-init.sha256 \ - && chmod +x /tmp/rustup-init \ - && /tmp/rustup-init -y --profile minimal --default-toolchain "${PICKLESCAN_RUST_TOOLCHAIN}" \ - && rm -f /tmp/rustup-init /tmp/rustup-init.sha256 \ + && . ./docker-install-rust.sh \ && PATH="/root/.cargo/bin:${PATH}" pip wheel --no-cache-dir --no-deps --wheel-dir /wheels \ ./packages/modelaudit-picklescan \ && pip install --no-cache-dir --prefix=/install -c requirements-tensorflow.txt \ diff --git a/docker-compose.yml b/docker-compose.yml index 8a8e8eebd..6cf2248e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: modelaudit-base: build: . - volumes: + volumes: &model-volume - ./models:/models command: scan --help @@ -11,14 +11,12 @@ services: build: context: . dockerfile: Dockerfile.tensorflow - volumes: - - ./models:/models + volumes: *model-volume command: scan --help modelaudit-full: build: context: . dockerfile: Dockerfile.full - volumes: - - ./models:/models + volumes: *model-volume command: scan --help diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 772de5602..fb0e70584 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -21,10 +21,5 @@ if [[ "$1" == "-c" ]] || [[ "$1" == "-m" ]]; then exec python "$@" fi -# If no arguments or first argument looks like a modelaudit option, run modelaudit -if [[ $# -eq 0 ]] || [[ "$1" == "--"* ]] || [[ "$1" == "-"* ]] || [[ "$1" == "scan" ]]; then - exec modelaudit "$@" -fi - -# For file paths or other arguments, assume it's a scan command +# Run ModelAudit for every other invocation. exec modelaudit "$@" \ No newline at end of file diff --git a/docker-install-rust.sh b/docker-install-rust.sh new file mode 100644 index 000000000..d83495031 --- /dev/null +++ b/docker-install-rust.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Sourced by builder stages to preserve TARGETARCH fallback assignment. + +case "${TARGETARCH:=$(dpkg --print-architecture)}" in \ + amd64) rustup_target="x86_64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_X86_64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ + arm64) rustup_target="aarch64-unknown-linux-gnu"; rustup_sha256="${RUSTUP_INIT_AARCH64_UNKNOWN_LINUX_GNU_SHA256}" ;; \ + *) echo "Unsupported Docker build architecture: ${TARGETARCH}" >&2; exit 1 ;; \ +esac \ +&& curl --proto '=https' --tlsv1.2 -fsSL \ + "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/${rustup_target}/rustup-init" \ + -o /tmp/rustup-init \ +&& printf '%s %s\n' "${rustup_sha256}" /tmp/rustup-init > /tmp/rustup-init.sha256 \ +&& sha256sum -c /tmp/rustup-init.sha256 \ +&& chmod +x /tmp/rustup-init \ +&& /tmp/rustup-init -y --profile minimal --default-toolchain "${PICKLESCAN_RUST_TOOLCHAIN}" \ +&& rm -f /tmp/rustup-init /tmp/rustup-init.sha256 diff --git a/pyproject.toml b/pyproject.toml index d81ace34b..1191c4245 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -354,11 +354,6 @@ module = "modelaudit.scanners.*" # Scanners use dynamic patterns that need Any warn_return_any = false -[[tool.mypy.overrides]] -module = "modelaudit.suspicious_symbols" -# Runtime validation needs to check types even with annotations -warn_unreachable = false - [[tool.mypy.overrides]] module = "modelaudit.scanners.oci_layer_scanner" # Runtime validation needs to check types even with annotations diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py index 2a6492446..de5da141f 100644 --- a/scripts/benchmark_report.py +++ b/scripts/benchmark_report.py @@ -114,10 +114,6 @@ def _merged_record_context( return workload, target, size, files -def _format_change(delta_ratio: float) -> str: - return f"{delta_ratio:+.1%}" - - def _build_summary( current: dict[str, BenchmarkRecord], baseline: dict[str, BenchmarkRecord] | None, @@ -216,7 +212,7 @@ def _build_summary( total_delta_ratio = 0.0 if baseline_total == 0 else (current_total - baseline_total) / baseline_total lines.append( f"Aggregate shared-benchmark median: {_format_duration(baseline_total)} " - f"-> {_format_duration(current_total)} ({_format_change(total_delta_ratio)})." + f"-> {_format_duration(current_total)} ({f'{total_delta_ratio:+.1%}'})." ) top_regressions = [row for row in sorted_rows if row.status == "regression"][:3] @@ -225,7 +221,7 @@ def _build_summary( lines.append("Top regressions:") for row in top_regressions: lines.append( - f"- `{row.name}` {_format_change(row.delta_ratio)} " + f"- `{row.name}` {f'{row.delta_ratio:+.1%}'} " f"({_format_duration(row.baseline_median)} -> {_format_duration(row.current_median)}, " f"{row.workload}, {row.target}, size={row.size}, files={row.files})" ) @@ -236,7 +232,7 @@ def _build_summary( lines.append("Top improvements:") for row in top_improvements: lines.append( - f"- `{row.name}` {_format_change(row.delta_ratio)} " + f"- `{row.name}` {f'{row.delta_ratio:+.1%}'} " f"({_format_duration(row.baseline_median)} -> {_format_duration(row.current_median)}, " f"{row.workload}, {row.target}, size={row.size}, files={row.files})" ) @@ -249,7 +245,7 @@ def _build_summary( lines.append( f"| `{row.workload}` | `{row.name}` | `{row.target}` | {row.size} | {row.files} | " f"{_format_duration(row.baseline_median)} | {_format_duration(row.current_median)} | " - f"{_format_change(row.delta_ratio)} | {row.status} |" + f"{f'{row.delta_ratio:+.1%}'} | {row.status} |" ) if new_in_current: diff --git a/scripts/compile_tensorflow_protos.sh b/scripts/compile_tensorflow_protos.sh index 40be0893b..ebf9fad7e 100755 --- a/scripts/compile_tensorflow_protos.sh +++ b/scripts/compile_tensorflow_protos.sh @@ -71,30 +71,19 @@ echo "Compiling protobuf files..." COMPILED=0 FAILED=0 -# Compile ALL proto files in the framework directory -echo " Compiling framework protos..." -for proto in tensorflow/core/framework/*.proto; do - if [[ -f "$proto" ]]; then - if protoc --python_out="$OUTPUT_DIR" -I. "$proto" 2>&1; then - COMPILED=$((COMPILED + 1)) - else - echo " WARNING: Failed to compile $proto" - FAILED=$((FAILED + 1)) +# Compile ALL proto files in the framework and protobuf directories. +for proto_group in framework protobuf; do + echo " Compiling $proto_group protos..." + for proto in tensorflow/core/"$proto_group"/*.proto; do + if [[ -f "$proto" ]]; then + if protoc --python_out="$OUTPUT_DIR" -I. "$proto" 2>&1; then + COMPILED=$((COMPILED + 1)) + else + echo " WARNING: Failed to compile $proto" + FAILED=$((FAILED + 1)) + fi fi - fi -done - -# Compile ALL proto files in the protobuf directory -echo " Compiling protobuf protos..." -for proto in tensorflow/core/protobuf/*.proto; do - if [[ -f "$proto" ]]; then - if protoc --python_out="$OUTPUT_DIR" -I. "$proto" 2>&1; then - COMPILED=$((COMPILED + 1)) - else - echo " WARNING: Failed to compile $proto" - FAILED=$((FAILED + 1)) - fi - fi + done done echo "" diff --git a/scripts/large_pickle_corpus_qa.py b/scripts/large_pickle_corpus_qa.py index 72598b60f..634dd5559 100644 --- a/scripts/large_pickle_corpus_qa.py +++ b/scripts/large_pickle_corpus_qa.py @@ -440,10 +440,6 @@ class NormalizedResult: } -def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - - def _json_default(value: Any) -> Any: if isinstance(value, Path): return str(value) @@ -547,32 +543,12 @@ def _tool_path(spec: ToolSpec, tools_root: Path) -> Path: return tools_root / default_path -def _tool_dirty_status(path: Path) -> str: - return _repo_git(["status", "--short"], cwd=path).stdout.strip() - - -def _git_describe(path: Path) -> str: - return _repo_git(["describe", "--tags", "--always", "--dirty"], cwd=path).stdout.strip() - - -def _git_branch(path: Path) -> str: - return _repo_git(["branch", "--show-current"], cwd=path).stdout.strip() - - -def _git_remote(path: Path) -> str: - return _repo_git(["remote", "get-url", "origin"], cwd=path).stdout.strip() - - -def _git_head(path: Path) -> str: - return _repo_git(["rev-parse", "HEAD"], cwd=path).stdout.strip() - - def _git_default_branch(path: Path) -> str: result = _repo_git(["symbolic-ref", "refs/remotes/origin/HEAD"], cwd=path, check=False) ref = result.stdout.strip() if ref.startswith("refs/remotes/origin/"): return ref.removeprefix("refs/remotes/origin/") - return _git_branch(path) or "main" + return _repo_git(["branch", "--show-current"], cwd=path).stdout.strip() or "main" def _sync_tool(spec: ToolSpec, *, tools_root: Path, allow_dirty: bool, skip_pull: bool) -> dict[str, Any]: @@ -583,18 +559,18 @@ def _sync_tool(spec: ToolSpec, *, tools_root: Path, allow_dirty: bool, skip_pull if not (path / ".git").exists(): raise ValueError(f"{path} exists but is not a Git repository") - remote = _git_remote(path) + remote = _repo_git(["remote", "get-url", "origin"], cwd=path).stdout.strip() if remote != spec.repo_url: raise ValueError(f"{spec.name} remote mismatch: expected {spec.repo_url}, got {remote}") - dirty = _tool_dirty_status(path) + dirty = _repo_git(["status", "--short"], cwd=path).stdout.strip() if dirty and not allow_dirty: raise ValueError(f"{spec.name} worktree is dirty:\n{dirty}") if not skip_pull: _repo_git(["fetch", "--tags", "origin"], cwd=path) default_branch = _git_default_branch(path) - current_branch = _git_branch(path) + current_branch = _repo_git(["branch", "--show-current"], cwd=path).stdout.strip() if not dirty and current_branch and current_branch != default_branch: _repo_git(["checkout", default_branch], cwd=path) _repo_git(["pull", "--ff-only"], cwd=path) @@ -603,12 +579,12 @@ def _sync_tool(spec: ToolSpec, *, tools_root: Path, allow_dirty: bool, skip_pull "name": spec.name, "path": str(path), "repo_url": spec.repo_url, - "remote": _git_remote(path), - "branch": _git_branch(path), - "commit": _git_head(path), - "describe": _git_describe(path), - "dirty": _tool_dirty_status(path), - "synced_at": _now_iso(), + "remote": _repo_git(["remote", "get-url", "origin"], cwd=path).stdout.strip(), + "branch": _repo_git(["branch", "--show-current"], cwd=path).stdout.strip(), + "commit": _repo_git(["rev-parse", "HEAD"], cwd=path).stdout.strip(), + "describe": _repo_git(["describe", "--tags", "--always", "--dirty"], cwd=path).stdout.strip(), + "dirty": _repo_git(["status", "--short"], cwd=path).stdout.strip(), + "synced_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), } @@ -634,10 +610,6 @@ def _select_entries( return entries -def _entry_local_path(corpus_root: Path, entry: CorpusEntry) -> Path: - return corpus_root / "raw" / entry.id / entry.path - - def _validated_artifact_id(value: object) -> str: artifact_id = str(value) if ( @@ -675,26 +647,6 @@ def _contained_output_path(root: Path, *parts: str | Path) -> Path: return candidate -def _validated_lock_entry_path(entry: Mapping[str, Any]) -> tuple[str, Path]: - return _validated_artifact_id(entry["id"]), _validated_remote_path(entry["path"]) - - -def _entry_to_lock(entry: CorpusEntry, *, corpus_root: Path) -> dict[str, Any]: - return { - **asdict(entry), - "revision": "main", - "remote_size_bytes": None, - "sha256": None, - "etag": None, - "license": None, - "source_url": f"https://huggingface.co/{entry.repo_id}/tree/main", - "downloaded_at": None, - "local_path": str(_entry_local_path(corpus_root, entry)), - "preflight_status": "pending", - "preflight_error": None, - } - - def _hf_file_metadata(repo_id: str, filename: str, *, revision: str) -> dict[str, Any]: try: from huggingface_hub import HfApi @@ -739,10 +691,6 @@ def _load_lock_entries(lock_path: Path) -> list[dict[str, Any]]: LOCK_CORE_KEYS = {"schema_version", "created_at", "tier", "entry_count", "entries"} -def _lock_extra_metadata(payload: Mapping[str, Any]) -> dict[str, Any]: - return {str(key): value for key, value in payload.items() if key not in LOCK_CORE_KEYS} - - def _write_lock( lock_path: Path, entries: list[dict[str, Any]], @@ -752,7 +700,7 @@ def _write_lock( ) -> None: payload = { "schema_version": 1, - "created_at": _now_iso(), + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "tier": tier, "entry_count": len(entries), "entries": entries, @@ -765,7 +713,7 @@ def _write_lock( def _finalize_downloaded_entry(entry: dict[str, Any], local_path: Path) -> dict[str, Any]: entry["local_path"] = str(local_path) entry["sha256"] = _sha256_file(local_path) - entry["downloaded_at"] = _now_iso() + entry["downloaded_at"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") entry["downloaded_size_bytes"] = local_path.stat().st_size return entry @@ -815,7 +763,7 @@ def _download_entry( direct_fallback: bool, direct_only: bool, ) -> dict[str, Any]: - artifact_id, relative_path = _validated_lock_entry_path(entry) + artifact_id, relative_path = (_validated_artifact_id(entry["id"]), _validated_remote_path(entry["path"])) try: from huggingface_hub import hf_hub_download @@ -895,24 +843,18 @@ def _legacy_verdict(result: ScanResult) -> str: return "unknown" -def _normalize_scan_result(result: ScanResult, *, engine: str) -> NormalizedResult: - return NormalizedResult( - engine=engine, - status=_legacy_status(result), - verdict=_legacy_verdict(result), - success=bool(result.success), - warning_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.WARNING), - critical_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.CRITICAL), - info_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.INFO), - rule_codes=tuple(sorted(issue.rule_code for issue in result.issues if issue.rule_code)), - messages=tuple(sorted(issue.message for issue in result.issues)), - metadata=_json_clean(dict(result.metadata)), - ) - - -def _normalize_package_report(report: PickleReport, *, engine: str) -> NormalizedResult: - return NormalizedResult( - engine=engine, +def _scan_package(path: Path, *, engine: str, artifact_id: str) -> dict[str, Any]: + if engine != "rust": + raise ValueError(f"unsupported picklescan engine after Rust migration: {engine}") + started = time.monotonic() + if artifact_id == "V10": + report = StandalonePickleScanner(options=ScanOptions(max_opcodes=1)).scan_file(path) + else: + report = package_scan_file(path) + duration = time.monotonic() - started + normalized_engine = f"package:{engine}" + normalized = NormalizedResult( + engine=normalized_engine, status=report.status.value, verdict=report.verdict.value, success=report.status == ScanStatus.COMPLETE @@ -924,18 +866,6 @@ def _normalize_package_report(report: PickleReport, *, engine: str) -> Normalize messages=tuple(sorted(finding.message for finding in report.findings)), metadata=_json_clean(dict(report.metadata)), ) - - -def _scan_package(path: Path, *, engine: str, artifact_id: str) -> dict[str, Any]: - if engine != "rust": - raise ValueError(f"unsupported picklescan engine after Rust migration: {engine}") - started = time.monotonic() - if artifact_id == "V10": - report = StandalonePickleScanner(options=ScanOptions(max_opcodes=1)).scan_file(path) - else: - report = package_scan_file(path) - duration = time.monotonic() - started - normalized = _normalize_package_report(report, engine=f"package:{engine}") return { "artifact_id": artifact_id, "scanner": "modelaudit-picklescan", @@ -960,7 +890,19 @@ def _scan_root(path: Path, *, engine: str, root_mode: str, artifact_id: str) -> else: raise ValueError(f"unsupported root mode: {root_mode}") duration = time.monotonic() - started - normalized = _normalize_scan_result(result, engine=f"root:{root_mode}:{engine}") + normalized_engine = f"root:{root_mode}:{engine}" + normalized = NormalizedResult( + engine=normalized_engine, + status=_legacy_status(result), + verdict=_legacy_verdict(result), + success=bool(result.success), + warning_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.WARNING), + critical_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.CRITICAL), + info_count=sum(1 for issue in result.issues if issue.severity == IssueSeverity.INFO), + rule_codes=tuple(sorted(issue.rule_code for issue in result.issues if issue.rule_code)), + messages=tuple(sorted(issue.message for issue in result.issues)), + metadata=_json_clean(dict(result.metadata)), + ) return { "artifact_id": artifact_id, "scanner": "modelaudit-root", @@ -979,13 +921,6 @@ def _stable_report_dict(report: PickleReport) -> dict[str, Any]: return cleaned -def _tool_command(spec: ToolSpec, *, tool_path: Path, artifact_path: Path, output_path: Path) -> list[str]: - return [ - part.format(path=str(tool_path), artifact=str(artifact_path), output=str(output_path)) - for part in spec.invocation - ] - - def _third_party_verdict(tool: str, exit_code: int, stdout: str, stderr: str, output_json: Path) -> str: combined = f"{stdout}\n{stderr}".lower() if "traceback (most recent call last)" in combined or "unhandled exception" in combined: @@ -1042,7 +977,9 @@ def _scan_third_party( tool_path = _tool_path(spec, tools_root) output_path = _contained_output_path(run_dir, "third-party-raw", artifact_id, f"{spec.name}.json") output_path.parent.mkdir(parents=True, exist_ok=True) - command = _tool_command(spec, tool_path=tool_path, artifact_path=path, output_path=output_path) + command = [ + part.format(path=str(tool_path), artifact=str(path), output=str(output_path)) for part in spec.invocation + ] try: completed, duration = _run_command(command, timeout_s=timeout_s) status = "complete" @@ -1066,7 +1003,9 @@ def _scan_third_party( "artifact_id": artifact_id, "tool": spec.name, "tool_path": str(tool_path), - "tool_commit": _git_head(tool_path) if (tool_path / ".git").exists() else None, + "tool_commit": _repo_git(["rev-parse", "HEAD"], cwd=tool_path).stdout.strip() + if (tool_path / ".git").exists() + else None, "command": command, "status": status, "verdict": _third_party_verdict(spec.name, exit_code, stdout, stderr, output_path), @@ -1116,14 +1055,6 @@ def _member_file_path(run_dir: Path, artifact_id: str, member_name: str, *, memb return _contained_output_path(run_dir, "members", artifact_id, safe_member_name) -def _malicious_reduce_payload() -> bytes: - return raw_os_system_reduce_payload() - - -def _stack_global_payload() -> bytes: - return raw_stack_global_eval_reduce_payload() - - def raw_os_system_reduce_payload() -> bytes: return b"\x80\x04cos\nsystem\n\x8c\x0cecho qa-noop\x85R." @@ -1134,8 +1065,8 @@ def raw_stack_global_eval_reduce_payload() -> bytes: def _write_synthetic_variants(output_dir: Path) -> list[dict[str, Any]]: output_dir.mkdir(parents=True, exist_ok=True) - malicious = _malicious_reduce_payload() - stack_global = _stack_global_payload() + malicious = raw_os_system_reduce_payload() + stack_global = raw_stack_global_eval_reduce_payload() nested = pickle.dumps({"outer": malicious}, protocol=4) base64_payload = __import__("base64").b64encode(malicious).decode("ascii") hex_payload = malicious.hex() @@ -1179,7 +1110,7 @@ def _write_synthetic_variants(output_dir: Path) -> list[dict[str, Any]]: "path": str(path), "size_bytes": path.stat().st_size, "sha256": _sha256_file(path), - "created_at": _now_iso(), + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), } ) @@ -1195,7 +1126,7 @@ def _write_synthetic_variants(output_dir: Path) -> list[dict[str, Any]]: "path": str(zip_variant), "size_bytes": zip_variant.stat().st_size, "sha256": _sha256_file(zip_variant), - "created_at": _now_iso(), + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), } ) @@ -1208,7 +1139,7 @@ def _write_synthetic_variants(output_dir: Path) -> list[dict[str, Any]]: "path": str(malformed), "size_bytes": malformed.stat().st_size, "sha256": _sha256_file(malformed), - "created_at": _now_iso(), + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), } ) _write_json(output_dir / "synthetic-manifest.json", {"entries": records}) @@ -1218,7 +1149,7 @@ def _write_synthetic_variants(output_dir: Path) -> list[dict[str, Any]]: def _environment_payload(*, tools: Sequence[dict[str, Any]] | None = None) -> dict[str, Any]: git_status = _repo_git(["status", "--short"], cwd=REPO_ROOT, check=False).stdout.strip() return { - "created_at": _now_iso(), + "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "repo_root": str(REPO_ROOT), "git_commit": _repo_git(["rev-parse", "HEAD"], cwd=REPO_ROOT, check=False).stdout.strip(), "git_branch": _repo_git(["branch", "--show-current"], cwd=REPO_ROOT, check=False).stdout.strip(), @@ -1277,7 +1208,19 @@ def cmd_preflight(args: argparse.Namespace) -> int: tier=args.tier, include_replacements=args.include_replacements, ): - record = _entry_to_lock(entry, corpus_root=corpus_root) + record = { + **asdict(entry), + "revision": "main", + "remote_size_bytes": None, + "sha256": None, + "etag": None, + "license": None, + "source_url": f"https://huggingface.co/{entry.repo_id}/tree/main", + "downloaded_at": None, + "local_path": str(corpus_root / "raw" / entry.id / entry.path), + "preflight_status": "pending", + "preflight_error": None, + } if not args.offline: try: record.update(_hf_file_metadata(entry.repo_id, entry.path, revision=args.revision)) @@ -1348,7 +1291,7 @@ def cmd_download(args: argparse.Namespace) -> int: updated.append(entry) continue try: - _validated_lock_entry_path(entry) + (_validated_artifact_id(entry["id"]), _validated_remote_path(entry["path"])) remote_size = entry.get("remote_size_bytes") if ( budget_bytes is not None @@ -1378,7 +1321,7 @@ def cmd_download(args: argparse.Namespace) -> int: lock_path, updated, tier=str(lock_payload.get("tier", "custom")), - extra=_lock_extra_metadata(lock_payload), + extra={str(key): value for key, value in lock_payload.items() if key not in LOCK_CORE_KEYS}, ) print(f"Updated {lock_path}; downloaded {downloaded_bytes} bytes") return 1 if download_failed else 0 @@ -1394,7 +1337,7 @@ def cmd_classify(args: argparse.Namespace) -> int: lock_path, entries, tier=str(lock_payload.get("tier", "custom")), - extra=_lock_extra_metadata(lock_payload), + extra={str(key): value for key, value in lock_payload.items() if key not in LOCK_CORE_KEYS}, ) print(f"Classified {len(entries)} entries in {lock_path}") return 0 @@ -1575,15 +1518,7 @@ def cmd_scan(args: argparse.Namespace) -> int: finally: logging.disable(previous_logging_disable_level) - _write_json(run_dir / "parity-drift.json", _build_parity_drift(all_scan_rows)) - _write_json( - run_dir / "third-party-differential.json", - _build_third_party_differential(all_third_party_rows, scan_rows=all_scan_rows), - ) - _write_json(run_dir / "coverage-matrix.json", _build_coverage_matrix(all_scan_rows)) - _write_json(run_dir / "benchmark-results.json", _build_benchmark_results(all_scan_rows, all_third_party_rows)) - _write_benchmark_csv(run_dir / "benchmark-summary.csv", all_scan_rows, all_third_party_rows) - _write_report(run_dir, all_scan_rows, all_third_party_rows) + _write_qa_outputs(run_dir, all_scan_rows, all_third_party_rows) print(f"Wrote QA run to {run_dir}") scan_error_count = sum(1 for row in all_scan_rows if _scan_row_has_error(row)) if scan_error_count: @@ -1602,17 +1537,6 @@ def _scan_row_has_error(row: Mapping[str, Any]) -> bool: return isinstance(result, Mapping) and result.get("status") == "error" -def _rows_by_artifact_and_scanner(rows: Iterable[Mapping[str, Any]]) -> dict[tuple[str, str], dict[str, Any]]: - indexed: dict[tuple[str, str], dict[str, Any]] = {} - for row in rows: - result = row.get("result") - if not isinstance(result, Mapping): - continue - key = (str(row.get("artifact_id")), f"{row.get('scanner')}:{row.get('mode')}") - indexed[key] = dict(row) - return indexed - - def _result_rank(result: Mapping[str, Any]) -> int: verdict = result.get("verdict") return {"clean": 0, "suspicious": 1, "malicious": 2, "unknown": -1}.get(str(verdict), -1) @@ -1884,7 +1808,7 @@ def _write_report( lines = [ "# PickleScan Rust Large-Corpus QA Report", "", - f"Generated: {_now_iso()}", + f"Generated: {datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')}", "", "## Summary", "", @@ -1894,38 +1818,31 @@ def _write_report( f"- Third-party failure count: {third_party['failure_count']}", f"- Coverage missing: {', '.join(coverage['missing']) if coverage['missing'] else 'none'}", "", - "## Parity Drift", - "", - "```json", - json.dumps(drift, indent=2, sort_keys=True, default=_json_default), - "```", - "", - "## Third-Party Differential", - "", - "```json", - json.dumps(third_party, indent=2, sort_keys=True, default=_json_default), - "```", - "", - "## Coverage Matrix", - "", - "```json", - json.dumps(coverage, indent=2, sort_keys=True, default=_json_default), - "```", - "", - "## Benchmark Summary", - "", - "```json", - json.dumps(benchmark, indent=2, sort_keys=True, default=_json_default), - "```", - "", ] + for title, payload in ( + ("Parity Drift", drift), + ("Third-Party Differential", third_party), + ("Coverage Matrix", coverage), + ("Benchmark Summary", benchmark), + ): + lines.extend( + [ + f"## {title}", + "", + "```json", + json.dumps(payload, indent=2, sort_keys=True, default=_json_default), + "```", + "", + ] + ) (run_dir / "qa-report.md").write_text("\n".join(lines), encoding="utf-8") -def cmd_report(args: argparse.Namespace) -> int: - run_dir = Path(args.run) - scan_rows = _read_jsonl(run_dir / "scan-results.jsonl") - third_party_rows = _read_jsonl(run_dir / "third-party-results.jsonl") +def _write_qa_outputs( + run_dir: Path, + scan_rows: Sequence[Mapping[str, Any]], + third_party_rows: Sequence[Mapping[str, Any]], +) -> None: _write_json(run_dir / "parity-drift.json", _build_parity_drift(scan_rows)) _write_json( run_dir / "third-party-differential.json", @@ -1935,6 +1852,13 @@ def cmd_report(args: argparse.Namespace) -> int: _write_json(run_dir / "benchmark-results.json", _build_benchmark_results(scan_rows, third_party_rows)) _write_benchmark_csv(run_dir / "benchmark-summary.csv", scan_rows, third_party_rows) _write_report(run_dir, scan_rows, third_party_rows) + + +def cmd_report(args: argparse.Namespace) -> int: + run_dir = Path(args.run) + scan_rows = _read_jsonl(run_dir / "scan-results.jsonl") + third_party_rows = _read_jsonl(run_dir / "third-party-results.jsonl") + _write_qa_outputs(run_dir, scan_rows, third_party_rows) print(f"Wrote report files in {run_dir}") return 0 diff --git a/tests/__init__.py b/tests/__init__.py index 825361d76..d2b09fd37 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1,5 @@ # This file makes tests a package + +import pytest + +pytest.register_assert_rewrite("tests.helpers") diff --git a/tests/helpers/workflows.py b/tests/helpers/workflows.py new file mode 100644 index 000000000..ad38ece36 --- /dev/null +++ b/tests/helpers/workflows.py @@ -0,0 +1,23 @@ +"""Shared accessors for parsed workflow contract tests.""" + +from typing import Any, cast + + +def _workflow_triggers(workflow: dict[str, Any]) -> dict[str, Any]: + raw_workflow = cast(dict[Any, Any], workflow) + triggers = raw_workflow.get("on", raw_workflow.get(True)) + assert isinstance(triggers, dict) + return triggers + + +def _step_by_name(steps: list[dict[str, Any]], name: str) -> dict[str, Any]: + for step in steps: + if step.get("name") == name: + return step + raise AssertionError(f"Step {name!r} not found") + + +def _jobs(workflow: dict[str, Any]) -> dict[str, Any]: + jobs = workflow["jobs"] + assert isinstance(jobs, dict) + return jobs diff --git a/tests/test_docker_workflow.py b/tests/test_docker_workflow.py index 335f71375..b3e17d51c 100644 --- a/tests/test_docker_workflow.py +++ b/tests/test_docker_workflow.py @@ -4,11 +4,13 @@ import subprocess import sys from pathlib import Path -from typing import Any, cast +from typing import Any import pytest import yaml +from tests.helpers.workflows import _jobs, _step_by_name, _workflow_triggers + _REPO_ROOT = Path(__file__).resolve().parents[1] _PINNED_PYTHON_IMAGE_RE = re.compile(r"^python:(?P\d+\.\d+-slim)@sha256:(?P[0-9a-f]{64})$") _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") @@ -28,13 +30,6 @@ def _load_docker_publish_workflow() -> dict[str, Any]: return workflow -def _workflow_triggers(workflow: dict[str, Any]) -> dict[str, Any]: - raw_workflow = cast(dict[Any, Any], workflow) - triggers = raw_workflow.get("on", raw_workflow.get(True)) - assert isinstance(triggers, dict) - return triggers - - def _run_tag_validator(tag: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [sys.executable, str(_REPO_ROOT / "scripts" / "validate_docker_publish_tag.py"), tag], @@ -45,7 +40,14 @@ def _run_tag_validator(tag: str) -> subprocess.CompletedProcess[str]: def _dockerfile_lines(path: str) -> list[str]: - return (_REPO_ROOT / path).read_text(encoding="utf-8").splitlines() + content = (_REPO_ROOT / path).read_text(encoding="utf-8") + source_command = " && . ./docker-install-rust.sh \\" + if source_command in content: + copy_command = "COPY . ." if path == "Dockerfile.full" else "COPY docker-install-rust.sh ./" + assert content.index(copy_command) < content.index(source_command) + installer = (_REPO_ROOT / "docker-install-rust.sh").read_text(encoding="utf-8") + content = content.replace(source_command, installer) + return content.splitlines() def _python_image_from_arg(path: str) -> str: @@ -67,12 +69,6 @@ def _assert_pinned_python_image(image: str, expected_version: str) -> None: assert match.group("version") == expected_version -def _jobs(workflow: dict[str, Any]) -> dict[str, Any]: - jobs = workflow["jobs"] - assert isinstance(jobs, dict) - return jobs - - def _job_steps(workflow: dict[str, Any], job_name: str) -> list[dict[str, Any]]: job = _jobs(workflow)[job_name] assert isinstance(job, dict) @@ -81,13 +77,6 @@ def _job_steps(workflow: dict[str, Any], job_name: str) -> list[dict[str, Any]]: return steps -def _step_by_name(steps: list[dict[str, Any]], name: str) -> dict[str, Any]: - for step in steps: - if step.get("name") == name: - return step - raise AssertionError(f"Step {name!r} not found") - - def test_dockerfiles_pin_python_base_images_by_digest() -> None: lightweight_image = _python_image_from_arg("Dockerfile") full_image = _python_image_from_arg("Dockerfile.full") @@ -293,7 +282,7 @@ def test_dockerfiles_verify_pinned_rustup_init_instead_of_streaming_shell() -> N assert _SHA256_RE.fullmatch(expected_arm64_sha256) for path in ("Dockerfile", "Dockerfile.full", "Dockerfile.tensorflow"): - content = (_REPO_ROOT / path).read_text(encoding="utf-8") + content = "\n".join(_dockerfile_lines(path)) assert "https://sh.rustup.rs" not in content assert "| sh" not in content assert "sh -s --" not in content diff --git a/tests/test_perf_workflow.py b/tests/test_perf_workflow.py index 63144d82e..228753877 100644 --- a/tests/test_perf_workflow.py +++ b/tests/test_perf_workflow.py @@ -10,6 +10,8 @@ import pytest import yaml +from tests.helpers.workflows import _jobs, _step_by_name, _workflow_triggers + def _load_workflow(filename: str) -> dict[str, Any]: current_path = Path(__file__).resolve() @@ -32,19 +34,6 @@ def _load_perf_workflow() -> dict[str, Any]: return _load_workflow("perf.yml") -def _workflow_triggers(workflow: dict[str, Any]) -> dict[str, Any]: - raw_workflow = cast(dict[Any, Any], workflow) - triggers = raw_workflow.get("on", raw_workflow.get(True)) - assert isinstance(triggers, dict) - return triggers - - -def _jobs(workflow: dict[str, Any]) -> dict[str, Any]: - jobs = workflow["jobs"] - assert isinstance(jobs, dict) - return jobs - - def _benchmarks_job(workflow: dict[str, Any]) -> dict[str, Any]: job = _jobs(workflow)["benchmarks"] assert isinstance(job, dict) @@ -57,13 +46,6 @@ def _job_steps(workflow: dict[str, Any]) -> list[dict[str, Any]]: return steps -def _step_by_name(steps: list[dict[str, Any]], name: str) -> dict[str, Any]: - for step in steps: - if step.get("name") == name: - return step - raise AssertionError(f"Step {name!r} not found") - - def _node_script(step: dict[str, Any]) -> str: run = step["run"] assert isinstance(run, str) diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index 2374a832d..848dbf556 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -16,12 +16,14 @@ import zlib from collections.abc import Iterator from pathlib import Path -from typing import Any, cast +from typing import Any import pytest import yaml from packaging.requirements import Requirement +from tests.helpers.workflows import _jobs, _step_by_name, _workflow_triggers + try: import tomllib except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility @@ -47,13 +49,6 @@ def _load_release_workflow() -> dict[str, Any]: return workflow -def _workflow_triggers(workflow: dict[str, Any]) -> dict[str, Any]: - raw_workflow = cast(dict[Any, Any], workflow) - triggers = raw_workflow.get("on", raw_workflow.get(True)) - assert isinstance(triggers, dict) - return triggers - - def _job_steps(workflow: dict[str, Any], job_name: str) -> list[dict[str, Any]]: jobs = workflow["jobs"] assert isinstance(jobs, dict) @@ -64,19 +59,6 @@ def _job_steps(workflow: dict[str, Any], job_name: str) -> list[dict[str, Any]]: return steps -def _step_by_name(steps: list[dict[str, Any]], name: str) -> dict[str, Any]: - for step in steps: - if step.get("name") == name: - return step - raise AssertionError(f"Step {name!r} not found") - - -def _jobs(workflow: dict[str, Any]) -> dict[str, Any]: - jobs = workflow["jobs"] - assert isinstance(jobs, dict) - return jobs - - def _run_manual_release_step( *, root_version: str, @@ -623,6 +605,7 @@ def test_release_workflow_verifies_published_picklescan_package() -> None: steps = _job_steps(workflow, "verify-picklescan-pypi") wait_step = _step_by_name(steps, "Wait for modelaudit-picklescan files on PyPI") + assert wait_step["env"] == {"PYPI_PROJECT": "modelaudit-picklescan"} wait_run = wait_step["run"] assert "https://pypi.org/pypi/modelaudit-picklescan/{version}/json" in wait_run assert "https://pypi.org/simple/modelaudit-picklescan/" in wait_run @@ -730,6 +713,7 @@ def fake_urlopen(url: str | urllib.request.Request, timeout: int = 20) -> io.Byt ticks = iter((0.0, 1.0, 2.0, 3.0)) monkeypatch.setenv("EXPECTED_VERSION", version) + monkeypatch.setenv("PYPI_PROJECT", project) monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) monkeypatch.setattr(time, "monotonic", lambda: next(ticks, 3.0)) monkeypatch.setattr(time, "sleep", lambda _seconds: None) @@ -879,6 +863,7 @@ def test_release_workflow_verifies_published_root_package_after_picklescan() -> steps = _job_steps(workflow, "verify-pypi") wait_step = _step_by_name(steps, "Wait for modelaudit files on PyPI") + assert wait_step["env"] == {"PYPI_PROJECT": "modelaudit"} wait_run = wait_step["run"] assert "https://pypi.org/pypi/modelaudit/{version}/json" in wait_run assert "https://pypi.org/simple/modelaudit/" in wait_run