From 237fc1882ee04dd4ca6e178a7276335ff4930e79 Mon Sep 17 00:00:00 2001 From: coldsofttech <161084795+coldsofttech@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:28:17 +0100 Subject: [PATCH 1/4] feat: add BatchVault package for batch archiving, splitting and restoration Native tar/xz/gzip/bzip2 engine and optional 7-Zip engine, folder-preserving pack/unpack, verify, inspect, list, keygen, sign and doctor commands, encryption and signed manifests, Python API, GitHub Action, Docker images, docs and tests. Defaults: --level auto, --jobs 0, --retries 3. --- .dockerignore | 20 + .github/ISSUE_TEMPLATE/bug_report.yml | 38 + .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 14 + .github/PULL_REQUEST_TEMPLATE.md | 10 + .github/dependabot.yml | 33 + .github/workflows/docker.yml | 77 ++ .github/workflows/e2e.yml | 338 +++++++ .github/workflows/lint.yml | 27 + .github/workflows/release.yml | 79 ++ .github/workflows/tests.yml | 45 + .gitignore | 3 + .pre-commit-config.yaml | 17 + .python-version | 1 + CHANGELOG.md | 26 + CODE_OF_CONDUCT.md | 15 + CONTRIBUTING.md | 34 + Dockerfile | 35 + README.md | 273 +++++- SECURITY.md | 51 ++ action.yml | 255 ++++++ docs/architecture.md | 164 ++++ docs/cli.md | 361 ++++++++ docs/compression.md | 199 +++++ docs/configuration.md | 75 ++ docs/development.md | 30 + docs/docker.md | 84 ++ docs/github-action.md | 138 +++ docs/python.md | 173 ++++ docs/testing.md | 99 +++ pyproject.toml | 53 ++ src/batchvault/__init__.py | 61 ++ src/batchvault/__main__.py | 3 + src/batchvault/api.py | 877 +++++++++++++++++++ src/batchvault/blocks.py | 176 ++++ src/batchvault/cli.py | 806 +++++++++++++++++ src/batchvault/config.py | 169 ++++ src/batchvault/crypto.py | 193 ++++ src/batchvault/doctor.py | 67 ++ src/batchvault/engines/__init__.py | 0 src/batchvault/engines/base.py | 57 ++ src/batchvault/engines/native.py | 237 +++++ src/batchvault/engines/registry.py | 61 ++ src/batchvault/engines/sevenzip.py | 303 +++++++ src/batchvault/errors.py | 38 + src/batchvault/eta.py | 39 + src/batchvault/filters.py | 51 ++ src/batchvault/inputs.py | 226 +++++ src/batchvault/levels.py | 169 ++++ src/batchvault/locate.py | 140 +++ src/batchvault/logs.py | 86 ++ src/batchvault/manifest.py | 210 +++++ src/batchvault/models.py | 164 ++++ src/batchvault/names.py | 36 + src/batchvault/notify.py | 73 ++ src/batchvault/options.py | 71 ++ src/batchvault/py.typed | 0 src/batchvault/security.py | 40 + src/batchvault/signing.py | 152 ++++ src/batchvault/ui.py | 298 +++++++ src/batchvault/volumes.py | 192 ++++ tests/conftest.py | 19 + tests/e2e/conftest.py | 38 + tests/e2e/scripts/compare.py | 48 + tests/e2e/scripts/make_data.py | 37 + tests/e2e/test_cli_e2e.py | 335 +++++++ tests/e2e/test_docker_e2e.py | 199 +++++ tests/e2e/test_package_e2e.py | 240 +++++ tests/integration/test_sevenzip_api.py | 154 ++++ tests/integration/test_sevenzip_roundtrip.py | 66 ++ tests/test_api.py | 249 ++++++ tests/test_cli.py | 170 ++++ tests/test_cli_features.py | 177 ++++ tests/test_config.py | 163 ++++ tests/test_crypto.py | 145 +++ tests/test_eta.py | 30 + tests/test_features.py | 265 ++++++ tests/test_filters.py | 31 + tests/test_folders.py | 318 +++++++ tests/test_hardening.py | 320 +++++++ tests/test_inputs.py | 135 +++ tests/test_levels.py | 266 ++++++ tests/test_logs.py | 30 + tests/test_manifest.py | 87 ++ tests/test_native.py | 209 +++++ tests/test_notify.py | 71 ++ tests/test_options.py | 37 + tests/test_sevenzip.py | 101 +++ tests/test_signing.py | 148 ++++ tests/test_summary.py | 198 +++++ tests/test_volume_io.py | 46 + tests/test_volumes.py | 45 + uv.lock | 523 +++++++++++ 93 files changed, 12666 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/e2e.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/tests.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .python-version create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 SECURITY.md create mode 100644 action.yml create mode 100644 docs/architecture.md create mode 100644 docs/cli.md create mode 100644 docs/compression.md create mode 100644 docs/configuration.md create mode 100644 docs/development.md create mode 100644 docs/docker.md create mode 100644 docs/github-action.md create mode 100644 docs/python.md create mode 100644 docs/testing.md create mode 100644 pyproject.toml create mode 100644 src/batchvault/__init__.py create mode 100644 src/batchvault/__main__.py create mode 100644 src/batchvault/api.py create mode 100644 src/batchvault/blocks.py create mode 100644 src/batchvault/cli.py create mode 100644 src/batchvault/config.py create mode 100644 src/batchvault/crypto.py create mode 100644 src/batchvault/doctor.py create mode 100644 src/batchvault/engines/__init__.py create mode 100644 src/batchvault/engines/base.py create mode 100644 src/batchvault/engines/native.py create mode 100644 src/batchvault/engines/registry.py create mode 100644 src/batchvault/engines/sevenzip.py create mode 100644 src/batchvault/errors.py create mode 100644 src/batchvault/eta.py create mode 100644 src/batchvault/filters.py create mode 100644 src/batchvault/inputs.py create mode 100644 src/batchvault/levels.py create mode 100644 src/batchvault/locate.py create mode 100644 src/batchvault/logs.py create mode 100644 src/batchvault/manifest.py create mode 100644 src/batchvault/models.py create mode 100644 src/batchvault/names.py create mode 100644 src/batchvault/notify.py create mode 100644 src/batchvault/options.py create mode 100644 src/batchvault/py.typed create mode 100644 src/batchvault/security.py create mode 100644 src/batchvault/signing.py create mode 100644 src/batchvault/ui.py create mode 100644 src/batchvault/volumes.py create mode 100644 tests/conftest.py create mode 100644 tests/e2e/conftest.py create mode 100644 tests/e2e/scripts/compare.py create mode 100644 tests/e2e/scripts/make_data.py create mode 100644 tests/e2e/test_cli_e2e.py create mode 100644 tests/e2e/test_docker_e2e.py create mode 100644 tests/e2e/test_package_e2e.py create mode 100644 tests/integration/test_sevenzip_api.py create mode 100644 tests/integration/test_sevenzip_roundtrip.py create mode 100644 tests/test_api.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_cli_features.py create mode 100644 tests/test_config.py create mode 100644 tests/test_crypto.py create mode 100644 tests/test_eta.py create mode 100644 tests/test_features.py create mode 100644 tests/test_filters.py create mode 100644 tests/test_folders.py create mode 100644 tests/test_hardening.py create mode 100644 tests/test_inputs.py create mode 100644 tests/test_levels.py create mode 100644 tests/test_logs.py create mode 100644 tests/test_manifest.py create mode 100644 tests/test_native.py create mode 100644 tests/test_notify.py create mode 100644 tests/test_options.py create mode 100644 tests/test_sevenzip.py create mode 100644 tests/test_signing.py create mode 100644 tests/test_summary.py create mode 100644 tests/test_volume_io.py create mode 100644 tests/test_volumes.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..40f1b7b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.github +.venv +.temp +.pytest_cache +.ruff_cache +**/__pycache__ +dist +tests +docs +.env +.env.* +*.key +*.pem +*.pfx +*.p12 +*.sig +manifest.json +archives +restored diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..4487436 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,38 @@ +name: Bug report +description: Something does not work as expected +labels: [bug] +body: + - type: textarea + id: what + attributes: + label: What happened? + description: What did you run, what did you expect, and what happened instead? + validations: + required: true + - type: textarea + id: command + attributes: + label: Command or code + render: shell + - type: input + id: version + attributes: + label: BatchVault version + description: Output of `batchvault --version` + validations: + required: true + - type: dropdown + id: engine + attributes: + label: Engine + options: [auto, native, 7zip] + - type: input + id: platform + attributes: + label: OS and Python version + - type: textarea + id: log + attributes: + label: Log or error output + description: Run `batchvault doctor` and attach the relevant lines of the log file. Remove anything private. + render: text diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..7c36301 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Security vulnerability + url: https://github.com/coldsofttech/BatchVault/security/advisories/new + about: Please report security problems privately. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..bb8a068 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,14 @@ +name: Feature request +description: Suggest an improvement +labels: [enhancement] +body: + - type: textarea + id: problem + attributes: + label: What are you trying to do? + validations: + required: true + - type: textarea + id: idea + attributes: + label: What would you like BatchVault to do? diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..8082409 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +## What changed + + + +## Checklist + +- [ ] Tests added or updated +- [ ] `uv run ruff check .` and `uv run pytest -m "not integration and not e2e"` pass +- [ ] `CHANGELOG.md` has a short `[Unreleased]` entry +- [ ] README or `docs/` updated if behaviour changed diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2291ec2 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,33 @@ +version: 2 + +updates: + # Python dependencies and the uv.lock file + - package-ecosystem: "uv" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(deps)" + groups: + python-dependencies: + patterns: ["*"] + update-types: ["minor", "patch"] + + # Actions used in .github/workflows + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(ci)" + groups: + github-actions: + patterns: ["*"] + + # Hook versions in .pre-commit-config.yaml + - package-ecosystem: "pre-commit" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "chore(deps)" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..6448abc --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,77 @@ +name: Docker + +# Builds the image on every pull request and publishes both variants to GHCR +# when a version tag such as v0.2.0 is pushed. +on: + push: + tags: ["v*.*.*"] + pull_request: + paths: + - Dockerfile + - .dockerignore + - pyproject.toml + - src/** + - .github/workflows/docker.yml + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + image: + name: Image (${{ matrix.variant }}) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - { variant: native, with_7zip: "false", suffix: "" } + - { variant: 7zip, with_7zip: "true", suffix: "-7zip" } + + steps: + - uses: actions/checkout@v7 + + - uses: docker/setup-qemu-action@v3 + + - uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: startsWith(github.ref, 'refs/tags/') + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tags + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/batchvault + flavor: | + latest=false + suffix=${{ matrix.suffix }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }} + + - name: Build${{ startsWith(github.ref, 'refs/tags/') && ' and push' || '' }} + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + build-args: WITH_7ZIP=${{ matrix.with_7zip }} + push: ${{ startsWith(github.ref, 'refs/tags/') }} + provenance: mode=max + sbom: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.variant }} + cache-to: type=gha,mode=max,scope=${{ matrix.variant }} diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..92f006b --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,338 @@ +name: End-to-end + +# Runs the real program the way users do: the command line and the built package +# (tests/e2e), the GitHub Action (this repository's action.yml, used as "./"), +# and the Docker images. Also run before every release (see release.yml). +on: + workflow_call: + workflow_dispatch: + push: + branches: [main] + pull_request: + paths: + - src/** + - tests/e2e/** + - action.yml + - Dockerfile + - .dockerignore + - pyproject.toml + - uv.lock + - .github/workflows/e2e.yml + schedule: + - cron: "0 4 * * 1" # Mondays; catches changes in runners, pip and Docker + +concurrency: + group: e2e-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + cli-and-package: + name: CLI and package / ${{ matrix.os }} / Python ${{ matrix.python }} + runs-on: ${{ matrix.os }} + # Same matrix as tests.yml. Python 3.15 may still be a pre-release; don't + # let it block the build. + continue-on-error: ${{ matrix.python == '3.15' }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, python: "3.11" } + - { os: ubuntu-latest, python: "3.12" } + - { os: ubuntu-latest, python: "3.13" } + - { os: ubuntu-latest, python: "3.14" } + - { os: ubuntu-latest, python: "3.15" } + - { os: windows-latest, python: "3.11" } + - { os: macos-latest, python: "3.11" } + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Install + run: uv sync --python ${{ matrix.python }} + + # No -n: the package tests build a wheel and virtual environments once. + - name: CLI and package tests + run: uv run pytest -m "e2e and not docker" -v + + docker: + name: Docker images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Install + run: uv sync --python 3.11 + + - name: Docker tests (builds both images) + run: uv run pytest -m docker -v + + action: + name: GitHub Action / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + defaults: + run: + shell: bash + env: + PYTHONUTF8: "1" + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # Only used by the steps below that create keys or edit files; the action + # installs its own copy. + - name: Install BatchVault for helper steps + run: python -m pip install --quiet ".[crypto]" + + - name: Create test data + run: python tests/e2e/scripts/make_data.py data + + # --- pack, verify, unpack ------------------------------------------- + - name: Pack + id: pack + uses: ./ + with: + files: | + data/report.bin + data/notes.txt + data/ünï cödé.txt + data/empty.dat + data/sub dir/nested file.csv + output: archives + engine: native + volume-size: 256k + verify: "true" + + - name: Pack reported success and wrote a manifest + env: + STATUS: ${{ steps.pack.outputs.status }} + OUTPUT: ${{ steps.pack.outputs.output }} + RESULT: ${{ steps.pack.outputs.result-json }} + run: | + test "$STATUS" = success + test "$OUTPUT" = archives + test -f archives/manifest.json + test -s "$RESULT" + python -c "import json,sys; d=json.load(open(sys.argv[1], encoding='utf-8')); assert len(d['entries'])==5" "$RESULT" + + - name: Verify against the originals + id: verify + uses: ./ + with: + command: verify + source: archives + against: data + + - name: Unpack + id: unpack + uses: ./ + with: + command: unpack + source: archives + output: restored + + - name: Restored files are identical + env: + VERIFY: ${{ steps.verify.outputs.status }} + UNPACK: ${{ steps.unpack.outputs.status }} + run: | + test "$VERIFY" = success + test "$UNPACK" = success + python tests/e2e/scripts/compare.py data restored + + # --- incremental mode ----------------------------------------------- + - name: Pack again with update (nothing changed) + id: update + uses: ./ + with: + files: | + data/report.bin + data/notes.txt + output: archives + engine: native + volume-size: 256k + update: "true" + fail-on-error: "false" + + - name: Nothing was re-archived + env: + STATUS: ${{ steps.update.outputs.status }} + run: test "$STATUS" = skipped + + # --- artifact upload ------------------------------------------------ + - name: Pack and upload as an artifact + uses: ./ + with: + files: data/notes.txt + output: uploaded + engine: native + upload-artifact: "true" + artifact-name: e2e-archives-${{ matrix.os }} + + - name: Download the artifact + uses: actions/download-artifact@v4 + with: + name: e2e-archives-${{ matrix.os }} + path: downloaded + + - name: Artifact holds the archive + run: | + test -f downloaded/manifest.json + ls downloaded/notes.txt + + # --- encryption ----------------------------------------------------- + - name: Pack encrypted + uses: ./ + with: + files: data/notes.txt + output: vault + engine: native + encrypt: "true" + password: e2e-throwaway-password + + - name: Unpack with the wrong password + id: wrong + uses: ./ + with: + command: unpack + source: vault + output: wrong-restore + password: not-the-password + fail-on-error: "false" + + - name: Unpack with the right password + id: right + uses: ./ + with: + command: unpack + source: vault + output: right-restore + password: e2e-throwaway-password + + - name: Only the right password restores + env: + WRONG: ${{ steps.wrong.outputs.status }} + RIGHT: ${{ steps.right.outputs.status }} + run: | + test "$WRONG" = failed + test "$RIGHT" = success + test ! -e wrong-restore/notes.txt + python tests/e2e/scripts/compare.py data right-restore notes.txt + + # --- signing -------------------------------------------------------- + - name: Create keys + run: batchvault keygen --private signing.key --public signing.pub -q + + - name: Pack with a signed manifest + uses: ./ + with: + files: data/notes.txt + output: signed + engine: native + sign-key: signing.key + + - name: Verify the signature + id: signed + uses: ./ + with: + command: verify + source: signed + public-key: signing.pub + + - name: Tamper with the manifest + run: | + python - <<'PY' + from pathlib import Path + path = Path("signed/manifest.json") + path.write_text(path.read_text(encoding="utf-8").replace("notes.txt", "other.txt", 1), encoding="utf-8") + PY + + - name: Verify the tampered manifest + id: tampered + uses: ./ + with: + command: verify + source: signed + public-key: signing.pub + fail-on-error: "false" + + - name: The signature check caught the change + env: + GOOD: ${{ steps.signed.outputs.status }} + BAD: ${{ steps.tampered.outputs.status }} + run: | + test "$GOOD" = success + test "$BAD" = failed + + # --- failures ------------------------------------------------------- + - name: Pack a file that does not exist + id: missing + continue-on-error: true + uses: ./ + with: + files: data/does-not-exist.txt + output: nope + + - name: Pack a missing file without failing the step + id: missing-quiet + uses: ./ + with: + files: data/does-not-exist.txt + output: nope + fail-on-error: "false" + + - name: Failures are reported + env: + OUTCOME: ${{ steps.missing.outcome }} + STATUS: ${{ steps.missing-quiet.outputs.status }} + run: | + test "$OUTCOME" = failure + test "$STATUS" = failed + test ! -e nope + + # --- 7-Zip (only where the runner has it) --------------------------- + - name: Detect 7-Zip + id: sevenzip + run: | + if command -v 7zz >/dev/null || command -v 7z >/dev/null || [ -f "/c/Program Files/7-Zip/7z.exe" ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Pack with 7-Zip + if: steps.sevenzip.outputs.found == 'true' + id: sz-pack + uses: ./ + with: + files: | + data/report.bin + data/notes.txt + output: sz-archives + engine: 7zip + volume-size: 256k + verify: "true" + + - name: Unpack the 7-Zip archives + if: steps.sevenzip.outputs.found == 'true' + uses: ./ + with: + command: unpack + source: sz-archives + output: sz-restored + + - name: 7-Zip round trip is identical + if: steps.sevenzip.outputs.found == 'true' + run: python tests/e2e/scripts/compare.py data sz-restored report.bin notes.txt diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..d17057d --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,27 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: lint-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Install + run: uv sync --python 3.11 + + - name: Ruff + run: uv run ruff check . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..dc2696f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,79 @@ +name: Release + +# Publishes to PyPI when a version tag such as v0.2.0 is pushed. +# Uses PyPI trusted publishing: register this repository and the "pypi" +# environment as a trusted publisher in the PyPI project settings. +on: + push: + tags: ["v*.*.*"] + +permissions: + contents: read + +jobs: + check: + name: Check version and tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Tag must match the package version + shell: bash + run: | + tag="${GITHUB_REF_NAME#v}" + project=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") + module=$(python3 -c "import re; print(re.search(r'__version__ = \"(.+?)\"', open('src/batchvault/__init__.py').read()).group(1))") + echo "tag=$tag pyproject=$project __init__=$module" + if [ "$tag" != "$project" ] || [ "$tag" != "$module" ]; then + echo "::error::Tag v$tag must match the version in pyproject.toml ($project) and src/batchvault/__init__.py ($module)." + exit 1 + fi + + - name: Install + run: uv sync --python 3.11 + + - name: Lint + run: uv run ruff check . + + - name: Unit tests + run: uv run pytest -m "not integration and not e2e" + + - name: Integration tests + run: uv run pytest -m integration + + e2e: + name: End-to-end tests + uses: ./.github/workflows/e2e.yml + + publish: + name: Build and publish + needs: [check, e2e] + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: write + id-token: write + attestations: write + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Build + run: uv build + + - name: Attest build provenance + uses: actions/attest-build-provenance@v2 + with: + subject-path: dist/* + + - name: Publish to PyPI + run: uv publish --trusted-publishing always + + - name: GitHub release + uses: softprops/action-gh-release@v3 + with: + generate_release_notes: true + files: dist/* diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..61ec555 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,45 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.os }} / Python ${{ matrix.python }} + runs-on: ${{ matrix.os }} + # Python 3.15 may still be a pre-release; don't let it block the build. + continue-on-error: ${{ matrix.python == '3.15' }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, python: "3.11" } + - { os: ubuntu-latest, python: "3.12" } + - { os: ubuntu-latest, python: "3.13" } + - { os: ubuntu-latest, python: "3.14" } + - { os: ubuntu-latest, python: "3.15" } + - { os: windows-latest, python: "3.11" } + - { os: macos-latest, python: "3.11" } + + steps: + - uses: actions/checkout@v7 + + - uses: astral-sh/setup-uv@v7 + + - name: Install + run: uv sync --python ${{ matrix.python }} + + - name: Unit tests + run: uv run pytest -m "not integration and not e2e" -n auto + + - name: Integration tests + run: uv run pytest -m integration -n auto diff --git a/.gitignore b/.gitignore index 6f596e1..487060e 100644 --- a/.gitignore +++ b/.gitignore @@ -218,3 +218,6 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +CLAUDE.md +.temp/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d48f71c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +# Install once with: uv run pre-commit install +# Integration tests (marker "integration") are slower and run only in CI. +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.17.2 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + + - repo: local + hooks: + - id: pytest + name: pytest (unit tests) + entry: uv run pytest -m "not integration and not e2e" -n auto -q + language: system + types: [python] + pass_filenames: false diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..2c07333 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1e181ae --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +All notable changes to this project are documented here. + +## [Unreleased] + +## [0.1.0] - 2026-10-24 + +First release. + +- Archive many files at once into volumes of a size you choose, and restore them from a manifest, a folder, any single volume or a list. +- Two engines: a built-in one that needs nothing installed (xz, gzip, bzip2 or no compression) and an optional 7-Zip engine. +- Your folder structure is kept: archives mirror the tree and `unpack` recreates it. `list files` remembers the folder you listed, and `--root DIR` overrides where paths start. +- Every archive has a checksummed manifest; restoring checks each volume and the restored file before anything is placed in the output folder. +- `verify` and `inspect` check and describe archives, with one consistent Source / Archive / Status / Details table and live progress with an ETA. +- `--level auto` picks a compression level for each file from its type: none for video, photos and archives, stronger for text. +- `--update` skips unchanged files, `--retries` retries failures, `--dry-run` previews a run, and `--include` / `--exclude` filter files. +- Optional encryption and signed manifests (`pip install "batchvault[crypto]"`), and `--delete-source` to remove originals only after the archive is proven good. +- Safer by default: archives are readable only by their owner on Linux and macOS; `unpack --max-output-size` refuses archives that expand beyond a size you choose; settings files found automatically cannot run programs, send data or delete files; and 7-Zip is never run from the current folder. +- `batchvault.toml` settings file, `--notify` webhooks, `--json` and `--json-events` output, and per-run log files in your system's log folder. +- A Python API, a GitHub Action (`coldsofttech/BatchVault@v1`) and Docker images with and without 7-Zip. +- Sensible defaults: `--level auto`, no worker threads (`--jobs 0`) and three retries. +- Requires Python 3.11.4 or later. + +[Unreleased]: https://github.com/coldsofttech/BatchVault/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/coldsofttech/BatchVault/releases/tag/v0.1.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..5b5e015 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Code of conduct + +Be respectful and constructive. Everyone taking part in this project is expected +to: + +- Treat others with courtesy, whatever their background or experience. +- Give and accept feedback on the work, not the person. +- Assume good intent, and ask when something is unclear. +- Keep discussions on topic and free of harassment, personal attacks or + discriminatory language. + +Maintainers may edit or remove comments, commits and issues that break these +expectations, and may block repeat offenders. To report a problem, use the +private reporting route described in [SECURITY.md](SECURITY.md) and mark it as a +conduct issue. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..af053b5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing + +Thanks for helping improve BatchVault. + +## Setup + +``` +uv sync +uv run pre-commit install +``` + +## Before you open a pull request + +``` +uv run ruff check . +uv run pytest -m "not integration and not e2e" -n auto +uv run pytest -m integration # needs 7-Zip installed; skipped otherwise +uv run pytest -m "e2e and not docker" # optional: end-to-end CLI and package tests +``` + +CI also runs the Docker and GitHub Action end-to-end tests; see +[docs/testing.md](docs/testing.md). + +- Add or update tests for the behaviour you change. +- Add a one or two line entry to `[Unreleased]` in `CHANGELOG.md`, written for + users (no file or function names). +- User-facing documentation goes in `README.md`; technical notes go in `docs/`. +- Keep the library free of `rich` imports; terminal output belongs in `ui.py` + and `cli.py`. + +## Reporting problems + +Use the issue templates. For security problems, see [SECURITY.md](SECURITY.md) +instead of opening a public issue. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..00c2376 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.13-slim AS build + +WORKDIR /src +COPY pyproject.toml README.md LICENSE ./ +COPY src ./src +RUN pip install --no-cache-dir build \ + && python -m build --wheel --outdir /dist + + +FROM python:3.13-slim + +# Build with --build-arg WITH_7ZIP=true to add the 7-Zip engine. 7-Zip is +# LGPL software installed from the Debian package; it is not part of BatchVault. +ARG WITH_7ZIP=false +RUN if [ "$WITH_7ZIP" = "true" ]; then \ + apt-get update \ + && apt-get install -y --no-install-recommends 7zip \ + && rm -rf /var/lib/apt/lists/*; \ + fi + +COPY --from=build /dist/*.whl /tmp/ +RUN pip install --no-cache-dir "$(ls /tmp/*.whl)[crypto]" && rm /tmp/*.whl + +RUN useradd --create-home --uid 1000 batchvault +USER batchvault + +# Mount your files here, e.g. docker run --rm -v "$PWD:/data" batchvault ... +WORKDIR /data + +LABEL org.opencontainers.image.title="BatchVault" \ + org.opencontainers.image.source="https://github.com/coldsofttech/BatchVault" \ + org.opencontainers.image.licenses="MIT" + +ENTRYPOINT ["batchvault"] +CMD ["--help"] diff --git a/README.md b/README.md index 72c3173..14ba400 100644 --- a/README.md +++ b/README.md @@ -1 +1,272 @@ -# BatchVault \ No newline at end of file +# BatchVault + +[![Tests](https://github.com/coldsofttech/BatchVault/actions/workflows/tests.yml/badge.svg)](https://github.com/coldsofttech/BatchVault/actions/workflows/tests.yml) +[![Lint](https://github.com/coldsofttech/BatchVault/actions/workflows/lint.yml/badge.svg)](https://github.com/coldsofttech/BatchVault/actions/workflows/lint.yml) +[![Release](https://img.shields.io/github/v/release/coldsofttech/BatchVault?logo=github)](https://github.com/coldsofttech/BatchVault/releases) +[![PyPI](https://img.shields.io/pypi/v/batchvault?logo=pypi&logoColor=white)](https://pypi.org/project/batchvault/) +[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue?logo=python&logoColor=white)](https://www.python.org/downloads/) +[![License](https://img.shields.io/github/license/coldsofttech/BatchVault)](LICENSE) +[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) +[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit) + +Archive, compress, split and restore many files in one go, from the command +line, from Python, from a GitHub workflow, or from a container. + +## Why BatchVault? + +Backing up or moving a pile of large files usually means a shell loop around +`7z`, with no record of what went where. When a copy goes missing or gets +damaged, you only find out when you try to restore. + +BatchVault does the whole job and keeps proof: + +- Every file becomes its own set of fixed-size volumes, so huge files fit + through upload limits, small disks and flaky links. +- A **manifest** records every file and volume with its checksum, so + restoring never depends on guessing folder names. +- Restoring checks each volume and the finished file against those checksums + before anything lands in your output folder. +- It works the same on Windows, Linux and macOS, with or without 7-Zip. + +## What you get + +- Compress a list of files into volumes of a size you choose (`10m`, `2g`, ...). +- Restore from a manifest, a folder, any single volume, or a list of volumes. +- Verify archives are complete and intact, and optionally compare them with the + originals. +- Pick up where you left off: `--update` skips what has not changed. +- Optional **encryption** (AES-256) and **signed manifests** to prove nothing + was tampered with. +- `--delete-source` removes an original only after the archive has been + restored and matched byte for byte. +- A live progress bar with an ETA, readable summaries, and JSON output for + scripts. +- Include and exclude patterns, retries, webhook notifications and a + settings file. +- A Python API, a GitHub Action and Docker images. + +## Install + +``` +pip install batchvault +``` + +Requires Python 3.11.4 or later. Works with [pipx](https://pipx.pypa.io/) and +[uv](https://docs.astral.sh/uv/) too: + +``` +pipx install batchvault +uv tool install batchvault +``` + +For encryption and signed manifests, add the extra: + +``` +pip install "batchvault[crypto]" +``` + +7-Zip is optional. If you want to use it, install it from +[7-zip.org](https://www.7-zip.org/) (`7zz` on Linux and macOS, for example +`brew install sevenzip`, or `apt install 7zip`). + +## Engines + +BatchVault has two engines. `--engine auto` (the default) uses 7-Zip when it +finds it and the built-in engine otherwise. + +| | Built-in (`native`) | 7-Zip (`7zip`) | +|---|---|---| +| Needs anything installed | No | An installed 7-Zip | +| Volume names | `name.bv.001`, `.002`, ... | `name.7z.001`, `.002`, ... | +| Compression | xz (default), gzip, bzip2 or none | 7z (LZMA2) | +| Levels | 0-9 (bzip2 has no 0; 1 is used) | 0-9 | +| Volume size, parallel jobs | Yes | Yes | +| Encryption | Yes, needs `batchvault[crypto]` | Yes | +| Plain-tool recovery | Unencrypted volumes join into a normal `.tar.xz` / `.tar.gz` / `.tar.bz2` (`cat name.bv.* > name.tar.xz`) | Opens in any 7-Zip | + +Things to know: + +- `--format` only applies to the built-in engine. 7-Zip always writes 7z, and + BatchVault warns if you ask for something else (`--strict` makes that an + error). +- 7-Zip can only take a password on its command line, so other users on a + shared machine could see it while it runs. Prefer the built-in engine there. +- An encrypted built-in archive cannot be opened with standard tools; use + BatchVault to restore it. +- Several engines can live in one archive folder: the manifest records which + one made each file. +- Videos, photos and other already-compressed files do not shrink, whatever the + level. For them use `--format store`. See the + [compression guide](docs/compression.md) for choosing a format and level, and + for memory use. + +## Use + +| I want to use BatchVault... | Read | +|---|---| +| in a terminal | [Command line guide](docs/cli.md) | +| in my own Python code | [Python guide](docs/python.md) | +| in a GitHub workflow | [GitHub Action guide](docs/github-action.md) | +| in a container | [Docker guide](docs/docker.md) | +| choosing a format and level | [Compression guide](docs/compression.md) | + +A **source** is an original file; an **archive** is the set of volumes made from +it. `pack` turns sources into archives, `unpack` turns archives back into files. + +A first run: + +``` +# 1. List the files you want to archive +batchvault list files D:\Media --exclude "*.tmp" -o files.txt + +# 2. Archive them into 100 MB volumes, checking each archive afterwards +batchvault pack --list files.txt -o archives --volume-size 100m --verify + +# 3. Look at what is in your archives (sizes, ratios, volumes, ...) +batchvault inspect archives + +# 4. Restore everything somewhere else +batchvault unpack archives -o restored +``` + +Your folder structure is kept automatically. Each file gets its own folder, inside +a copy of the folders it came from, plus a manifest: + +``` +batchvault list files D:\Media -o files.json +batchvault pack --list files.json -o archives + +archives/ + manifest.json + Media/ + Holidays/ + holiday.mp4/ + holiday.mp4.bv.001 + holiday.mp4.bv.002 + report.pdf/ + report.pdf.bv.001 +``` + +`unpack` puts every file back in the same place (`restored\Media\Holidays\holiday.mp4`), +and files with the same name in different folders do not clash. + +The folder you listed (`Media`) is kept because `list files` remembers it. If you +give `pack` files directly instead, the folders below the one they all share are +kept; files that sit directly in it are not nested. + +## Cryptography (optional) + +Encryption and signed manifests need one extra package. Everything else works +without it. + +``` +pip install "batchvault[crypto]" +``` + +The Docker images already include it. Without the extra, asking for encryption +or signing stops with a message telling you to install it. + +### Encrypt your archives + +``` +export BATCHVAULT_PASSWORD='a long passphrase' # PowerShell: $env:BATCHVAULT_PASSWORD = '...' +batchvault pack secrets.db -o vault --encrypt +batchvault unpack vault -o restored +``` + +- Encrypting and decrypting use about 128 MiB of memory for a moment while the + password is turned into a key; that is deliberate, to make guessing slow. +- The password comes from `BATCHVAULT_PASSWORD` or `--password-file`, never from + a command-line option, so it stays out of your shell history. +- Restoring and verifying need the same password. A wrong password or a damaged + archive is reported as an error; nothing is written to your output folder. +- Tampering is detected too: changed, reordered or missing pieces of an encrypted + archive fail the check instead of restoring bad data. +- **There is no way to recover encrypted data if the password is lost.** + +What is and is not hidden: + +- The contents of each file are encrypted. +- File names, sizes and checksums still appear in `manifest.json` and in the + output folder names. Keep the manifest private, or do not put sensitive + names in file names. +- The 7-Zip engine encrypts with 7-Zip's own AES-256 (and hides the names inside + the archive), but 7-Zip can only take the password on its command line, so + other users on the same machine could see it while the command runs. On a + shared machine, use the built-in engine. + +### Sign your manifest + +A signature proves the manifest was made by you and has not been changed since, +so restoring from an untrusted copy can be refused. + +``` +batchvault keygen # creates batchvault.key and batchvault.pub +batchvault pack --list files.txt -o archives --sign-key batchvault.key +batchvault verify archives --public-key batchvault.pub +batchvault unpack archives -o restored --public-key batchvault.pub +``` + +- Keep `batchvault.key` secret (set `BATCHVAULT_KEY_PASSWORD` before `keygen` to + store it encrypted). Share only `batchvault.pub`. +- The signature is saved next to the manifest as `manifest.json.sig`. Adding more + files to the archive folder later changes the manifest, so sign it again; an + old signature is removed automatically and you get a warning. +- With `--public-key`, a missing, wrong or outdated signature stops the run + before anything is restored. + +## Config file + +Keep your usual options in a `batchvault.toml` next to your files, or in +`[tool.batchvault]` inside `pyproject.toml`: + +```toml +[pack] +level = 7 +volume-size = "100m" +exclude = ["*.tmp"] +update = true +``` + +Command-line options always win over the file. See +[docs/configuration.md](docs/configuration.md) for every setting and the +environment variables. + +## Check your setup + +``` +batchvault doctor +``` + +shows whether Python has the compression modules it needs, whether 7-Zip was +found, and whether the log folder is writable. If something is missing it says +what to install. + +## Logs + +Every run writes a log file you can attach to a bug report. Passwords are never +logged, and web addresses with credentials in them are masked. + +| OS | Folder | +|---|---| +| Windows | `%LOCALAPPDATA%\BatchVault\Logs` | +| macOS | `~/Library/Logs/BatchVault` | +| Linux | `$XDG_STATE_HOME/batchvault` (default `~/.local/state/batchvault`) | + +The newest 20 logs are kept. Change the folder with `BATCHVAULT_LOG_DIR`, pick a +file with `--log-file`, or turn logging off with `--no-log`. + +## Contributing and security + +Contributions are welcome. Start with [CONTRIBUTING.md](CONTRIBUTING.md) and +follow the [code of conduct](CODE_OF_CONDUCT.md). To report a vulnerability, +please read [SECURITY.md](SECURITY.md) first and do not open a public issue. +Technical notes live in [docs/architecture.md](docs/architecture.md) and +[docs/development.md](docs/development.md). + +## License + +See [LICENSE](LICENSE). BatchVault depends on [Rich](https://github.com/Textualize/rich) +(MIT) and [pathspec](https://github.com/cpburnett/python-pathspec) (MPL-2.0), and +optionally on [cryptography](https://github.com/pyca/cryptography). 7-Zip is a +separate program with its own license and is never bundled. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4b906ec --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,51 @@ +# Security policy + +## Reporting a vulnerability + +Please do not open a public issue. Use GitHub's private vulnerability reporting: +the repository's **Security** tab, then **Report a vulnerability**. + +Include the version, what you did, what you expected, and what happened. You +will get a reply as soon as the maintainer can; fixes are released as patch +versions and noted in the changelog. + +## Supported versions + +Only the latest released version receives security fixes. + +## Things worth knowing + +- Settings files found automatically (in the current or a parent folder) cannot + change which program runs, where notifications go, which keys are trusted or + whether originals are deleted. Name the file with `--config` to trust it. +- BatchVault never runs a program from the current folder. 7-Zip is taken from + `--7z`, `BATCHVAULT_7Z`, the `PATH` folders (absolute entries only) or its + usual install folders. +- Archives, volumes and `manifest.json` are created readable only by their owner + on Linux and macOS (on Windows, folder permissions apply). +- When restoring archives you do not trust, use `--max-output-size`; with a + manifest, an archive can never expand beyond the size it records. 7-Zip links + and special files are refused, and 7-Zip file names are never treated as + wildcards. +- Webhook URLs (which carry a secret) are masked in the log file; the notification + itself contains file names and paths, so use `https://`. +- Restoring only writes plain files with simple names. Links, devices, folders + inside the archive and names Windows treats specially (`C:x`, `NUL`, ...) are + refused. + +- Archives made with `--encrypt` are protected by your password; there is no + way to recover data if it is lost. +- The 7-Zip engine passes the password on 7-Zip's command line, where other + users of the same machine could see it while the process runs. Use the native + engine on shared machines. +- A signature proves who made a manifest, not that it is the newest one. Someone + who can replace both the manifest and the volumes with an older, validly + signed set will not be detected; keep your own record of what the latest + version should be if that matters to you. +- On Windows, owner-only file permissions are not applied by BatchVault; private + keys, logs and archives rely on the permissions of the folder they are in. + Keep them in a folder only you can read. +- Encrypting or decrypting briefly needs about 128 MiB of memory for the password + hashing, on top of the compression work. +- Signing and encryption need `pip install "batchvault[crypto]"`. +- Keep private signing keys out of repositories and CI logs. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..cdcbec8 --- /dev/null +++ b/action.yml @@ -0,0 +1,255 @@ +name: BatchVault +description: Archive, split, verify and restore files with BatchVault. +author: coldsofttech + +branding: + icon: archive + color: blue + +inputs: + command: + description: "What to run: pack, unpack or verify." + default: pack + files: + description: Files to archive (pack), one path per line. + default: "" + list-file: + description: Text or JSON list of files to archive (pack). + default: "" + source: + description: Manifest, folder, volume or list file (unpack and verify). + default: "" + output: + description: Output folder. Defaults to archives (pack) or restored (unpack). + default: "" + engine: + description: auto, native or 7zip (pack). + default: auto + format: + description: Native engine format - xz, gzip, bzip2 or store (pack). + default: "" + level: + description: Compression level 0-9, or auto to pick one per file type (pack). + default: auto + volume-size: + description: Volume size such as 500k, 10m or 2g (pack). + default: 10m + jobs: + description: Worker threads to use; 0 means none, one file at a time. + default: "0" + include: + description: Only keep matching paths, one glob per line (pack). + default: "" + exclude: + description: Drop matching paths, one glob per line (pack). + default: "" + root: + description: Keep folders, storing each file relative to this folder (pack). + default: "" + force: + description: Replace existing archives (pack) or files (unpack). + default: "false" + verify: + description: Test each archive after creating it (pack). + default: "false" + manifest: + description: Manifest location (pack). + default: "" + against: + description: Folder of original files to compare with (verify). + default: "" + seven-zip: + description: Path to a 7-Zip executable (optional). + default: "" + version: + description: >- + BatchVault version to install from PyPI. Empty installs the version + that matches this action's ref. + default: "" + python-version: + description: Python version used to run BatchVault. + default: "3.12" + upload-artifact: + description: Upload the output folder as a workflow artifact. + default: "false" + artifact-name: + description: Name of the uploaded artifact. + default: batchvault + update: + description: Skip unchanged files and re-archive changed ones (pack, unpack). + default: "false" + retries: + description: Retry a file this many times after an I/O or engine error. + default: "3" + encrypt: + description: Encrypt the archives (pack). Needs the password input. + default: "false" + password: + description: Archive password for pack --encrypt, unpack and verify. Use a secret. + default: "" + sign-key: + description: Path to an Ed25519 private key used to sign the manifest (pack). + default: "" + public-key: + description: Public key; unpack and verify then require a valid signature. + default: "" + fail-on-error: + description: Fail the step if any file failed or was skipped. + default: "true" + +outputs: + status: + description: success, failed, skipped or partial. + value: ${{ steps.run.outputs.status }} + result-json: + description: Path of the JSON result file. + value: ${{ steps.run.outputs.result-json }} + output: + description: The output folder that was used. + value: ${{ steps.run.outputs.output }} + +runs: + using: composite + steps: + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install BatchVault + shell: bash + env: + VERSION: ${{ inputs.version }} + ACTION_PATH: ${{ github.action_path }} + NEEDS_CRYPTO: ${{ inputs.encrypt == 'true' || inputs.password != '' || inputs.sign-key != '' || inputs.public-key != '' }} + run: | + extras="" + [ "$NEEDS_CRYPTO" = "true" ] && extras="[crypto]" + if [ -n "$VERSION" ]; then + python -m pip install --quiet "batchvault${extras}==$VERSION" + else + python -m pip install --quiet "${ACTION_PATH}${extras}" + fi + + - name: Run BatchVault + id: run + shell: bash + env: + COMMAND: ${{ inputs.command }} + FILES: ${{ inputs.files }} + LIST_FILE: ${{ inputs.list-file }} + SOURCE: ${{ inputs.source }} + OUTPUT: ${{ inputs.output }} + ENGINE: ${{ inputs.engine }} + FORMAT: ${{ inputs.format }} + LEVEL: ${{ inputs.level }} + VOLUME_SIZE: ${{ inputs.volume-size }} + JOBS: ${{ inputs.jobs }} + INCLUDE: ${{ inputs.include }} + EXCLUDE: ${{ inputs.exclude }} + ROOT: ${{ inputs.root }} + FORCE: ${{ inputs.force }} + VERIFY: ${{ inputs.verify }} + MANIFEST: ${{ inputs.manifest }} + AGAINST: ${{ inputs.against }} + SEVEN_ZIP: ${{ inputs.seven-zip }} + UPDATE: ${{ inputs.update }} + RETRIES: ${{ inputs.retries }} + ENCRYPT: ${{ inputs.encrypt }} + SIGN_KEY: ${{ inputs.sign-key }} + PUBLIC_KEY: ${{ inputs.public-key }} + BATCHVAULT_PASSWORD: ${{ inputs.password }} + FAIL_ON_ERROR: ${{ inputs.fail-on-error }} + PYTHONUTF8: "1" + BATCHVAULT_LOG_DIR: ${{ runner.temp }}/batchvault-logs + run: | + # Inputs reach the command only through environment variables and an + # argument array, never through shell interpolation. + case "$COMMAND" in + pack|unpack|verify) ;; + *) echo "::error::command must be pack, unpack or verify"; exit 2 ;; + esac + + # Keep the password out of the logs even if it did not come from a secret. + [ -n "$BATCHVAULT_PASSWORD" ] && echo "::add-mask::$BATCHVAULT_PASSWORD" + + # Options go first and file paths come after "--", so a path that starts + # with "-" is never read as an option. + args=(--json) + positional=() + [ -n "$JOBS" ] && args+=(--jobs "$JOBS") + [ -n "$SEVEN_ZIP" ] && args+=(--7z "$SEVEN_ZIP") + + case "$COMMAND" in + pack) + while IFS= read -r line; do [ -n "$line" ] && positional+=("$line"); done <<< "$FILES" + [ -n "$LIST_FILE" ] && args+=(--list "$LIST_FILE") + [ -n "$OUTPUT" ] && args+=(--output "$OUTPUT") + args+=(--engine "$ENGINE" --level "$LEVEL" --volume-size "$VOLUME_SIZE") + [ -n "$FORMAT" ] && args+=(--format "$FORMAT") + [ -n "$MANIFEST" ] && args+=(--manifest "$MANIFEST") + [ -n "$ROOT" ] && args+=(--root "$ROOT") + while IFS= read -r line; do [ -n "$line" ] && args+=(--include "$line"); done <<< "$INCLUDE" + while IFS= read -r line; do [ -n "$line" ] && args+=(--exclude "$line"); done <<< "$EXCLUDE" + [ "$VERIFY" = "true" ] && args+=(--verify) + [ "$FORCE" = "true" ] && args+=(--force) + [ "$UPDATE" = "true" ] && args+=(--update) + [ -n "$RETRIES" ] && args+=(--retries "$RETRIES") + [ "$ENCRYPT" = "true" ] && args+=(--encrypt) + [ -n "$SIGN_KEY" ] && args+=(--sign-key "$SIGN_KEY") + out_dir="${OUTPUT:-archives}" + ;; + unpack) + positional+=("$SOURCE") + [ -n "$OUTPUT" ] && args+=(--output "$OUTPUT") + [ "$FORCE" = "true" ] && args+=(--force) + [ "$UPDATE" = "true" ] && args+=(--update) + [ -n "$RETRIES" ] && args+=(--retries "$RETRIES") + [ -n "$PUBLIC_KEY" ] && args+=(--public-key "$PUBLIC_KEY") + out_dir="${OUTPUT:-restored}" + ;; + verify) + positional+=("$SOURCE") + [ -n "$AGAINST" ] && args+=(--against "$AGAINST") + [ -n "$PUBLIC_KEY" ] && args+=(--public-key "$PUBLIC_KEY") + out_dir="" + ;; + esac + + result="$RUNNER_TEMP/batchvault-result.json" + set +e + batchvault "$COMMAND" "${args[@]}" -- "${positional[@]}" > "$result" + code=$? + set -e + + status=$(python -c "import json,sys; print(json.load(open(sys.argv[1]))['status'])" "$result" 2>/dev/null || echo failed) + { + echo "status=$status" + echo "result-json=$result" + echo "output=$out_dir" + } >> "$GITHUB_OUTPUT" + + python - "$result" "$COMMAND" >> "$GITHUB_STEP_SUMMARY" <<'PY' || true + import json, sys + data = json.load(open(sys.argv[1], encoding="utf-8")) + print(f"### BatchVault {sys.argv[2]}: {data['status']}\n") + print("| File | Status | Details |\n|---|---|---|") + for entry in data["entries"]: + name = entry["source"].replace("\\", "/").rsplit("/", 1)[-1] + message = entry["message"].replace("|", "\\|").replace("\n", " ") + print(f"| {name} | {entry['status']} | {message} |") + for warning in data.get("warnings", []): + print(f"\n> Warning: {warning}") + PY + + if [ "$code" -ne 0 ] && [ "$FAIL_ON_ERROR" = "true" ]; then + echo "::error::BatchVault exited with code $code (status: $status)" + exit "$code" + fi + + - name: Upload output + if: ${{ always() && inputs.upload-artifact == 'true' && steps.run.outputs.output != '' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ steps.run.outputs.output }} + if-no-files-found: warn diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8e53ceb --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,164 @@ +# Architecture + +This document records the design as implemented. See `.temp/REQUIREMENTS.md` +for the original requirements. + +## Layers + +The command-line interface is a thin wrapper over the Python API in `api.py`. +The API never imports `rich`; it reports progress through an optional callback +and the CLI renders it. + +``` +src/batchvault/ + api.py pack, unpack, verify, inspect, list_files, list_archives + cli.py argparse commands, exit codes, JSON / event output + ui.py Rich progress, tables, panels (CLI only) + notify.py webhook notifications (CLI only) + config.py batchvault.toml / [tool.batchvault] -> option defaults + models.py Status, FileResult, RunResult, ProgressEvent, InspectEntry + options.py PackOptions, size parsing, formats + errors.py exception hierarchy + inputs.py list files (text/JSON/stdin), dedupe, conflict checks + filters.py include / exclude via pathspec (gitignore-style) + locate.py resolve a manifest / folder / volume / list to archives + manifest.py manifest read / write / validate, SHA-256 helpers + volumes.py volume naming, discovery, writer/reader, staging cleanup + blocks.py block-parallel compression, format detection + crypto.py optional AES-256-GCM record encryption (native engine) + signing.py optional Ed25519 manifest signatures + logs.py, security.py, eta.py, doctor.py + engines/ base, native, sevenzip, registry +``` + +## Engines + +| Engine | Format | Volumes | +|---|---|---| +| native | `tar` stream, compressed in independent blocks (xz, gzip, bzip2 or store), byte-split | `name.bv.001`, `.002`, ... | +| 7zip | `.7z` (LZMA2) through an installed 7-Zip executable | `name.7z.001`, ... | + +- Native uses the standard library. Concatenated xz, gzip and bzip2 members are + valid single streams, so an unencrypted joined volume set (`cat x.bv.*`) is a + normal `.tar.xz`, `.tar.gz` or `.tar.bz2`. +- 7-Zip is run with `subprocess` and never bundled. +- Archives are built in a `.batchvault-*` staging folder and moved into place + only when complete, so a failed or interrupted run never damages an existing + archive. Staging folders older than a day are removed on the next run. +- `--jobs` is a total thread budget, split into files in parallel and threads + per file (`-mmt` for 7-Zip, block threads for native). + +## Manifest + +JSON, `schema_version` 1: one entry per source file with name, `folder` (its +folder under the stored root, forward slashes, `""` when not in a subfolder; +chosen by `inputs.assign_folders`: an explicit `--root`, else the root recorded by +`list files`, else the deepest folder the files share; validated so +it can never be absolute or contain `..`), size, SHA-256, +engine, format, level, volume size, `encrypted`, `mtime_ns` and an ordered +volume list (relative path, size, SHA-256). Restoration never depends on folder +naming. Volume paths are relative to the manifest folder. The manifest is +written atomically and merged across runs. + +Because the manifest records every volume, a missing last volume is detected; +without a manifest only gaps in the numbering can be. + +## Encryption (`batchvault[crypto]`) + +- **Native:** a header (magic, format, random salt and nonce prefix) followed by + one AES-256-GCM record per compressed block and an authenticated end marker. + The key comes from the password with scrypt (N = 2^17, about 128 MiB of memory, + r = 8, p = 1). The cost is stored in the header as log2(N), so the default can + rise later, and values outside 2^14 to 2^18 are refused so a damaged header + cannot demand huge amounts of memory. The header, the record number and a + last-record flag are authenticated, so reordering, dropping or truncating + records is detected. A record larger than 128 MiB is refused as damage. Encrypted volumes are not plain tar streams. +- **7-Zip:** AES-256 with header encryption (`-p`, `-mhe=on`). 7-Zip only takes + the password on its command line, where other local users could see it while + the process runs. Prefer the native engine on shared machines. +- Passwords come from `BATCHVAULT_PASSWORD` or `--password-file`, never a flag. + +## Signing (`batchvault[crypto]`) + +Ed25519. `.sig` holds a base64 signature over the exact manifest +bytes. `unpack` and `verify` accept `--public-key` to require a valid signature +before touching any archive. The signature is checked on the exact bytes that are +then parsed (`locate.resolve_targets` takes a manifest loader), so the file +cannot be swapped between the check and the use. A signature proves who made a +manifest, not that it is the latest one: replacing a manifest and its volumes +with an older, validly signed set is not detected. Changing the manifest without re-signing removes +the stale signature and warns. + +## Safe source cleanup + +`pack --delete-source` restores each archive to a temporary folder, compares its +SHA-256 with the original, checks the original did not change, and only then +deletes it. A terminal confirmation is required unless `--yes` is given; +without a terminal and without `--yes` nothing is deleted. + +## Incremental mode, retries and resume + +- `pack --update` skips a file when its size, modification time and the + settings (engine, format, level, volume size, encryption) match the manifest, + and re-archives it otherwise. `unpack --update` skips files that already match + the recorded checksum. +- `--retries N` retries I/O and engine errors with exponential backoff. + Missing files, existing outputs and corrupt data are never retried. +- Resuming is re-running with `--update`: finished files are skipped and an + interrupted file is rebuilt, because nothing is installed until it is complete. + +## Configuration + +`batchvault.toml` (or `[tool.batchvault]` in `pyproject.toml`), found by walking +up from the current folder or given with `--config` / `BATCHVAULT_CONFIG`. +Precedence: command line, then config file, then built-in defaults. A file found +by searching (not named with `--config` / `BATCHVAULT_CONFIG`) is rejected if it +sets any option in `config.RESTRICTED_OPTIONS` (programs, webhooks, trusted keys, +password files, log files, deletion). See `docs/configuration.md`. + +## Hardening notes + +- 7-Zip is located without ever searching the current folder + (`engines/sevenzip.py::_search_path`). +- Resource limits on restore: the native engine charges each member's size + against a budget before extracting (budget = `--max-output-size`, and with a + manifest also the recorded file size), and refuses more than 4 MiB of data + after the tar end marker. The 7-Zip engine watches the output folder while + it extracts and stops the program as soon as the folder grows past the limit, + and refuses links and special files. +- 7-Zip is called with `-spd` so file names are never wildcard patterns. +- Volumes are made owner-only (`0600`) before they are moved into place, and the + manifest is written through an unpredictable `mkstemp` file, also owner-only. +- `--notify` values are masked in the logged command line. +- Staging cleanup never follows symbolic links. +- CI: third-party actions are referenced by major version tag (Dependabot keeps + them current), releases carry build provenance attestations, and pushed images + carry provenance and an SBOM. +- The GitHub Action builds `batchvault -- `. +- Restored tar members must be regular files with a single-part name, must not + be Windows-special names on Windows, must resolve inside the staging folder, + and are extracted with Python's `data` filter (hence Python 3.11.4+). + +## Output and exit codes + +`--json` prints the final result; `--json-events` prints one JSON object per +line (`plan`, `start`, `finish`, `result`). Progress and warnings go to stderr. +Exit codes: 0 success, 1 failed / skipped items, 2 usage or configuration +error, 3 unexpected error, 130 interrupted. + +## Logging + +One timestamped log file per run in the OS default folder +(`%LOCALAPPDATA%\BatchVault\Logs`, `~/Library/Logs/BatchVault`, +`$XDG_STATE_HOME/batchvault`; override with `BATCHVAULT_LOG_DIR`), with +credential redaction, owner-only permissions and the newest 20 kept. + +## Distribution + +PyPI (also usable through pipx), a GitHub Action (`action.yml`), and Docker +images on GHCR (native, and `-7zip`). + +## Third-party + +`rich` (MIT), `pathspec` (MPL-2.0), optional `cryptography` (Apache-2.0/BSD). +7-Zip is an optional external tool. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..3652ce6 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,361 @@ +# Command line guide + +``` +batchvault [options] +``` + +| Command | What it does | +|---|---| +| `pack` | Compress files into split volumes and write a manifest | +| `unpack` | Restore files from a manifest, folder, volume or list | +| `verify` | Check archives are complete and intact | +| `inspect` | Show what is in your archives: sizes, ratios, engine, format, completeness | +| `list files` | List the files in a folder | +| `list archives` | List the archives in a folder | +| `keygen`, `sign` | Create a signing key pair, sign a manifest | +| `doctor` | Check this machine's setup | + +Add `--help` to any command for its options. + +## Sources and archives + +Two words are used throughout: + +- A **source** is an original file: the thing you want to protect. +- An **archive** is what BatchVault makes from one source: a set of volumes + (`report.pdf.bv.001`, `.002`, ...) plus an entry in `manifest.json`. + +`pack` turns sources into archives. `unpack` turns archives back into files. +`verify` and `inspect` look at archives. All four tables start with the same +columns, **Source** (the original file), **Archive** (its first volume) and +**Status**, so the rows read the same way whichever command you run: + +``` +Pack: source files -> archives + Source Archive Status Details + report.pdf report.pdf.bv.001 OK 3 volumes, 28.4 MiB -> 9.1 MiB (32%), verified + notes.txt - SKIPPED archive already exists, use --force + +Unpack: archives -> restored files + Source Archive Status Details + report.pdf report.pdf.bv.001 OK 28.4 MiB restored, checksum verified + +Verify: archives + Source Archive Status Details + report.pdf report.pdf.bv.001 OK volumes, checksums and integrity test OK +``` + +`inspect` adds Engine, Format, Level, Original, Stored, Ratio and Volumes between +**Status** and **Details**. In `unpack` the Source is the file that was restored. + +**Status** is the outcome for that file: `OK`, `SKIPPED` (nothing was done, and +the details say why) or `FAILED`. **Details** is the explanation: for `OK` it +describes what was done (sizes, volumes, what was checked); for `SKIPPED` and +`FAILED` it gives the reason. + +Options every command has: `--json`, `-q/--quiet`, `--log-file`, `--log-level`, +`--no-log`, `--config`. + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | Everything succeeded | +| 1 | Some files failed or were skipped, or a check failed | +| 2 | Usage or configuration error (bad option, missing file, bad config) | +| 3 | Unexpected error (see the log) | +| 130 | Interrupted | + +## pack + +``` +batchvault pack [FILE...] [--list FILE|-] -o DIR [options] +``` + +Give files as arguments, with `--list`, or both. Each file gets its own folder +under the output folder, and one `manifest.json` records them all. Existing +archives are skipped unless you pass `--force` or `--update`. + +Your folder structure is kept: a file in a subfolder is archived under the same +subfolder (see "Folders are kept" below). + +| Option | Meaning | Default | +|---|---|---| +| `-o, --output DIR` | Output folder | `archives` | +| `--list FILE` | Text or JSON list of files (`-` reads standard input) | | +| `--engine auto\|native\|7zip` | Engine to use | `auto` | +| `--format xz\|gzip\|bzip2\|store` | Built-in engine compression; use `store` for video, photos and other already-compressed files ([guide](compression.md)) | `xz` | +| `--level 0-9\|auto` | Compression level, or `auto` to pick one for each file from its type ([guide](compression.md#automatic-level)) | `auto` | +| `--volume-size SIZE` | `500k`, `10m`, `2g`, ... | `10m` | +| `-j, --jobs N` | Worker threads; `0` means none, one file at a time | `0` | +| `--include GLOB`, `--exclude GLOB` | Filter files; repeatable; `@FILE` reads patterns from a file; exclude wins | | +| `--force` | Replace existing archives | | +| `--update` | Skip unchanged files, re-archive changed ones | | +| `--verify` | Test each archive after creating it | | +| `--encrypt` | Encrypt (password from `BATCHVAULT_PASSWORD` or `--password-file`) | | +| `--sign-key FILE` | Sign the manifest with this private key | | +| `--delete-source` | Delete each original after a full restore-and-compare proves the archive | | +| `--yes` | Do not ask before `--delete-source` deletes | | +| `--retries N` | Retry a file after an I/O or engine error | `3` | +| `--manifest PATH` | Manifest location | `/manifest.json` | +| `--root DIR` | Advanced: store paths relative to `DIR` instead of the automatic choice | automatic | +| `--7z PATH` | 7-Zip executable (or `BATCHVAULT_7Z`) | found automatically | +| `--allow-archives` | Allow archiving volumes and manifests | | +| `--strict` | Treat settings the engine cannot honour as errors | | +| `--dry-run` | Show what would happen, write nothing | | +| `--notify URL`, `--notify-on` | Webhook summary (`failure`, `success`, `always`) | `failure` | +| `--json-events` | One JSON event per line on standard output | | + +`--delete-source` never deletes without an explicit `--yes` unless you answer +`y` at a terminal prompt. + +### Folders are kept + +Your folder structure is kept automatically; no option is needed. A file in a +subfolder is archived under the same subfolder, and `unpack` puts it back in the +same place. + +``` +D:\Media\report.pdf +D:\Media\Holidays\holiday.mp4 + +batchvault list files D:\Media -o files.json +batchvault pack --list files.json -o archives +archives\Media\report.pdf\report.pdf.bv.001 +archives\Media\Holidays\holiday.mp4\holiday.mp4.bv.001 + +batchvault unpack archives -o restored +restored\Media\report.pdf +restored\Media\Holidays\holiday.mp4 +``` + +Where the stored paths start from: + +1. **Files from `list files`:** the folder you listed is kept, including its own + name (`Media` above). `list files` records this in its output (a `root` value + in JSON, or a `# batchvault-root:` first line in a text list). +2. **Files you give directly** (arguments, or a hand-written list): the folders + below the deepest folder they all share are kept. Files that sit directly in + that folder are stored without a folder, so `pack a.pdf b.pdf` from one folder + gives `archives\a.pdf\...` and `archives\b.pdf\...`. +3. **`--root DIR`** (optional): overrides both. Paths are stored relative to `DIR`, + and every file must be inside it. `--root D:\` keeps `Media`; `--root D:\Media` + leaves it out. + +Notes: + +- The folders are recorded in `manifest.json`, so restoring needs it. Archives + restored without their manifest (from a single volume, for example) come back + flat. +- Files with the same name in different folders are fine. Only two files that + would end up in the very same place are refused. +- `--update`, `verify --against` and the result tables all use the full path, so + two `notes.txt` files in different folders are never confused. +- If you edit a list so that a file is no longer inside its recorded folder, that + file falls back to rule 2. + +## unpack + +``` +batchvault unpack SOURCE -o DIR [options] +``` + +`SOURCE` can be a manifest, a folder, any single volume (`.003` finds the whole +set), or a list of volumes. With a manifest, volumes and the restored file are +checked against their checksums, and nothing is placed in the output folder if a +check fails. Existing files are skipped unless `--force` is given; `--update` +skips files that already match and replaces the rest. + +Other options: `-j`, `--retries`, `--7z`, `--password-file`, `--public-key FILE` +(require a valid manifest signature first), and `--max-output-size SIZE`. + +`--max-output-size` (for example `50g`) refuses any archive that would expand +beyond that size, before anything is written. Use it when you restore archives +you do not trust: a tiny archive can be built to expand to terabytes. With a +manifest you are already protected, because an archive may never expand beyond +the size the manifest records for its file. + +Files are written with owner-only permissions on Linux and macOS. + +## verify + +``` +batchvault verify SOURCE [--against FOLDER] [--public-key FILE] +``` + +Checks volume sizes and checksums, then the archive's own integrity test. +`--against` also compares each archive with its original, looked up by file name +in that folder and then in its subfolders. + +## inspect + +``` +batchvault inspect ARCHIVES [--json] +``` + +Shows what is in a set of **archives** you already made. It does not look at the +original files. `ARCHIVES` is where the archives are: a manifest, an archive +folder, any single volume, or a list of volumes (the same inputs as `unpack` and +`verify`; `--help` calls this argument `SOURCE`). + +It looks like the other commands: a panel with the totals, then one row per +archived file, then a summary line. + +``` +BatchVault inspect + Archives archives + Files 2 + Size 28.4 MiB -> 9.1 MiB (32%) + +Inspect: archives + Source Archive Status Engine Format Level Original Stored Ratio Volumes Details + report.pdf report.pdf.bv.001 OK native xz 5 28.4 MiB 9.1 MiB 32% 3 + secrets.db secrets.db.bv.001 MISSING native xz 5 1.0 MiB 0.4 MiB 40% 2 encrypted, some volumes are missing + +Summary: 2 archives, 1 with missing volumes +``` + +| Column | Meaning | +|---|---| +| Source | The original file (with its stored folder, if any) | +| Archive | The first volume of its archive | +| Status | `OK` if every volume the manifest lists is present, `MISSING` if not, `-` when there is no manifest to check against | +| Engine, Format, Level | How it was archived (native or 7-Zip; xz, gzip, bzip2, store or 7z; 0-9) | +| Original, Stored | Size of the original file, and the total size of its volumes | +| Ratio | Stored size as a percentage of the original | +| Volumes | How many volumes there are | +| Details | `encrypted`, `some volumes are missing`, or `no manifest: sizes and checks unknown` | + +Without a manifest, only the file name, engine (guessed from `.bv` or `.7z`), +stored size and volume count are known; the other columns show `-`. + +`inspect` is a quick look. It checks that the volume files exist, but it does not +read them or check checksums; use `verify` for that. `--json` prints the same +information for scripts. + +``` +batchvault inspect archives +batchvault inspect archives/manifest.json --json +batchvault inspect archives/report.pdf/report.pdf.bv.002 +``` + +## list + +``` +batchvault list files FOLDER [-o FILE | --output FILE] [--include GLOB] [--exclude GLOB] +batchvault list archives FOLDER [-o FILE | --output FILE] +``` + +With no `-o` / `--output`, a JSON list goes to standard output. With it, the +extension decides: `.json` writes JSON, anything else writes one path per line. +Both can be fed straight back into `pack --list` / `unpack`. + +``` +batchvault list files D:\Media -o files.json +batchvault list archives archives --output found.txt +``` + +If you run `list files` on an archive folder, or `list archives` on a plain +folder, a warning suggests the other command. + +## keygen and sign + +``` +batchvault keygen --private batchvault.key --public batchvault.pub +batchvault sign archives/manifest.json --key batchvault.key +``` + +Needs `batchvault[crypto]`. If `BATCHVAULT_KEY_PASSWORD` is set, the private key +is stored encrypted. Keep the private key secret; share the public key. + +## File names that start with a dash + +Put `--` before the file names so they are never read as options: + +``` +batchvault pack -o archives -- -notes.txt --weird-name.csv +``` + +## Input lists + +A text list has one path per line. Blank lines and lines starting with `#` are +ignored, and paths may contain spaces and Unicode (do not add quotes). A JSON +list is an array of paths or `{"path": ...}` objects, which is exactly what +`list files` prints. + +## Examples + +Archive two files with a progress bar and a final summary: + +``` +batchvault pack report.pdf "holiday video.mp4" -o archives +``` + +Archive everything in a folder except temporary files, 1 GB volumes, using the +built-in engine, and check the result: + +``` +batchvault list files D:\Media --exclude "*.tmp" | batchvault pack --list - \ + -o archives --engine native --volume-size 1g --verify +``` + +Preview without writing anything: + +``` +batchvault pack --list files.txt -o archives --dry-run +``` + +Re-run later and only archive what changed: + +``` +batchvault pack --list files.txt -o archives --update +``` + +Encrypt, then restore: + +``` +export BATCHVAULT_PASSWORD='correct horse' # PowerShell: $env:BATCHVAULT_PASSWORD = '...' +batchvault pack secrets.db -o vault --encrypt +batchvault unpack vault -o restored +``` + +Sign the manifest and refuse to restore anything that was changed: + +``` +batchvault keygen +batchvault pack --list files.txt -o archives --sign-key batchvault.key +batchvault verify archives --public-key batchvault.pub +batchvault unpack archives -o restored --public-key batchvault.pub +``` + +Archive, then delete the originals once each archive is proven good: + +``` +batchvault pack --list files.txt -o archives --delete-source --yes +``` + +Restore a single file from one of its volumes: + +``` +batchvault unpack archives/report.pdf/report.pdf.bv.002 -o restored +``` + +Use it in a script: + +``` +batchvault pack --list files.txt -o archives --json > result.json +echo $? # 0 if everything worked +``` + +Watch progress from another program (one JSON object per line): + +``` +batchvault pack --list files.txt -o archives --json-events +``` + +Get a message in Slack, Teams or Discord when something goes wrong: + +``` +export BATCHVAULT_NOTIFY_URL='https://hooks.example.com/...' +batchvault pack --list files.txt -o archives +``` diff --git a/docs/compression.md b/docs/compression.md new file mode 100644 index 0000000..058d836 --- /dev/null +++ b/docs/compression.md @@ -0,0 +1,199 @@ +# Compression + +How BatchVault compresses, which format and level to choose, and what they cost +in time and memory. + +## The short version + +- **Already-compressed files** (video, audio, photos, ZIP/7z/RAR, most PDFs and + Office files): use `--format store`. Nothing can shrink them, so compressing + only wastes time and memory. +- **Text-like files** (logs, CSV, JSON, SQL dumps, source code, uncompressed + images): the default (`xz`, level `auto`, which picks 6 for text) is a very good choice. Level 9 rarely adds + more than a few percent. +- **Mixed folders:** use `--level auto`, which picks a level for each file from its + type (see "Automatic level"), or keep the defaults. + +``` +batchvault pack --list files.json -o archives --format store # videos, photos, archives +batchvault pack --list files.json -o archives # defaults: xz, level auto +``` + +## What happens to a file + +1. **Native engine:** the file is written into a tar stream, cut into 16 MiB + blocks, and each block is compressed on its own (in parallel, up to `--jobs` + at a time). The compressed blocks are joined and split into volumes of + `--volume-size`. +2. **7-Zip engine:** 7-Zip builds a `.7z` archive (LZMA2) and splits it into + volumes itself. +3. If you encrypt (`--encrypt`), the data is compressed first and encrypted + after, so encryption does not hurt the compression ratio. + +Volume size has no effect on how well data compresses; it only decides where the +archive is cut. + +## Formats + +`--format` applies to the **native** engine. The 7-Zip engine always writes 7z +(LZMA2) and ignores it with a warning (an error with `--strict`). + +| Format | What it is | Ratio | Speed | Memory | Choose it when | +|---|---|---|---|---|---| +| `xz` (default) | LZMA2 | Best | Slow | Highest at high levels | Text, data and anything that compresses well; archives you keep long term | +| `gzip` | Deflate | Good | Fast | Very low | You want speed, or a quick backup of text-like data | +| `bzip2` | Block sorting | Between gzip and xz | Slowest | Low | Rarely; mostly for compatibility | +| `store` | No compression | None | Fastest | Lowest | Video, audio, photos, other archives | + +Notes: + +- `store` still splits the file into volumes and adds the manifest and checksums. + It is the right choice when you only need the splitting and the safety checks. +- With `store`, `--level` has no effect. +- Native `xz` and the 7-Zip engine both use LZMA2, so on the same data their + sizes are close. + +## Levels + +`--level` goes from 0 to 9 (default `auto`; a fixed level is 5 when unsure). Higher levels search harder for +repetition: smaller files, but more time and more memory. + +| Level | Meaning | +|---|---| +| 0 | No compression (gzip stored, xz fastest preset). With the 7-Zip engine, level 0 is "copy" mode, the same as store | +| 1 to 3 | Fast; a good fit for quick backups | +| 4 to 6 | Balanced. **5 is the default** | +| 7 to 9 | Slow and memory hungry; gains over 6 are usually small | + +Details per format: + +- **xz:** the level is the xz preset (0 to 9). +- **gzip:** the level is the deflate level (0 to 9; 0 stores). +- **bzip2:** levels 1 to 9 set the block size. bzip2 has no level 0, so 0 uses 1. +- **7-Zip:** the level is `-mx`. + +### Automatic level + +`--level auto` picks a level for each file from its type, so one command can +handle a mixed folder sensibly: + +``` +batchvault pack --list files.json -o archives --level auto +``` + +BatchVault looks at the file's first bytes (its signature) and then its extension. +The signature wins, so a video renamed to `.txt` is still recognised. + +| Type | Level | Why | +|---|---|---| +| Already compressed: video (mp4, mkv, mov, avi, webm), photos (jpg, png, gif, webp, heic, camera raw), audio (mp3, aac, flac, ogg), archives (zip, 7z, rar, gz, xz, bz2, zst), Office and ebook files (docx, xlsx, pptx, odt, epub) | 0 | Nothing to gain; a higher level only costs time and memory | +| Text and code: txt, log, csv, json, xml, html, sql, md, source files | 6 | Highly repetitive; higher levels add little | +| Uncompressed pictures and sound: wav, bmp, tiff | 6 | Large savings | +| Programs, disk images, databases, tar: exe, dll, iso, vmdk, sqlite | 5 | Compressible, but the strongest levels cost a lot of memory for little | +| PDF | 3 | Usually partly compressed already | +| Anything else | 5 | The default | + +Things to know: + +- Level 0 means the lowest level of the chosen compressor. The file is still run + through it (fast, almost no gain), and the archive format stays the same. + If you want files that cannot shrink to skip compression entirely, use + `--format store` instead. +- The level used for each file is shown in the result (`level 6 (text)`) and + recorded in the manifest, so `--update` re-archives a file only if its chosen + level changes. +- It judges the type, not the contents. A `.bin` or `.dat` file that happens to + be already compressed gets level 5 and will not shrink; use `--format store` for + those. +- With `--format store`, `auto` has no effect. +- `auto` never picks 7, 8 or 9, so it also keeps memory use down. + +### Memory + +Each compression thread needs its own memory, so total use is roughly +`threads x memory per thread`, plus the blocks waiting to be compressed +(`--jobs` is the thread budget). + +Approximate xz memory per thread: + +| xz level | Compress | Decompress | +|---|---|---| +| 0 | 3 MiB | 1 MiB | +| 1 | 9 MiB | 2 MiB | +| 2 | 17 MiB | 3 MiB | +| 3 | 32 MiB | 5 MiB | +| 4 | 48 MiB | 5 MiB | +| 5 and 6 | 94 MiB | 9 MiB | +| 7 | 186 MiB | 17 MiB | +| 8 | 370 MiB | 33 MiB | +| 9 | 674 MiB | 65 MiB | + +At level 9 with 8 threads that is more than 5 GiB. If a run is slow or the machine +starts swapping, lower the level or `--jobs`. gzip and bzip2 need only a few MiB +per thread. Encrypting adds about 128 MiB for a moment while the password is +turned into a key. + +## Which data compresses + +| Data | Typical result | +|---|---| +| MP4, MKV, MOV, MP3, AAC, JPEG, PNG, WebP | 0 to 2 percent smaller; use `store` | +| ZIP, 7z, RAR, GZ, XZ, DOCX, XLSX, PPTX | About 0 percent; use `store` | +| PDF | Varies, often little | +| Text, logs, CSV, JSON, XML, SQL dumps, source code | 70 to 90 percent smaller | +| WAV, BMP, TIFF, raw images | Often 30 to 70 percent smaller | +| Databases, VM and disk images | Depends on how empty they are; often large savings | +| Encrypted files | About 0 percent: encrypted data looks random | + +You can tell whether compression helped from the result table after `pack`: + +``` +report.pdf OK report.pdf.bv.001 3 volumes, 28.4 MiB -> 9.1 MiB (32%) +holiday.mp4 OK holiday.mp4.bv.001 3 volumes, 28.4 MiB -> 28.2 MiB (99%) +``` + +A percentage near 100 means the file was already compressed. Next time use +`--format store` for files like that. + +## Block compression and ratio + +The native engine compresses 16 MiB blocks independently. This is what lets it use +all your threads on a single large file, and a block can be read without the +others. The cost is that repetition between distant blocks is not exploited, so +for data with a lot of long-range repetition the result can be slightly larger +than one big xz stream. For most data the difference is small. Files smaller than +one block are not affected. + +## Getting your files back without BatchVault + +- **Native, not encrypted:** joining the volumes gives a standard file. For + example `cat name.bv.* > name.tar.xz` (gzip: `.tar.gz`, bzip2: `.tar.bz2`, + store: `.tar`), and any tar tool opens it. +- **Native, encrypted:** only BatchVault can restore it. +- **7-Zip:** open the `.001` volume with 7-Zip. + +## Recipes + +``` +# Course videos and photos: just split, keep checksums +batchvault pack --list files.json -o archives --format store --volume-size 500m + +# Logs and database dumps: strong compression, modest memory +batchvault pack --list files.json -o archives --level 6 --jobs 4 + +# A mixed folder: each file gets the level that suits its type +batchvault pack --list files.json -o archives --level auto + +# Fast first pass over a big mixed folder +batchvault pack --list files.json -o archives --format gzip --level 3 + +# Smallest possible size, plenty of memory and time +batchvault pack --list files.json -o archives --level 9 --jobs 2 + +# 7-Zip engine, copy mode (same as store) +batchvault pack --list files.json -o archives --engine 7zip --level 0 +``` + +`--level auto` chooses the level for each file but does not change the format. +The ratio shown after each file tells you when a file would have been better +stored with `--format store`. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..bf4b6c4 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,75 @@ +# Configuration + +BatchVault reads defaults from `batchvault.toml`, or from a `[tool.batchvault]` +table in `pyproject.toml`. It looks in the current folder and each parent, or +use `--config FILE` / `BATCHVAULT_CONFIG`. Precedence: command line, config +file, built-in defaults. `include` and `exclude` lists from the file and the +command line are combined. An on/off option set to `true` in the file cannot be +switched off from the command line. + +## Trusting a config file + +A config file found by searching the current folder and its parents might belong +to someone else (a cloned repository, a downloaded folder). So a file found this +way **cannot** set options that run programs, send data out, choose trusted keys, +read passwords or delete files: + +`seven-zip`, `notify` (and `[notify] url`), `delete-source`, `yes`, `public-key`, +`sign-key`, `password-file`, and `log-file` (and `[log] file`). + +Using one of them in such a file is an error that tells you how to proceed. If you +wrote the file and trust it, name it yourself, which counts as trusting it: + +``` +batchvault pack --list files.txt --config batchvault.toml +``` + +or set `BATCHVAULT_CONFIG` to its path. Everything else (levels, volume size, +engine, filters, retries, `update`, ...) works from a found file as usual. + +## Keys + +Keys use the option names (dashes or underscores): `volume-size` for +`--volume-size`. Unknown sections, keys or invalid values are errors (exit +code 2). + +```toml +[defaults] # every command that has the option +jobs = 4 +seven-zip = "/usr/bin/7zz" # --7z; only allowed with --config (see "Trusting a config file") + +[log] +level = "DEBUG" # --log-level +file = "run.log" # --log-file + +[notify] +url = "https://hooks.example.com/..." # --notify +on = "failure" # failure, success or always + +[pack] # pack, unpack, verify and inspect have their own tables +engine = "native" +level = 7 +volume-size = "100m" +include = ["*.mp4"] +exclude = ["tmp/"] +update = true +retries = 2 + +[unpack] +update = true +``` + +Never put passwords in the file. Use `BATCHVAULT_PASSWORD`, +`--password-file`, and `BATCHVAULT_KEY_PASSWORD` for an encrypted signing key. +A notification URL often contains a secret; prefer `BATCHVAULT_NOTIFY_URL`. + +## Environment variables + +| Variable | Meaning | +|---|---| +| `BATCHVAULT_CONFIG` | Config file path | +| `BATCHVAULT_7Z` | 7-Zip executable | +| `BATCHVAULT_PASSWORD` | Archive password (used by `--encrypt`, `unpack`, `verify`) | +| `BATCHVAULT_KEY_PASSWORD` | Password of the signing private key | +| `BATCHVAULT_NOTIFY_URL` | Webhook URL for `--notify` | +| `BATCHVAULT_LOG_DIR` | Folder for log files | diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 0000000..065db23 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,30 @@ +# Development + +## Setup + +``` +uv sync +uv run pre-commit install +``` + +## Common tasks + +``` +uv run pytest -m "not integration and not e2e" -n auto # unit tests +uv run pytest -m integration # needs 7-Zip +uv run pytest -m "e2e and not docker" # CLI and package, end to end +uv run pytest -m docker # needs Docker +uv run ruff check . # lint +``` + +See [testing.md](testing.md) for what each layer covers and when CI runs it. + +## Releasing + +1. Update the version in `pyproject.toml` and `src/batchvault/__init__.py`. +2. Move the `[Unreleased]` changelog entries under the new version. +3. Push a tag `vX.Y.Z`. The release workflow checks the versions, runs lint and + tests, publishes to PyPI via trusted publishing, and creates a GitHub release. + +PyPI trusted publishing needs the repository and the `pypi` environment +registered as a trusted publisher in the PyPI project settings. diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..68db6f5 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,84 @@ +# Docker guide + +Images are published to the GitHub Container Registry: + +| Image | Contains | +|---|---| +| `ghcr.io/coldsofttech/batchvault:` (and `:latest`) | BatchVault with the built-in engine, encryption and signing | +| `ghcr.io/coldsofttech/batchvault:-7zip` (and `:latest-7zip`) | The same, plus 7-Zip | + +Both support `linux/amd64` and `linux/arm64`. The 7-Zip variant installs 7-Zip +from the operating system's package; it is separate software under its own +license. + +## Running + +The container works in `/data`, so mount the folder you want to work in there: + +``` +docker run --rm -v "$PWD:/data" ghcr.io/coldsofttech/batchvault \ + pack report.pdf -o archives --volume-size 100m +``` + +``` +docker run --rm -v "$PWD:/data" ghcr.io/coldsofttech/batchvault \ + unpack archives -o restored +``` + +Show the help: + +``` +docker run --rm ghcr.io/coldsofttech/batchvault --help +``` + +Use 7-Zip: + +``` +docker run --rm -v "$PWD:/data" ghcr.io/coldsofttech/batchvault:latest-7zip \ + pack report.pdf -o archives --engine 7zip +``` + +## Passwords and keys + +Pass secrets through a mounted file or the environment, not on the command +line. A value given with `-e` can be seen by anyone who can run `docker inspect` +on the container, so a mounted password file (below) is safer: + +``` +docker run --rm -v "$PWD:/data" -e BATCHVAULT_PASSWORD \ + ghcr.io/coldsofttech/batchvault pack secrets.db -o vault --encrypt +``` + +``` +docker run --rm -v "$PWD:/data" ghcr.io/coldsofttech/batchvault \ + unpack vault -o restored --password-file /data/pw.txt +``` + +## File ownership + +The container runs as user ID 1000. On Linux, files it creates in the mounted +folder belong to that user. To keep your own ownership: + +``` +docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/data" \ + ghcr.io/coldsofttech/batchvault pack report.pdf -o archives +``` + +## Logs and settings + +Logs are written inside the container and disappear with it. To keep them, +mount a folder and point `BATCHVAULT_LOG_DIR` at it: + +``` +docker run --rm -v "$PWD:/data" -v "$PWD/logs:/logs" -e BATCHVAULT_LOG_DIR=/logs \ + ghcr.io/coldsofttech/batchvault pack report.pdf -o archives +``` + +A `batchvault.toml` in the mounted folder is picked up automatically. + +## Building the image yourself + +``` +docker build -t batchvault . +docker build --build-arg WITH_7ZIP=true -t batchvault:7zip . +``` diff --git a/docs/github-action.md b/docs/github-action.md new file mode 100644 index 0000000..de0db51 --- /dev/null +++ b/docs/github-action.md @@ -0,0 +1,138 @@ +# GitHub Action guide + +```yaml +- uses: coldsofttech/BatchVault@v1 + with: + command: pack + files: | + reports/summary.pdf + data/export.csv + output: archives +``` + +The action installs BatchVault, runs one command, writes a per-file table to the +job summary, and fails the step if anything failed or was skipped (turn that off +with `fail-on-error: false`). + +By default it installs the version that matches the action's own ref +(`@v1`, a tag or a commit). Set `version` to install a specific release from +PyPI instead. + +## Inputs + +| Input | Used by | Meaning | Default | +|---|---|---|---| +| `command` | all | `pack`, `unpack` or `verify` | `pack` | +| `files` | pack | Files to archive, one path per line | | +| `list-file` | pack | Text or JSON list of files | | +| `source` | unpack, verify | Manifest, folder, volume or list | | +| `output` | pack, unpack | Output folder | `archives` / `restored` | +| `engine` | pack | `auto`, `native` or `7zip` | `auto` | +| `format` | pack | `xz`, `gzip`, `bzip2`, `store` (built-in engine) | | +| `level` | pack | Compression level 0-9, or `auto` to pick one for each file from its type | `auto` | +| `volume-size` | pack | e.g. `500k`, `10m`, `2g` | `10m` | +| `jobs` | all | Worker threads; `0` means none, one file at a time | `0` | +| `include`, `exclude` | pack | Globs, one per line | | +| `root` | pack | Advanced: store paths relative to this folder instead of the automatic choice (folders are kept either way) | automatic | +| `force` | pack, unpack | Replace existing archives / files | `false` | +| `update` | pack, unpack | Skip unchanged files | `false` | +| `verify` | pack | Test each archive after creating it | `false` | +| `retries` | pack, unpack | Retries after an I/O or engine error | `3` | +| `manifest` | pack | Manifest location | | +| `against` | verify | Folder of originals to compare with | | +| `encrypt` | pack | Encrypt the archives (needs `password`) | `false` | +| `password` | pack, unpack, verify | Archive password; use a secret | | +| `sign-key` | pack | Private key path for signing the manifest | | +| `public-key` | unpack, verify | Require a valid manifest signature | | +| `seven-zip` | all | Path to a 7-Zip executable | | +| `version` | all | BatchVault version from PyPI | matches the action ref | +| `python-version` | all | Python used to run BatchVault | `3.12` | +| `upload-artifact` | all | Upload the output folder as an artifact | `false` | +| `artifact-name` | all | Name of that artifact | `batchvault` | +| `fail-on-error` | all | Fail the step on any failure or skip | `true` | + +## Outputs + +| Output | Meaning | +|---|---| +| `status` | `success`, `failed`, `skipped` or `partial` | +| `result-json` | Path of the JSON result file | +| `output` | The output folder that was used | + +## Examples + +Archive build output in 100 MB volumes and keep it as an artifact: + +```yaml +- uses: coldsofttech/BatchVault@v1 + with: + command: pack + files: | + build/app.zip + build/symbols.zip + volume-size: 100m + verify: true + upload-artifact: true + artifact-name: release-archives +``` + +Archive everything under a folder, except logs (using a list file): + +```yaml +- name: List files + run: | + pip install batchvault + batchvault list files data --exclude "*.log" -o files.json + +- uses: coldsofttech/BatchVault@v1 + with: + list-file: files.json + output: archives +``` + +Encrypt with a password from a repository secret: + +```yaml +- uses: coldsofttech/BatchVault@v1 + with: + files: secrets/database.dump + encrypt: true + password: ${{ secrets.ARCHIVE_PASSWORD }} +``` + +Restore, refusing a manifest that was not signed with your key: + +```yaml +- uses: coldsofttech/BatchVault@v1 + with: + command: unpack + source: archives + output: restored + public-key: keys/batchvault.pub + password: ${{ secrets.ARCHIVE_PASSWORD }} +``` + +Use the result in a later step: + +```yaml +- id: vault + uses: coldsofttech/BatchVault@v1 + with: + files: data.db + fail-on-error: false + +- if: steps.vault.outputs.status != 'success' + run: echo "Archiving did not fully succeed: ${{ steps.vault.outputs.status }}" +``` + +## Notes + +- Inputs reach the command through environment variables and an argument list, + never through shell interpolation, so a value cannot inject commands. File + paths are passed after `--`, so a path that starts with `-` is never read as + an option, even if it comes from untrusted data. +- The `password` input is masked in the log. +- GitHub-hosted runners usually include 7-Zip, so `engine: auto` may pick it. + Set `engine: native` for identical behaviour on every runner. +- Pin `@v1` (a moving tag), a full version tag, or a commit SHA, depending on how + much stability you need. diff --git a/docs/python.md b/docs/python.md new file mode 100644 index 0000000..d16130d --- /dev/null +++ b/docs/python.md @@ -0,0 +1,173 @@ +# Python guide + +Everything the command line does is available as plain functions. The library +never prints and never imports the terminal UI; it returns structured results +and raises meaningful exceptions. + +```python +import batchvault as bv + +result = bv.pack( + ["report.pdf", "holiday.mp4"], + "archives", + level=7, + volume_size="100m", + verify=True, +) + +print(result.status) # Status.SUCCESS / FAILED / SKIPPED / PARTIAL +for entry in result.entries: + print(entry.source.name, entry.status, entry.message) +``` + +## Functions + +| Function | Purpose | +|---|---| +| `bv.pack(files, output, **options)` | Archive files; returns a `RunResult` | +| `bv.unpack(source, output, **options)` | Restore files; returns a `RunResult` | +| `bv.verify(source, **options)` | Check archives; returns a `RunResult` | +| `bv.inspect(source)` | Describe existing archives (`source` is a manifest, archive folder, volume or list of volumes, not the original files); returns `list[InspectEntry]` | +| `bv.list_files(folder, include=, exclude=)` | Files in a folder; `list[FileEntry]` | +| `bv.list_archives(folder)` | Archives in a folder; `list[FileEntry]` | + +Paths can be `str` or `pathlib.Path`. Sizes accept `"10m"` or a number of bytes. + +### pack + +```python +bv.pack( + files=(), output="archives", *, + list_file=None, engine="auto", seven_zip=None, format=None, + level="auto", # 0-9, or "auto" to pick one for each file from its type + volume_size="10m", jobs=0, + root=None, # optional override for where stored folders start + include=(), exclude=(), force=False, update=False, verify=False, + manifest=None, allow_archives=False, strict=False, dry_run=False, + password=None, retries=3, sign_key=None, sign_key_password=None, + delete_source=False, confirm=None, progress=None, +) +``` + +- `engine`: `"auto"`, `"native"` or `"7zip"`. `format`: `"xz"`, `"gzip"`, + `"bzip2"` or `"store"` (built-in engine only). +- `password` encrypts the archives (the built-in engine needs + `batchvault[crypto]`). `sign_key` is the path of an Ed25519 private key. +- `delete_source=True` deletes each original after a restore-and-compare proves + the archive. `confirm(paths) -> bool`, if given, is asked first; with no + `confirm`, deletion is automatic once the check passes. +- `jobs` is the thread budget; `0` (the default) uses no worker threads and handles one file at a time. +- Folders are kept automatically: a file is stored under + `///` and `unpack` recreates the tree. The folder starts + from the root that `bv.list_files()` recorded on each `FileEntry` (so the listed + folder's name is kept), or, for files you pass directly, from the deepest + folder they all share. `root` overrides this with a folder of your choice. + Files with the same name in different folders are fine. `FileResult.folder` and + `InspectEntry.folder` hold the stored folder (`""` when the file is not in a + subfolder). + + ```python + files = bv.list_files("D:/Media") # each entry remembers D:/ as its root + bv.pack([f.path for f in files], "archives") # shared folder: keeps below D:/Media + # to keep "Media" too, pass the listing's root: + bv.pack([f.path for f in files], "archives", root=files[0].root) + ``` + +### unpack and verify + +```python +result = bv.unpack("archives", "restored", force=False, update=False, + password=None, public_key=None, retries=3) + +result = bv.verify("archives", against="originals", password=None, + public_key=None) +``` + +`source` can be a manifest, a folder, any volume, or a list of volumes. +`public_key` requires a valid manifest signature first. `max_output_size` +(`"50g"` or bytes) refuses an archive that would expand beyond that size before +anything is written; with a manifest, an archive can never expand beyond the +size recorded for its file. + +## Results + +`RunResult` has: + +- `status`: `Status.SUCCESS`, `FAILED`, `SKIPPED` or `PARTIAL` +- `entries`: one `FileResult` per file (`source`, `status`, `message`, `size`, + `volumes`, `warnings`, `archive`, `stored_size`, `restored`, `name`, `folder`). For `pack`, + `source` is the original file and `archive` its first volume; for `unpack` and + `verify`, `source` is the archive (its first volume) and `restored` lists the + files written. `message` describes what happened, for example + `3 volumes, 28.4 MiB -> 9.1 MiB (32%), verified`. +- `manifest`: path of the manifest that was written +- `engine`, `warnings`, `count(status)` +- `to_dict()` for JSON (`json.dumps(result.to_dict())`) + +One failing file does not stop the others; check `result.status` or +`result.count(bv.Status.FAILED)`. + +## Errors + +Problems that stop a run before any work (bad input, damaged manifest, missing +engine, bad signature) raise exceptions that all derive from +`bv.BatchVaultError`: + +`InputError`, `OptionsError`, `ManifestError`, `EngineNotFoundError`, +`EngineError`, `OutputExistsError`, `VolumeError`, `SignatureError`, +`ConfigError`. + +```python +try: + bv.unpack("archives", "restored", public_key="batchvault.pub") +except bv.SignatureError as exc: + print("Do not trust this archive:", exc) +``` + +## Progress + +Pass a callback to follow a run. It is called from worker threads, so keep it +thread-safe. + +```python +def show(event: bv.ProgressEvent): + print(event.kind, event.source.name, event.status) + +bv.pack(["a.bin", "b.bin"], "archives", progress=show) +``` + +Events are `"plan"` (once, with the total size and file count), then `"start"` +and `"finish"` for each file. + +## Logging + +The library only writes to the `batchvault` logger, and adds no handlers. Send +it where you like: + +```python +import logging +logging.getLogger("batchvault").addHandler(logging.StreamHandler()) +logging.getLogger("batchvault").setLevel(logging.INFO) +``` + +## Examples + +Only re-archive what changed, without ever deleting anything: + +```python +result = bv.pack(bv_files, "archives", update=True) +print(result.count(bv.Status.SKIPPED), "files were up to date") +``` + +Restore only if the manifest is signed by you: + +```python +bv.unpack("archives", "restored", public_key="batchvault.pub", password="...") +``` + +Free disk space safely, asking first: + +```python +bv.pack(files, "archives", delete_source=True, + confirm=lambda paths: input(f"Delete {len(paths)} files? ") == "y") +``` diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..6bf9b90 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,99 @@ +# Testing + +There are four layers. Each has a marker, a place in the repository and a +trigger. + +| Layer | Where | Marker | What it proves | +|---|---|---|---| +| Unit | `tests/test_*.py` | none | Modules behave correctly, fast, no external tools | +| Integration | `tests/integration/` | `integration` | The 7-Zip engine and the API work with a real 7-Zip (skipped without it) | +| End-to-end | `tests/e2e/` | `e2e` | The installed program works the way users use it | +| End-to-end, GitHub Action | `.github/workflows/e2e.yml` (job `action`) | n/a | `action.yml` works on real runners | + +Everything under `tests/e2e/` is marked `e2e` automatically. Two sub-markers +allow narrower runs: `package` (builds the wheel and installs it into a fresh +virtual environment) and `docker` (builds and runs the images). + +## What each end-to-end suite covers + +- **CLI** (`tests/e2e/test_cli_e2e.py`): runs the real `batchvault` command in a + separate process. Full journey (list, pack, inspect, verify, unpack, byte + comparison) on both engines, `--update`, standard-input pipelines, + `--json-events`, exit codes, damaged archives, restoring from one volume, + config files, parallel runs, log files, `--delete-source`, encryption and + signing. +- **Python package** (`tests/e2e/test_package_e2e.py`): builds the wheel, checks + its contents, installs it into a clean virtual environment, and checks the + metadata, the console script, `python -m batchvault`, the API (and that + importing it does not load `rich`), the message when the optional `crypto` + extra is missing, and encryption and signing once it is installed. +- **Docker** (`tests/e2e/test_docker_e2e.py`): builds both images from the + `Dockerfile` and runs containers with a mounted folder: default command, + non-root user, pack and unpack, exit codes, encryption, config file, and 7-Zip + only in the `-7zip` image. +- **GitHub Action** (workflow job `action`): uses the action from this checkout + (`uses: ./`) on Linux, Windows and macOS: pack, verify, unpack, `update`, + artifact upload and download, encryption with right and wrong passwords, + signing and a tampered manifest, failure reporting, and 7-Zip where the + runner has it. Assertions are shell and Python steps that compare outputs and + restored bytes (`tests/e2e/scripts/`). + +## Running them locally + +``` +uv run pytest -m "not integration and not e2e" -n auto # unit tests (default, fast) +uv run pytest -m integration # needs 7-Zip +uv run pytest -m "e2e and not docker" # CLI + package (needs network for pip) +uv run pytest -m "e2e and not docker and not package" # CLI only, quickest +uv run pytest -m package # package only +uv run pytest -m docker # needs Docker; builds the images +``` + +Do not use `-n` with the package tests; they share a wheel and environments. + +To test images you already built or pulled, skip the build: + +``` +BATCHVAULT_E2E_IMAGE=ghcr.io/coldsofttech/batchvault:latest \ +BATCHVAULT_E2E_IMAGE_7ZIP=ghcr.io/coldsofttech/batchvault:latest-7zip \ +uv run pytest -m docker +``` + +The Action job cannot run under pytest. To try it locally, use +[act](https://github.com/nektos/act) (`act -j action`); Linux only, and the +artifact steps may need extra setup. Otherwise push a branch and run the +workflow by hand (below). + +## Platforms in CI + +Unit and integration tests (`tests.yml`) and the CLI and package end-to-end +tests (`e2e.yml`) use the same matrix: Ubuntu with Python 3.11, 3.12, 3.13, 3.14 +and 3.15 (3.15 is allowed to fail while it may be a pre-release), Windows with +3.11 and macOS with 3.11. The Docker tests run on Ubuntu only, and the Action +tests run on Ubuntu, Windows and macOS with Python 3.12. + +## When they run in CI + +| Trigger | Unit | Integration | CLI + package | Docker | Action | +|---|---|---|---|---|---| +| Pull request | yes | yes | if `src/`, `tests/e2e/`, `action.yml`, `Dockerfile`, `pyproject.toml` changed | same | same | +| Push to `main` | yes | yes | yes | yes | yes | +| Weekly schedule (Mondays) | no | no | yes | yes | yes | +| Manual (**Actions** tab, **End-to-end**, **Run workflow**) | no | no | yes | yes | yes | +| Release tag (`v*.*.*`) | yes | yes | yes | yes | yes, and publishing waits for it | + +Unit and integration tests run from `tests.yml`; end-to-end suites run from +`e2e.yml`, which `release.yml` calls before it publishes anything. + +## Adding a test + +- Behaviour of one module: a unit test in `tests/`. +- Something a user would notice when running the program: add it to + `tests/e2e/test_cli_e2e.py`. +- Something about packaging or installation: `test_package_e2e.py`. +- Something that only exists inside the image: `test_docker_e2e.py`. +- A new Action input: add a step to the `action` job in `e2e.yml` that uses it + and asserts on the outputs or files. + +Keep end-to-end tests independent: each uses its own `tmp_path` and a private +log folder, and never touches your real archives or log folder. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1d65651 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "batchvault" +version = "0.1.0" +description = "Batch archive, compress, split and restore files with a native or 7-Zip engine." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.11.4" +dependencies = ["rich>=13", "pathspec>=0.12"] + +[project.optional-dependencies] +# Encryption (native engine) and manifest signing. +crypto = ["cryptography>=42"] + +[project.scripts] +batchvault = "batchvault.cli:main" + +[dependency-groups] +dev = [ + "pytest>=8", + "pytest-xdist>=3", + "ruff>=0.6", + "pre-commit>=3", + "cryptography>=42", +] + +[tool.uv] +package = true + +[tool.hatch.build.targets.wheel] +packages = ["src/batchvault"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] +addopts = "--strict-markers" +markers = [ + "integration: runs against a real 7-Zip (slower; excluded from pre-commit, run in CI)", + "e2e: end-to-end tests of the installed program (tests/e2e; run on demand and in CI)", + "package: e2e tests that build the wheel and install it into a fresh environment", + "docker: e2e tests that build and run the Docker images (need Docker)", +] + +[tool.ruff] +line-length = 88 +target-version = "py311" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] diff --git a/src/batchvault/__init__.py b/src/batchvault/__init__.py new file mode 100644 index 0000000..91433dc --- /dev/null +++ b/src/batchvault/__init__.py @@ -0,0 +1,61 @@ +import logging + +__version__ = "0.1.0" + +# Library code only logs; the command-line interface decides where logs go. +logging.getLogger("batchvault").addHandler(logging.NullHandler()) + +from batchvault.api import ( # noqa: E402 + inspect, + list_archives, + list_files, + pack, + unpack, + verify, +) +from batchvault.errors import ( # noqa: E402 + BatchVaultError, + ConfigError, + EngineError, + EngineNotFoundError, + InputError, + ManifestError, + OptionsError, + OutputExistsError, + SignatureError, + VolumeError, +) +from batchvault.models import ( # noqa: E402 + FileEntry, + FileResult, + InspectEntry, + ProgressEvent, + RunResult, + Status, +) + +__all__ = [ + "BatchVaultError", + "ConfigError", + "EngineError", + "EngineNotFoundError", + "FileEntry", + "FileResult", + "InputError", + "InspectEntry", + "ManifestError", + "OptionsError", + "OutputExistsError", + "ProgressEvent", + "RunResult", + "SignatureError", + "Status", + "VolumeError", + "__version__", + "inspect", + "list_archives", + "list_files", + "pack", + "unpack", + "verify", +] diff --git a/src/batchvault/__main__.py b/src/batchvault/__main__.py new file mode 100644 index 0000000..01eab0c --- /dev/null +++ b/src/batchvault/__main__.py @@ -0,0 +1,3 @@ +from batchvault.cli import main + +raise SystemExit(main()) diff --git a/src/batchvault/api.py b/src/batchvault/api.py new file mode 100644 index 0000000..75ddcda --- /dev/null +++ b/src/batchvault/api.py @@ -0,0 +1,877 @@ +"""Public Python API. The command-line interface is a thin wrapper over this.""" + +from __future__ import annotations + +import dataclasses +import logging +import os +import tempfile +import time +from collections.abc import Callable, Iterable +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from batchvault import signing +from batchvault.crypto import require_cryptography +from batchvault.engines.base import Engine +from batchvault.engines.registry import engine_for_volume, get_engine, option_warnings +from batchvault.errors import ( + BatchVaultError, + EngineError, + InputError, + ManifestError, + OptionsError, + OutputExistsError, + SignatureError, +) +from batchvault.filters import PathFilter +from batchvault.inputs import gather_sources +from batchvault.levels import AUTO, LevelChoice, choose_level, parse_level +from batchvault.locate import Target, resolve_targets +from batchvault.manifest import ( + MANIFEST_NAME, + Manifest, + ManifestEntry, + VolumeInfo, + hash_file, +) +from batchvault.models import ( + FileEntry, + FileResult, + InspectEntry, + ProgressEvent, + RunResult, + Status, +) +from batchvault.names import unsafe_on_windows +from batchvault.options import PackOptions, format_size, parse_size +from batchvault.volumes import STAGING_PREFIX, cleanup_staging, existing_volumes + +Progress = Callable[[ProgressEvent], None] +Confirm = Callable[[list[Path]], bool] + +log = logging.getLogger(__name__) + +RETRY_BASE_DELAY = 1.0 +RETRY_MAX_DELAY = 30.0 +ENCRYPTED_MESSAGE = "Archive is encrypted; a password is required." + + +def _log_result(action: str, outcome: FileResult) -> None: + level = logging.WARNING if outcome.status == Status.FAILED else logging.INFO + log.log( + level, "%s %s: %s %s", action, outcome.source, outcome.status, outcome.message + ) + + +def _split_jobs(jobs: int | None, files: int) -> tuple[int, int]: + """Split one thread budget into (files in parallel, threads per file). + + A budget of 0 means no worker threads: one file at a time, compressed in + the calling thread. + """ + total = jobs or 0 + if total < 0: + raise InputError("jobs cannot be negative") + if total == 0: + return 1, 0 + workers = max(1, min(files, total)) + return workers, max(1, total // workers) + + +def _notify(progress: Progress | None, event: ProgressEvent) -> None: + if progress: + progress(event) + + +def _relative(path: Path, base: Path) -> str: + try: + return Path(os.path.relpath(path, base)).as_posix() + except ValueError: # different drive on Windows + return str(path) + + +def _retry(call, retries: int, retry_on: tuple[type[Exception], ...]): + """Run ``call()``, retrying transient failures with exponential backoff.""" + attempt = 0 + while True: + try: + return call() + except FileNotFoundError: + raise + except retry_on as exc: + if attempt >= retries: + raise + delay = min(RETRY_BASE_DELAY * 2**attempt, RETRY_MAX_DELAY) + log.warning( + "Attempt %d failed (%s); retrying in %.1fs", attempt + 1, exc, delay + ) + time.sleep(delay) + attempt += 1 + + +def _format_name(backend: Engine, options: PackOptions) -> str: + return "7z" if backend.name == "7zip" else options.format + + +def _up_to_date( + entry: ManifestEntry, + stat: os.stat_result, + backend: Engine, + options: PackOptions, + manifest_dir: Path, +) -> bool: + """True if an archive already matches the file and the requested settings.""" + if not entry.mtime_ns or entry.mtime_ns != stat.st_mtime_ns: + return False + if entry.size != stat.st_size: + return False + if ( + entry.engine != backend.name + or entry.format != _format_name(backend, options) + or entry.level != options.level + or entry.volume_size != options.volume_size + or entry.encrypted != bool(options.password) + ): + return False + + for volume in entry.volumes: + path = manifest_dir / volume.file + if not path.is_file() or path.stat().st_size != volume.size: + return False + return True + + +def pack( + files: Iterable[str | Path] = (), + output: str | Path = "archives", + *, + list_file: str | Path | None = None, + root: str | Path | None = None, + engine: str = "auto", + seven_zip: str | Path | None = None, + format: str | None = None, + level: int | str = AUTO, + volume_size: str | int = "10m", + jobs: int = 0, + include: Iterable[str] = (), + exclude: Iterable[str] = (), + force: bool = False, + update: bool = False, + verify: bool = False, + manifest: str | Path | None = None, + allow_archives: bool = False, + strict: bool = False, + dry_run: bool = False, + password: str | None = None, + retries: int = 3, + sign_key: str | Path | None = None, + sign_key_password: str | None = None, + delete_source: bool = False, + confirm: Confirm | None = None, + progress: Progress | None = None, +) -> RunResult: + """Archive files into split volumes and record them in a manifest. + + Each file gets its own folder ``//`` holding its volumes. + Folders are kept: a file in a subfolder is stored under + ``///`` and ``unpack`` puts it back there. The + folder is relative to the one ``list files`` recorded (so the listed folder's + name is kept), else to the deepest folder all the files share; ``root`` + overrides this with a folder of your choice. + + Existing archives are skipped unless ``force`` is set; with ``update`` they + are skipped only if the file and settings are unchanged, and re-archived + otherwise. ``jobs`` is the total thread budget; ``progress`` is called from + worker threads. With ``strict``, settings the engine cannot honour raise an + error instead of a warning. With ``dry_run`` nothing is written. + + ``password`` encrypts the archives (the native engine needs + ``batchvault[crypto]``). ``retries`` retries a file that failed on an I/O or + engine error. ``sign_key`` signs the manifest with an Ed25519 private key. + ``delete_source`` deletes each original after a full restore-and-compare + proves the archive is good (and ``confirm``, if given, agrees). + """ + if retries < 0: + raise OptionsError("retries cannot be negative") + level = parse_level(level) + auto_level = level == AUTO # each file gets the level that suits its type + + root_path = Path(root).expanduser().resolve() if root is not None else None + sourced = gather_sources( + files, + list_file, + path_filter=PathFilter(include, exclude), + allow_archives=allow_archives, + root=root_path, + ) + sources = [path for path, _ in sourced] + folders = dict(sourced) + backend = get_engine(engine, seven_zip) + warnings = option_warnings(backend, format, level) + if strict and warnings: + raise OptionsError(" ".join(warnings)) + if password and backend.name == "native": + require_cryptography() + + private_key = None + if sign_key and not dry_run: + private_key = signing.load_private_key(Path(sign_key), sign_key_password) + + workers, per_file_jobs = _split_jobs(jobs, len(sources)) + options = PackOptions( + level=5 if auto_level else level, + volume_size=parse_size(volume_size), + jobs=per_file_jobs, + force=force, + format=format or "xz", + password=password, + ) + + out_root = Path(output).expanduser().resolve() + manifest_path = ( + Path(manifest).expanduser().resolve() if manifest else out_root / MANIFEST_NAME + ) + # Load first so a damaged manifest stops the run before any work is done. + known = Manifest.load(manifest_path) if manifest_path.exists() else Manifest() + if dry_run: + return _plan_pack( + backend, sources, folders, out_root, manifest_path.parent, options, + warnings, known if update else None, auto_level, + ) # fmt: skip + + cleanup_staging(out_root) + out_root.mkdir(parents=True, exist_ok=True) + + log.info("Packing %d file(s) with %s into %s", len(sources), backend.name, out_root) + total = sum(source.stat().st_size for source in sources) + _notify(progress, ProgressEvent("plan", out_root, total, count=len(sources))) + + def work(source: Path) -> tuple[FileResult, ManifestEntry | None]: + return _pack_one( + backend, source, folders[source], out_root, manifest_path.parent, + options, verify, progress, known if update else None, retries, + auto_level, + ) # fmt: skip + + with ThreadPoolExecutor(max_workers=workers) as executor: + outcomes = list(executor.map(work, sources)) + + result = RunResult( + warnings=warnings, manifest=manifest_path, engine=backend.name + ) + for file_result, entry in outcomes: + result.entries.append(file_result) + if entry: + known.entries[entry.path] = entry + + if any(entry for _, entry in outcomes): + known.save(manifest_path) + _sign_or_clear(manifest_path, private_key, result) + + if delete_source: + _delete_sources(backend, outcomes, password, confirm, result) + return result + + +def _sign_or_clear(manifest_path: Path, private_key, result: RunResult) -> None: + if private_key is not None: + signing.sign_manifest(manifest_path, private_key) + return + + stale = signing.signature_path(manifest_path) + if stale.exists(): + stale.unlink() + result.warnings.append( + "The manifest changed, so its old signature was removed. " + "Sign it again with a private key." + ) + + +def _file_options( + options: PackOptions, source: Path, auto_level: bool +) -> tuple[PackOptions, LevelChoice | None]: + """The options for one file; with ``--level auto`` the level suits its type.""" + if not auto_level or options.format == "store": # store ignores the level + return options, None + choice = choose_level(source) + return dataclasses.replace(options, level=choice.level), choice + + +def _folder_path(base: Path, folder: str) -> Path: + """``base`` joined with a stored forward-slash folder (``base`` if empty).""" + return base.joinpath(*folder.split("/")) if folder else base + + +def _plan_pack( + backend: Engine, + sources: list[Path], + folders: dict[Path, str], + out_root: Path, + manifest_dir: Path, + options: PackOptions, + warnings: list[str], + known: Manifest | None, + auto_level: bool = False, +) -> RunResult: + result = RunResult(warnings=warnings, engine=backend.name) + suffix = backend.capabilities.archive_suffix + + for source in sources: + stat = source.stat() + folder = folders[source] + existing = existing_volumes( + _folder_path(out_root, folder) / source.name, f"{source.name}{suffix}" + ) + key = f"{folder}/{source.name}" if folder else source.name + entry = known.entries.get(key) if known else None + file_options, choice = _file_options(options, source, auto_level) + + if entry and _up_to_date(entry, stat, backend, file_options, manifest_dir): + outcome = FileResult( + source, Status.SKIPPED, "Would skip: up to date.", stat.st_size + ) + elif existing and not (file_options.force or known is not None): + outcome = FileResult( + source, + Status.SKIPPED, + "Would skip: archive already exists.", + stat.st_size, + ) + else: + message = "Would update" if existing else "Would archive" + if choice: + message += f" at level {choice.level} ({choice.kind})" + outcome = FileResult(source, Status.SUCCESS, message + ".", stat.st_size) + outcome.folder = folder + outcome.level = file_options.level + outcome.name = source.name + result.entries.append(outcome) + return result + + +def _pack_summary( + size: int, stored: int, volumes: int, verified: bool, choice: LevelChoice | None +) -> str: + """e.g. ``3 volumes, 12.3 MiB -> 4.1 MiB (33%), level 6 (text), verified``.""" + sizes = f"{format_size(size)} -> {format_size(stored)}" + if size: + sizes += f" ({stored / size:.0%})" + parts = [f"{volumes} volume{'' if volumes == 1 else 's'}", sizes] + if choice: + parts.append(f"level {choice.level} ({choice.kind})") + if verified: + parts.append("verified") + return ", ".join(parts) + + +def _pack_one( + backend: Engine, + source: Path, + folder: str, + out_root: Path, + manifest_dir: Path, + options: PackOptions, + verify: bool, + progress: Progress | None, + known: Manifest | None, + retries: int, + auto_level: bool = False, +) -> tuple[FileResult, ManifestEntry | None]: + stat = source.stat() + size = stat.st_size + _notify(progress, ProgressEvent("start", source, size)) + options, choice = _file_options(options, source, auto_level) + + entry = None + try: + key = f"{folder}/{source.name}" if folder else source.name + previous = known.entries.get(key) if known is not None else None + if previous and _up_to_date(previous, stat, backend, options, manifest_dir): + outcome = FileResult(source, Status.SKIPPED, "Up to date.", size) + else: + # In update mode a changed file replaces its old archive. + file_options = options + if known is not None: + file_options = dataclasses.replace(options, force=True) + + destination = _folder_path(out_root, folder) / source.name + volumes = _retry( + lambda: backend.pack(source, destination, file_options), + retries, + (OSError, EngineError), + ) + entry = ManifestEntry( + name=source.name, + folder=folder, + size=size, + sha256=hash_file(source), + engine=backend.name, + format=_format_name(backend, options), + level=options.level, + volume_size=options.volume_size, + volumes=[ + VolumeInfo( + _relative(volume, manifest_dir), + volume.stat().st_size, + hash_file(volume), + ) + for volume in volumes + ], + encrypted=bool(options.password), + mtime_ns=stat.st_mtime_ns, + ) + if verify: + backend.test(volumes[0], password=options.password) + stored = sum(volume.size for volume in entry.volumes) + outcome = FileResult( + source, + Status.SUCCESS, + _pack_summary(size, stored, len(volumes), verify, choice), + size, + volumes, + archive=volumes[0], + stored_size=stored, + level=options.level, + ) + except OutputExistsError as exc: + outcome = FileResult(source, Status.SKIPPED, str(exc), size) + except (BatchVaultError, OSError) as exc: + outcome, entry = FileResult(source, Status.FAILED, str(exc), size), None + + outcome.folder = folder + outcome.name = source.name + _log_result("pack", outcome) + _notify(progress, ProgressEvent("finish", source, size, outcome.status)) + return outcome, entry + + +def _roundtrip_problem( + backend: Engine, outcome: FileResult, entry: ManifestEntry, password: str | None +) -> str | None: + """Restore the archive and compare it with the original; None if identical.""" + try: + parent = outcome.volumes[0].parent + with tempfile.TemporaryDirectory(dir=parent, prefix=STAGING_PREFIX) as tmp: + restored = backend.unpack( + outcome.volumes[0], + Path(tmp), + force=True, + password=password, + max_bytes=entry.size, + ) + if [path.name for path in restored] != [entry.name]: + return "the archive does not contain the expected file" + if hash_file(restored[0]) != entry.sha256: + return "the restored file differs from the original" + if hash_file(outcome.source) != entry.sha256: + return "the original changed while it was being archived" + except (BatchVaultError, OSError) as exc: + return str(exc) + return None + + +def _delete_sources( + backend: Engine, + outcomes: list[tuple[FileResult, ManifestEntry | None]], + password: str | None, + confirm: Confirm | None, + result: RunResult, +) -> None: + verified = [] + for outcome, entry in outcomes: + if outcome.status != Status.SUCCESS or entry is None: + continue + problem = _roundtrip_problem(backend, outcome, entry, password) + if problem: + outcome.warnings.append(f"Original kept: {problem}.") + log.warning("Original kept for %s: %s", outcome.source, problem) + else: + verified.append(outcome) + + if not verified: + return + if confirm is not None and not confirm([o.source for o in verified]): + result.warnings.append("Original files were kept (deletion not confirmed).") + return + + for outcome in verified: + try: + outcome.source.unlink() + except OSError as exc: + outcome.warnings.append(f"Could not delete the original: {exc}") + else: + outcome.message += "; original deleted" + log.info("Deleted original %s", outcome.source) + + +def _resolve(source: str | Path, public_key: str | Path | None) -> list[Target]: + """Find the archives for ``source``. + + With ``public_key``, each manifest's signature is checked on the very bytes + that are then parsed, so the file cannot be swapped in between. + """ + if not public_key: + return resolve_targets(source) + + key = Path(public_key) + + def load_signed(path: Path) -> Manifest: + try: + raw = path.read_bytes() + except OSError as exc: + raise ManifestError(f"Cannot read manifest {path}: {exc}") from exc + signing.verify_manifest_bytes(raw, path, key) + return Manifest.from_bytes(raw, path) + + targets = resolve_targets(source, load_signed) + if any(target.manifest is None for target in targets): + raise SignatureError("Signature check needs a manifest, but none was found.") + return targets + + +def _plan_event(where: Path, targets: list[Target]) -> ProgressEvent: + # Sizes are only known from a manifest; without one the UI counts files. + known = all(target.entry for target in targets) + total = sum(target.entry.size for target in targets if target.entry) + return ProgressEvent("plan", where, total if known else 0, count=len(targets)) + + +def unpack( + source: str | Path, + output: str | Path = "restored", + *, + force: bool = False, + update: bool = False, + jobs: int = 0, + seven_zip: str | Path | None = None, + password: str | None = None, + public_key: str | Path | None = None, + retries: int = 3, + max_output_size: str | int | None = None, + progress: Progress | None = None, +) -> RunResult: + """Restore files from a manifest, a folder, any volume, or a list of volumes. + + With a manifest, volumes are checked first and the restored file is checked + against its recorded SHA-256 before it is moved into place. Existing files + are skipped unless ``force`` is set; with ``update`` they are skipped only + when they already match the archive, and replaced otherwise. ``public_key`` + requires a valid manifest signature before anything is restored. + + ``max_output_size`` refuses an archive that would expand beyond that many + bytes (a decompression bomb) before anything is written. With a manifest an + archive may never expand beyond the size recorded for its file. + """ + if retries < 0: + raise OptionsError("retries cannot be negative") + max_bytes = parse_size(max_output_size) if max_output_size is not None else None + + targets = _resolve(source, public_key) + + out_root = Path(output).expanduser().resolve() + cleanup_staging(out_root) + workers, _ = _split_jobs(jobs, len(targets)) + + log.info("Unpacking %d archive(s) into %s", len(targets), out_root) + _notify(progress, _plan_event(out_root, targets)) + + def work(target: Target) -> FileResult: + size = target.entry.size if target.entry else 0 + _notify(progress, ProgressEvent("start", target.volume, size)) + outcome = _restore_one( + target, out_root, force, update, seven_zip, password, retries, max_bytes + ) + outcome.folder = target.entry.folder if target.entry else "" + outcome.name = target.name + _log_result("unpack", outcome) + _notify(progress, ProgressEvent("finish", target.volume, size, outcome.status)) + return outcome + + with ThreadPoolExecutor(max_workers=workers) as executor: + outcomes = list(executor.map(work, targets)) + + result = RunResult(entries=outcomes) + if any(target.entry is None for target in targets): + result.warnings.append( + "No manifest found for some archives; their checksums were not verified." + ) + return result + + +def _destination_folder(out_root: Path, folder: str) -> Path: + """Where files stored under ``folder`` are restored, kept inside ``out_root``.""" + destination = _folder_path(out_root, folder) + if not folder: + return destination + + if os.name == "nt" and any(unsafe_on_windows(part) for part in folder.split("/")): + raise InputError(f"The stored folder {folder!r} has names Windows cannot use.") + + root = out_root.resolve() + if root not in destination.resolve().parents: + raise InputError(f"The stored folder {folder!r} points outside the output.") + return destination + + +def _restore_one( + target: Target, + out_root: Path, + force: bool, + update: bool, + seven_zip: str | Path | None, + password: str | None, + retries: int, + max_bytes: int | None, +) -> FileResult: + volume = target.volume + entry = target.entry + try: + limit = max_bytes + if entry: # the manifest says how big the file is; never accept more + limit = entry.size if limit is None else min(limit, entry.size) + + folder = entry.folder if entry else "" + destination_dir = _destination_folder(out_root, folder) + destination = destination_dir / target.name + if destination.exists() and not force: + if not update: + return FileResult( + volume, + Status.SKIPPED, + f"{target.name} already exists. Use force to overwrite.", + ) + if entry is None: + return FileResult( + volume, + Status.SKIPPED, + f"{target.name} exists and there is no manifest to compare with.", + ) + if _matches(destination, entry): + return FileResult(volume, Status.SKIPPED, "Already up to date.") + + if entry: + if entry.encrypted and not password: + return FileResult(volume, Status.FAILED, ENCRYPTED_MESSAGE) + problems = entry.check_volumes(target.base_dir) + if problems: + return FileResult(volume, Status.FAILED, "; ".join(problems)) + + backend = engine_for_volume(volume, seven_zip) + out_root.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(dir=out_root, prefix=STAGING_PREFIX) as tmp: + restored = _retry( + lambda: backend.unpack( + volume, Path(tmp), force=True, password=password, max_bytes=limit + ), + retries, + (OSError,), + ) + + if entry: + names = [path.name for path in restored] + if names != [entry.name]: + return FileResult( + volume, + Status.FAILED, + f"Archive contains {names}, manifest expects {entry.name!r}.", + ) + if hash_file(restored[0]) != entry.sha256: + return FileResult( + volume, + Status.FAILED, + f"Restored {entry.name} does not match its checksum.", + ) + + destination_dir.mkdir(parents=True, exist_ok=True) + final = [] + for path in restored: + os.replace(path, destination_dir / path.name) + final.append(destination_dir / path.name) + + size = sum(path.stat().st_size for path in final) + checked = "checksum verified" if entry else "checksum not verified" + message = f"{format_size(size)} restored, {checked}" + return FileResult(volume, Status.SUCCESS, message, size, restored=final) + except (BatchVaultError, OSError) as exc: + return FileResult(volume, Status.FAILED, str(exc)) + + +def _matches(path: Path, entry: ManifestEntry) -> bool: + return path.is_file() and path.stat().st_size == entry.size and ( + hash_file(path) == entry.sha256 + ) + + +def verify( + source: str | Path, + *, + against: str | Path | None = None, + jobs: int = 0, + seven_zip: str | Path | None = None, + password: str | None = None, + public_key: str | Path | None = None, + progress: Progress | None = None, +) -> RunResult: + """Check archives are complete and readable, without restoring anything. + + With a manifest, volume sizes and checksums are checked. ``against`` is a + folder of the original files, compared by SHA-256 (manifest required); each + original is looked for by name in that folder and then in its subfolders. + ``public_key`` also requires a valid manifest signature. + """ + targets = _resolve(source, public_key) + + workers, _ = _split_jobs(jobs, len(targets)) + originals = Path(against).expanduser() if against else None + + _notify(progress, _plan_event(Path(source), targets)) + + def work(target: Target) -> FileResult: + size = target.entry.size if target.entry else 0 + _notify(progress, ProgressEvent("start", target.volume, size)) + outcome = _verify_one(target, originals, seven_zip, password) + outcome.folder = target.entry.folder if target.entry else "" + outcome.name = target.name + _log_result("verify", outcome) + _notify(progress, ProgressEvent("finish", target.volume, size, outcome.status)) + return outcome + + with ThreadPoolExecutor(max_workers=workers) as executor: + outcomes = list(executor.map(work, targets)) + + result = RunResult(entries=outcomes) + if any(target.entry is None for target in targets): + result.warnings.append( + "No manifest found for some archives; only their internal integrity " + "checks were run." + ) + return result + + +def _find_originals(root: Path, folder: str, name: str) -> list[Path]: + """The original called ``name``: at its stored folder under ``root``, else + directly in ``root``, else anywhere below it.""" + for candidate in (_folder_path(root, folder) / name, root / name): + if candidate.is_file(): + return [candidate] + return [path for path in root.rglob(name) if path.is_file()] + + +def _verify_one( + target: Target, + originals: Path | None, + seven_zip: str | Path | None, + password: str | None, +) -> FileResult: + volume = target.volume + entry = target.entry + try: + if entry: + if entry.encrypted and not password: + return FileResult(volume, Status.FAILED, ENCRYPTED_MESSAGE) + problems = entry.check_volumes(target.base_dir) + if problems: + return FileResult(volume, Status.FAILED, "; ".join(problems)) + + engine_for_volume(volume, seven_zip).test(volume, password=password) + + if originals and entry: + candidates = _find_originals(originals, entry.folder, entry.name) + if not candidates: + message = f"Original not found under {originals}: {entry.name}" + return FileResult(volume, Status.FAILED, message) + if not any(hash_file(path) == entry.sha256 for path in candidates): + message = f"{entry.name} differs from the archive." + return FileResult(volume, Status.FAILED, message) + + if entry: + message = "volumes, checksums and integrity test OK" + else: + message = "integrity test OK (no manifest, so checksums were not checked)" + if originals and entry: + message += "; matches the original" + return FileResult(volume, Status.SUCCESS, message, entry.size if entry else 0) + except (BatchVaultError, OSError) as exc: + return FileResult(volume, Status.FAILED, str(exc)) + + +def inspect(source: str | Path) -> list[InspectEntry]: + """Describe archives from their manifest (or file names when there is none).""" + found = [] + for target in resolve_targets(source): + entry = target.entry + if entry is None: + volumes = existing_volumes( + target.volume.parent, target.volume.name.rsplit(".", 1)[0] + ) + found.append( + InspectEntry( + name=target.name, + path=target.volume, + engine="7zip" if ".7z." in target.volume.name.lower() else "native", + stored_size=sum(v.stat().st_size for v in volumes), + volumes=len(volumes), + ) + ) + continue + + paths = [target.base_dir / volume.file for volume in entry.volumes] + found.append( + InspectEntry( + name=entry.name, + folder=entry.folder, + path=paths[0], + engine=entry.engine, + format=entry.format, + level=entry.level, + encrypted=entry.encrypted, + size=entry.size, + stored_size=sum(volume.size for volume in entry.volumes), + volumes=len(paths), + complete=all(path.is_file() for path in paths), + ) + ) + return found + + +def list_files( + folder: str | Path, + *, + include: Iterable[str] = (), + exclude: Iterable[str] = (), +) -> list[FileEntry]: + """Files under ``folder`` (recursively), filtered by include/exclude patterns. + + Patterns match against the path relative to ``folder``. + """ + root = Path(folder).expanduser() + if not root.is_dir(): + raise InputError(f"Folder not found: {root}") + + path_filter = PathFilter(include, exclude) + # Record the folder's parent so that packing the listing keeps the folder's + # own name (Resources/...), not just what is inside it. + recorded = root.resolve().parent + return [ + FileEntry(path.resolve(), path.stat().st_size, recorded) + for path in sorted(root.rglob("*")) + if path.is_file() and path_filter.matches(path.relative_to(root).as_posix()) + ] + + +def list_archives(folder: str | Path) -> list[FileEntry]: + """Archives under ``folder``: from its manifest if present, else by scanning.""" + root = Path(folder).expanduser() + if not root.is_dir(): + raise InputError(f"Folder not found: {root}") + + try: + targets = resolve_targets(root) + except InputError: + return [] + return [ + FileEntry(target.volume.resolve(), target.entry.size if target.entry else None) + for target in targets + ] diff --git a/src/batchvault/blocks.py b/src/batchvault/blocks.py new file mode 100644 index 0000000..7ff7145 --- /dev/null +++ b/src/batchvault/blocks.py @@ -0,0 +1,176 @@ +"""Block-parallel compression for the native engine. + +The tar stream is cut into independent blocks. Each block is compressed as its +own xz, gzip or bzip2 member on a thread pool and the members are written in +order. Concatenated members are a valid single stream for all three formats, so +the joined volumes can be read by the standard tools too. +""" + +from __future__ import annotations + +import bz2 +import gzip +from collections import deque +from concurrent.futures import ThreadPoolExecutor + +from batchvault import crypto +from batchvault.errors import EngineError, EngineNotFoundError +from batchvault.options import FORMATS, MIB + +try: + import lzma +except ImportError: # Python built without liblzma + lzma = None + +BLOCK_SIZE = 16 * MIB + +_XZ_MAGIC = b"\xfd7zXZ\x00" +_GZIP_MAGIC = b"\x1f\x8b" +_BZIP2_MAGIC = b"BZh" +_TAR_MAGIC_OFFSET = 257 + + +def check_format(fmt: str) -> None: + if fmt not in FORMATS: + raise EngineError(f"Unknown format: {fmt}") + if fmt == "xz" and lzma is None: + raise EngineNotFoundError( + "This Python was built without lzma support, so the xz format " + "is unavailable. Use --format gzip or bzip2." + ) + + +def compress_block(fmt: str, level: int, data: bytes) -> bytes: + """Compress one block as a complete, independent member.""" + if fmt == "xz": + return lzma.compress(data, format=lzma.FORMAT_XZ, preset=level) + if fmt == "gzip": + return gzip.compress(data, compresslevel=level, mtime=0) + if fmt == "bzip2": + # bzip2 has no level 0; the lowest level is the closest match. + return bz2.compress(data, compresslevel=max(level, 1)) + return data + + +def open_decompressed(fmt: str, fileobj): + """Wrap a file object so reading it yields the uncompressed tar stream.""" + check_format(fmt) + if fmt == "xz": + return lzma.LZMAFile(fileobj, "rb") + if fmt == "gzip": + return gzip.GzipFile(fileobj=fileobj, mode="rb") + if fmt == "bzip2": + return bz2.BZ2File(fileobj, "rb") + return fileobj + + +def sniff_archive(path) -> tuple[str, bool]: + """Identify ``(format, encrypted)`` from the first bytes of a volume.""" + with open(path, "rb") as handle: + head = handle.read(max(_TAR_MAGIC_OFFSET + 5, crypto.HEADER_SIZE)) + + if crypto.is_encrypted_header(head): + return crypto.parse_header(head)[0], True + if head.startswith(_XZ_MAGIC): + return "xz", False + if head.startswith(_GZIP_MAGIC): + return "gzip", False + if head.startswith(_BZIP2_MAGIC): + return "bzip2", False + if head[_TAR_MAGIC_OFFSET : _TAR_MAGIC_OFFSET + 5] == b"ustar": + return "store", False + raise EngineError(f"Unrecognised archive format: {path}") + + +def sniff_format(path) -> str: + """Identify the compression format from the first bytes of a volume.""" + return sniff_archive(path)[0] + + +def _process(fmt: str, level: int, block: bytes, cipher, index: int) -> bytes: + data = compress_block(fmt, level, block) + return cipher.encrypt(index, data) if cipher else data + + +class BlockWriter: + """Write-only file object that compresses fixed-size blocks in parallel. + + At most ``2 * jobs`` blocks are in flight, which bounds memory use. + """ + + def __init__( + self, + sink, + fmt: str, + level: int, + block_size: int, + jobs: int, + cipher=None, + header: bytes = b"", + ): + self._sink = sink + self._fmt = fmt + self._level = level + self._block_size = block_size + self._cipher = cipher + self._max_pending = jobs * 2 + # With no jobs, blocks are compressed in the calling thread. + self._executor = ThreadPoolExecutor(max_workers=jobs) if jobs else None + self._pending = deque() + self._buffer = bytearray() + self._count = 0 + if cipher: + sink.write(header) + + def write(self, data) -> int: + self._buffer += data + while len(self._buffer) >= self._block_size: + block = bytes(self._buffer[: self._block_size]) + del self._buffer[: self._block_size] + self._submit(block) + return len(data) + + def _submit(self, block: bytes) -> None: + if self._executor is None: + self._sink.write( + _process(self._fmt, self._level, block, self._cipher, self._count) + ) + self._count += 1 + return + self._pending.append( + self._executor.submit( + _process, self._fmt, self._level, block, self._cipher, self._count + ) + ) + self._count += 1 + while len(self._pending) > self._max_pending: + self._write_oldest() + + def _write_oldest(self) -> None: + self._sink.write(self._pending.popleft().result()) + + def close(self) -> None: + try: + if self._buffer: + self._submit(bytes(self._buffer)) + self._buffer.clear() + while self._pending: + self._write_oldest() + if self._cipher: + # The end marker lets a reader detect a truncated archive. + self._sink.write(self._cipher.encrypt(self._count, b"", last=True)) + finally: + self._shutdown() + + def _shutdown(self) -> None: + if self._executor: + self._executor.shutdown(wait=True, cancel_futures=True) + + def __enter__(self): + return self + + def __exit__(self, exc_type, *exc_info): + if exc_type is None: + self.close() + else: + self._shutdown() diff --git a/src/batchvault/cli.py b/src/batchvault/cli.py new file mode 100644 index 0000000..772107c --- /dev/null +++ b/src/batchvault/cli.py @@ -0,0 +1,806 @@ +"""Command-line interface: a thin wrapper over :mod:`batchvault.api`. + +Exit codes: 0 success, 1 some files failed, were skipped or the checks failed, +2 usage or configuration error, 3 unexpected error, 130 interrupted. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import sys +import threading +from contextlib import contextmanager +from pathlib import Path + +from rich.console import Console +from rich.markup import escape + +from batchvault import __version__, api, notify, signing +from batchvault.config import ( + defaults_for, + find_config, + load_config, + reject_restricted, +) +from batchvault.doctor import passed, run_checks +from batchvault.engines.registry import ENGINE_CHOICES +from batchvault.errors import BatchVaultError, ConfigError, InputError +from batchvault.inputs import ROOT_HEADER, looks_like_archive +from batchvault.levels import AUTO, level_argument +from batchvault.logs import LOG_LEVELS, close_logging, setup_logging +from batchvault.manifest import Manifest +from batchvault.models import RunResult, Status +from batchvault.options import FORMATS +from batchvault.security import redact, redact_arguments + +log = logging.getLogger(__name__) + +ENV_PASSWORD = "BATCHVAULT_PASSWORD" +ENV_KEY_PASSWORD = "BATCHVAULT_KEY_PASSWORD" +ENV_NOTIFY_URL = "BATCHVAULT_NOTIFY_URL" +ENV_CONFIG = "BATCHVAULT_CONFIG" + + +def _parents() -> dict[str, argparse.ArgumentParser]: + parents = {name: argparse.ArgumentParser(add_help=False) for name in ( + "common", "jobs", "seven", "patterns", "run", "retry", "public", + )} # fmt: skip + + common = parents["common"] + common.add_argument( + "--json", action="store_true", help="Print the result as JSON on stdout." + ) + common.add_argument( + "-q", "--quiet", action="store_true", help="No progress or summary output." + ) + common.add_argument("--log-file", help="Write the log here instead of the default.") + common.add_argument( + "--log-level", choices=LOG_LEVELS, default="INFO", help="Log level." + ) + common.add_argument( + "--no-log", action="store_true", help="Do not write a log file." + ) + common.add_argument( + "--config", metavar="FILE", help="Config file (default: batchvault.toml)." + ) + + parents["jobs"].add_argument( + "-j", + "--jobs", + type=int, + default=0, + metavar="N", + help="Worker threads to use; 0 means none, one file at a time (default: 0).", + ) + + parents["seven"].add_argument( + "--7z", dest="seven_zip", metavar="PATH", help="Path to the 7-Zip executable." + ) + + patterns = parents["patterns"] + patterns.add_argument( + "--include", + action="append", + default=[], + metavar="GLOB", + help="Only keep matching paths (repeatable; @FILE reads patterns from a file).", + ) + patterns.add_argument( + "--exclude", + action="append", + default=[], + metavar="GLOB", + help="Drop matching paths (repeatable; wins over --include).", + ) + + run = parents["run"] + run.add_argument( + "--json-events", + action="store_true", + help="Print one JSON event per line on stdout while running.", + ) + run.add_argument( + "--notify", + metavar="URL", + help=f"POST a JSON summary to this webhook (or set {ENV_NOTIFY_URL}).", + ) + run.add_argument( + "--notify-on", + choices=notify.NOTIFY_ON, + default="failure", + help="When to notify (default: failure).", + ) + run.add_argument( + "--password-file", + metavar="FILE", + help=f"Read the archive password from this file (or set {ENV_PASSWORD}).", + ) + + parents["retry"].add_argument( + "--retries", + type=int, + default=3, + metavar="N", + help="Retry a file this many times after an I/O or engine error (default: 3).", + ) + + parents["public"].add_argument( + "--public-key", + metavar="FILE", + help="Require a valid manifest signature made with the matching private key.", + ) + return parents + + +def _configure(subparser, name: str, config: dict | None) -> None: + """Apply defaults from the config file to one command.""" + if not config: + return + actions = {action.dest: action for action in subparser._actions} + defaults = defaults_for(config, name, actions) + if defaults: + subparser.set_defaults(**defaults) + + +def build_parser(config: dict | None = None) -> argparse.ArgumentParser: + p = _parents() + common, jobs, seven, patterns, run, retry, public = ( + p["common"], p["jobs"], p["seven"], p["patterns"], p["run"], p["retry"], + p["public"], + ) # fmt: skip + + parser = argparse.ArgumentParser( + prog="batchvault", + description="Batch archive, compress, split and restore files.", + ) + parser.add_argument( + "--version", action="version", version=f"%(prog)s {__version__}" + ) + commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND") + + pack = commands.add_parser( + "pack", + parents=[common, jobs, seven, patterns, run, retry], + help="Compress files into split volumes.", + description="Compress files into split volumes and write a manifest.", + epilog="Example: batchvault pack --list archive_list.txt -o archives", + ) + pack.add_argument("files", nargs="*", metavar="FILE", help="Files to archive.") + pack.add_argument( + "--list", + metavar="FILE", + help="Text or JSON list of files to archive ('-' reads standard input).", + ) + pack.add_argument("-o", "--output", default="archives", help="Output folder.") + pack.add_argument("--engine", choices=ENGINE_CHOICES, default="auto") + pack.add_argument( + "--format", + choices=FORMATS, + help="Native engine compression (default: xz). 7-Zip always writes 7z.", + ) + pack.add_argument( + "--level", + type=level_argument, + choices=[*range(10), AUTO], + default=AUTO, + metavar="0-9|auto", + help="Compression level, or auto to pick one per file type (default: auto).", + ) + pack.add_argument( + "--volume-size", + default="10m", + metavar="SIZE", + help="Volume size, e.g. 500k, 10m, 2g (default: 10m).", + ) + pack.add_argument("--manifest", metavar="PATH", help="Manifest file location.") + pack.add_argument( + "--root", + metavar="DIR", + help="Advanced: store paths relative to DIR. By default folders are kept " + "automatically (relative to the folder you listed, or the folder your " + "files share).", + ) + pack.add_argument("--force", action="store_true", help="Replace existing archives.") + pack.add_argument( + "--update", + action="store_true", + help="Skip unchanged files and re-archive changed ones.", + ) + pack.add_argument( + "--verify", action="store_true", help="Test each archive after creating it." + ) + pack.add_argument( + "--encrypt", + action="store_true", + help=f"Encrypt with the password from {ENV_PASSWORD} or --password-file.", + ) + pack.add_argument( + "--sign-key", + metavar="FILE", + help=f"Sign the manifest with this private key (password: {ENV_KEY_PASSWORD}).", + ) + pack.add_argument( + "--delete-source", + action="store_true", + help="Delete each original after proving the archive restores identically.", + ) + pack.add_argument( + "--yes", action="store_true", help="Do not ask before --delete-source deletes." + ) + pack.add_argument( + "--allow-archives", + action="store_true", + help="Allow archiving BatchVault/7-Zip volumes and manifests.", + ) + pack.add_argument( + "--strict", + action="store_true", + help="Treat settings the engine cannot honour as errors.", + ) + pack.add_argument( + "--dry-run", + action="store_true", + help="Show what would be archived without writing anything.", + ) + pack.set_defaults(handler=cmd_pack) + _configure(pack, "pack", config) + + unpack = commands.add_parser( + "unpack", + parents=[common, jobs, seven, run, retry, public], + help="Restore files from archives.", + description="Restore files from a manifest, folder, volume or list file.", + epilog="Example: batchvault unpack archives -o restored", + ) + unpack.add_argument( + "source", metavar="SOURCE", help="Manifest, folder, volume or list file." + ) + unpack.add_argument("-o", "--output", default="restored", help="Output folder.") + unpack.add_argument( + "--force", action="store_true", help="Overwrite existing files." + ) + unpack.add_argument( + "--update", + action="store_true", + help="Skip files that already match; replace the rest.", + ) + unpack.add_argument( + "--max-output-size", + metavar="SIZE", + help="Refuse an archive that would expand beyond this size, per file " + "(e.g. 50g). Use it when restoring archives you do not trust.", + ) + unpack.set_defaults(handler=cmd_unpack) + _configure(unpack, "unpack", config) + + verify = commands.add_parser( + "verify", + parents=[common, jobs, seven, run, public], + help="Check that archives are complete and intact.", + description="Check archives without restoring them.", + ) + verify.add_argument( + "source", metavar="SOURCE", help="Manifest, folder, volume or list file." + ) + verify.add_argument( + "--against", + metavar="FOLDER", + help="Also compare with the original files in this folder (and below).", + ) + verify.set_defaults(handler=cmd_verify) + _configure(verify, "verify", config) + + inspect = commands.add_parser( + "inspect", + parents=[common], + help="Show what is in an archive set.", + description="Show sizes, ratios, engine, format and completeness.", + ) + inspect.add_argument( + "source", metavar="SOURCE", help="Manifest, folder, volume or list file." + ) + inspect.set_defaults(handler=cmd_inspect) + _configure(inspect, "inspect", config) + + listing = commands.add_parser("list", help="List files or archives.") + kinds = listing.add_subparsers(dest="kind", required=True, metavar="KIND") + + files = kinds.add_parser( + "files", + parents=[common, patterns], + help="List the files in a folder.", + description="List files (JSON on stdout unless -o is given).", + ) + files.add_argument("folder") + files.add_argument("-o", "--output", metavar="FILE", help="Write .json or .txt.") + files.set_defaults(handler=cmd_list_files) + _configure(files, "list-files", config) + + archives = kinds.add_parser( + "archives", + parents=[common], + help="List the archives in a folder.", + description="List archives (JSON on stdout unless -o is given).", + ) + archives.add_argument("folder") + archives.add_argument("-o", "--output", metavar="FILE", help="Write .json or .txt.") + archives.set_defaults(handler=cmd_list_archives) + _configure(archives, "list-archives", config) + + keygen = commands.add_parser( + "keygen", + parents=[common], + help="Create a key pair for signing manifests.", + description=( + "Create an Ed25519 key pair (needs batchvault[crypto]). The private " + f"key is encrypted if {ENV_KEY_PASSWORD} is set." + ), + ) + keygen.add_argument("--private", default="batchvault.key", metavar="FILE") + keygen.add_argument("--public", default="batchvault.pub", metavar="FILE") + keygen.add_argument("--force", action="store_true", help="Replace existing keys.") + keygen.set_defaults(handler=cmd_keygen) + _configure(keygen, "keygen", config) + + sign = commands.add_parser( + "sign", + parents=[common], + help="Sign a manifest.", + description="Write a detached signature next to a manifest.", + ) + sign.add_argument("manifest", metavar="MANIFEST") + sign.add_argument("--key", required=True, metavar="FILE", help="Private key.") + sign.set_defaults(handler=cmd_sign) + _configure(sign, "sign", config) + + doctor = commands.add_parser( + "doctor", + parents=[common, seven], + help="Check this machine's setup.", + description="Check Python modules, 7-Zip and the log folder.", + ) + doctor.set_defaults(handler=cmd_doctor) + _configure(doctor, "doctor", config) + + return parser + + +def _consoles() -> tuple[Console, Console]: + return Console(), Console(stderr=True) + + +class EventStream: + """Progress callback that prints one JSON object per line on stdout.""" + + def __init__(self): + self._lock = threading.Lock() + + def __call__(self, event) -> None: + self._write(event.to_dict()) + + def result(self, result: RunResult) -> None: + self._write({"event": "result", **result.to_dict()}) + + def _write(self, data: dict) -> None: + with self._lock: + print(json.dumps(data, ensure_ascii=False), flush=True) + + +@contextmanager +def _progress(args, label: str, err: Console): + from batchvault import ui + + if args.json_events: + yield EventStream() + else: + with ui.RunProgress(err, label, enabled=not args.quiet) as progress: + yield progress + + +def _check_output_flags(args) -> None: + if args.json and args.json_events: + raise InputError("Use either --json or --json-events, not both.") + + +def _read_password(args) -> str | None: + path = getattr(args, "password_file", None) + if path: + try: + lines = Path(path).expanduser().read_text(encoding="utf-8").splitlines() + except OSError as exc: + raise InputError(f"Cannot read password file {path}: {exc}") from exc + if not lines or not lines[0]: + raise InputError(f"Password file {path} is empty.") + return lines[0] + return os.environ.get(ENV_PASSWORD) or None + + +def _notify_url(args) -> str | None: + url = args.notify or os.environ.get(ENV_NOTIFY_URL) + if not url: + return None + + notify.validate_url(url) + if notify.is_insecure(url): + _, err = _consoles() + err.print( + "[yellow]Warning: the notification URL uses http://, so the summary " + "(file names and paths) is sent unencrypted. Prefer https://.[/yellow]" + ) + return url + + +def _finish( + args, command: str, result: RunResult, title: str | None, progress, url +) -> int: + from batchvault import ui + + out, err = _consoles() + if args.json_events: + progress.result(result) + elif args.json: + print(json.dumps(result.to_dict(), indent=2, ensure_ascii=False)) + elif not args.quiet: + ui.print_summary(out, result, title, command) + + if url and notify.should_notify(args.notify_on, result): + problem = notify.send(url, notify.build_payload(command, result)) + if problem: + log.warning("Notification failed: %s", redact(problem)) + err.print( + f"[yellow]Warning: notification failed ({escape(problem)}).[/yellow]" + ) + + return 0 if result.status == Status.SUCCESS else 1 + + +def _confirm_deletion(args, progress): + def confirm(paths: list[Path]) -> bool: + if args.yes: + return True + quiet = args.json or args.json_events or args.quiet + if quiet or not sys.stdin.isatty(): + return False # never delete without an explicit yes + + if hasattr(progress, "pause"): + progress.pause() + print(f"{len(paths)} original file(s) were archived and verified:") + for path in paths[:10]: + print(f" {path}") + if len(paths) > 10: + print(f" ... and {len(paths) - 10} more") + answer = input("Delete them? [y/N] ") + return answer.strip().lower() in ("y", "yes") + + return confirm + + +def _describe_level(level) -> str: + return "auto (chosen for each file's type)" if level == AUTO else str(level) + + +def _describe_sources(args) -> str: + parts = [] + if args.files: + parts.append(f"{len(args.files)} given") + if args.list: + parts.append("list: " + ("standard input" if args.list == "-" else args.list)) + return ", ".join(parts) or "-" + + +def cmd_pack(args) -> int: + from batchvault import ui + + _check_output_flags(args) + if not args.files and not args.list: + raise InputError("Give at least one FILE or use --list.") + + password = None + if args.encrypt: + password = _read_password(args) + if not password: + raise InputError( + f"--encrypt needs a password: set {ENV_PASSWORD} " + "or use --password-file." + ) + url = _notify_url(args) + + out, err = _consoles() + if not (args.quiet or args.json or args.json_events): + ui.print_run_info( + out, + "BatchVault pack", + [ + ("Engine", args.engine), + ("Format", args.format or "xz (native engine)"), + ("Level", _describe_level(args.level)), + ("Volume size", args.volume_size), + ("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"), + ("Source files", _describe_sources(args)), + ( + "Folders", + f"kept, relative to {Path(args.root).resolve()}" + if args.root + else "kept", + ), + ("Archives folder", str(Path(args.output).resolve())), + ], + ) + + with _progress(args, "Archiving", err) as progress: + result = api.pack( + args.files, + args.output, + list_file=args.list, + root=args.root, + engine=args.engine, + seven_zip=args.seven_zip, + format=args.format, + level=args.level, + volume_size=args.volume_size, + jobs=args.jobs, + include=args.include, + exclude=args.exclude, + force=args.force, + update=args.update, + verify=args.verify, + manifest=args.manifest, + allow_archives=args.allow_archives, + strict=args.strict, + dry_run=args.dry_run, + password=password, + retries=args.retries, + sign_key=args.sign_key, + sign_key_password=os.environ.get(ENV_KEY_PASSWORD) or None, + delete_source=args.delete_source and not args.dry_run, + confirm=_confirm_deletion(args, progress), + progress=progress, + ) + + title = None # the default title names the source files and archives + if args.dry_run: + title = "Pack plan (dry run): source files -> archives" + if not (args.quiet or args.json or args.json_events): + out.print("[bold]Dry run:[/bold] nothing was written.") + return _finish(args, "pack", result, title, progress, url) + + +def cmd_unpack(args) -> int: + from batchvault import ui + + _check_output_flags(args) + password = _read_password(args) + url = _notify_url(args) + + out, err = _consoles() + if not (args.quiet or args.json or args.json_events): + ui.print_run_info( + out, + "BatchVault unpack", + [ + ("Archives", args.source), + ("Restore files to", str(Path(args.output).resolve())), + ("Existing files", "overwrite" if args.force else "skip"), + ("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"), + ], + ) + with _progress(args, "Restoring", err) as progress: + result = api.unpack( + args.source, + args.output, + force=args.force, + update=args.update, + jobs=args.jobs, + seven_zip=args.seven_zip, + password=password, + public_key=args.public_key, + retries=args.retries, + max_output_size=args.max_output_size, + progress=progress, + ) + return _finish(args, "unpack", result, None, progress, url) + + +def cmd_verify(args) -> int: + from batchvault import ui + + _check_output_flags(args) + password = _read_password(args) + url = _notify_url(args) + + out, err = _consoles() + if not (args.quiet or args.json or args.json_events): + ui.print_run_info( + out, + "BatchVault verify", + [ + ("Archives", args.source), + ("Original files", args.against or "not compared"), + ("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"), + ], + ) + with _progress(args, "Verifying", err) as progress: + result = api.verify( + args.source, + against=args.against, + jobs=args.jobs, + seven_zip=args.seven_zip, + password=password, + public_key=args.public_key, + progress=progress, + ) + return _finish(args, "verify", result, None, progress, url) + + +def cmd_inspect(args) -> int: + from batchvault import ui + + entries = api.inspect(args.source) + if args.json: + print(json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False)) + elif not args.quiet: + ui.print_inspect(Console(), entries, args.source) + return 0 + + +def cmd_keygen(args) -> int: + private, public = Path(args.private).expanduser(), Path(args.public).expanduser() + signing.generate_keypair( + private, + public, + password=os.environ.get(ENV_KEY_PASSWORD) or None, + overwrite=args.force, + ) + if args.json: + print(json.dumps({"private": str(private), "public": str(public)})) + elif not args.quiet: + out, _ = _consoles() + out.print(f"Private key: {escape(str(private))} (keep it secret)") + out.print(f"Public key: {escape(str(public))} (share it)") + return 0 + + +def cmd_sign(args) -> int: + manifest_path = Path(args.manifest).expanduser() + Manifest.load(manifest_path) # refuse to sign something that is not a manifest + key = signing.load_private_key( + Path(args.key).expanduser(), os.environ.get(ENV_KEY_PASSWORD) or None + ) + signature = signing.sign_manifest(manifest_path, key) + if args.json: + print(json.dumps({"signature": str(signature)})) + elif not args.quiet: + _consoles()[0].print(f"Signature written to {escape(str(signature))}") + return 0 + + +def _write_listing(args, entries, noun: str) -> None: + _, err = _consoles() + if not args.output: + print(json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False)) + return + + path = Path(args.output).expanduser() + if path.suffix.lower() == ".json": + text = json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False) + text += "\n" + else: + text = "".join(f"{entry.path}\n" for entry in entries) + roots = {entry.root for entry in entries if entry.root} + if len(roots) == 1: + # Remembered by ``pack`` so the listed folder's name is kept. + text = f"{ROOT_HEADER} {roots.pop()}\n" + text + path.write_text(text, encoding="utf-8") + + if not args.quiet: + err.print(f"Found {len(entries)} {noun}(s). List saved to: {escape(str(path))}") + + +def cmd_list_files(args) -> int: + _, err = _consoles() + entries = api.list_files(args.folder, include=args.include, exclude=args.exclude) + + if any(looks_like_archive(entry.path) for entry in entries) and not args.quiet: + err.print( + "[yellow]Warning: this folder contains archive volumes or a manifest. " + "Did you mean `batchvault list archives`?[/yellow]" + ) + _write_listing(args, entries, "file") + return 0 + + +def cmd_list_archives(args) -> int: + _, err = _consoles() + entries = api.list_archives(args.folder) + + if not entries and not args.quiet: + err.print( + "[yellow]Warning: no archives found. " + "Did you mean `batchvault list files`?[/yellow]" + ) + _write_listing(args, entries, "archive") + return 0 + + +def cmd_doctor(args) -> int: + from batchvault import ui + + checks = run_checks(args.seven_zip) + if args.json: + print( + json.dumps( + [vars(check) for check in checks], indent=2, ensure_ascii=False + ) + ) + elif not args.quiet: + ui.print_checks(Console(), checks) + return 0 if passed(checks) else 1 + + +def _use_utf8() -> None: + # Keep file names readable when output is redirected on Windows. + for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(encoding="utf-8") + except (AttributeError, ValueError): + pass + + +def _load_config(argv: list[str]) -> dict | None: + pre = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + pre.add_argument("--config") + known, _ = pre.parse_known_args(argv) + + explicit = known.config or os.environ.get(ENV_CONFIG) + if explicit: + path = Path(explicit).expanduser() + if not path.is_file(): + raise ConfigError(f"Config file not found: {path}") + return load_config(path) # named by the user, so trusted + + path = find_config() + if path is None: + return None + config = load_config(path) + reject_restricted(config, path) # found by searching: not fully trusted + return config + + +def main(argv: list[str] | None = None) -> int: + _use_utf8() + _, err = _consoles() + argv = list(sys.argv[1:] if argv is None else argv) + + try: + parser = build_parser(_load_config(argv)) + except BatchVaultError as exc: + err.print(f"[bold red]Error:[/bold red] {escape(str(exc))}") + return 2 + + args = parser.parse_args(argv) + + log_path = None + if not args.no_log: + log_path = setup_logging(args.log_level, args.log_file) + log.info("batchvault %s: %s", __version__, " ".join(redact_arguments(argv))) + + try: + return args.handler(args) + except BatchVaultError as exc: + log.error("%s", exc) + err.print(f"[bold red]Error:[/bold red] {escape(str(exc))}") + return 2 + except KeyboardInterrupt: + log.warning("Interrupted") + err.print("[yellow]Interrupted.[/yellow]") + return 130 + except Exception as exc: + log.exception("Unexpected error") + where = f" See the log: {log_path}" if log_path else "" + err.print(f"[bold red]Unexpected error:[/bold red] {escape(str(exc))}.{where}") + return 3 + finally: + close_logging() diff --git a/src/batchvault/config.py b/src/batchvault/config.py new file mode 100644 index 0000000..f638a94 --- /dev/null +++ b/src/batchvault/config.py @@ -0,0 +1,169 @@ +"""Project configuration from ``batchvault.toml`` or ``[tool.batchvault]``. + +Layout (keys use the same names as the command-line options):: + + [defaults] # applies to every command that has the option + jobs = 4 + + [log] + level = "DEBUG" # --log-level + file = "run.log" # --log-file + + [notify] + url = "https://hooks.example.com/..." # --notify + on = "failure" # --notify-on + + [pack] # per command: pack, unpack, verify, inspect + level = 7 + volume-size = "100m" + include = ["*.mp4"] + +Command-line options override the file. List options (``include``, +``exclude``) from the file and the command line are combined. +""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from batchvault.errors import ConfigError + +CONFIG_FILE = "batchvault.toml" +PYPROJECT = "pyproject.toml" + +COMMAND_TABLES = ("pack", "unpack", "verify", "inspect") +# Tables that map their own key names onto command-line options. +SECTION_KEYS = { + "log": {"level": "log_level", "file": "log_file"}, + "notify": {"url": "notify", "on": "notify_on"}, +} +KNOWN_TABLES = {"defaults", *COMMAND_TABLES, *SECTION_KEYS} + +# Options that run a program, send data out, choose which keys to trust, read a +# password from somewhere or delete files. A config file found automatically in +# the current or a parent folder may belong to someone else (a cloned +# repository, a downloaded folder), so it may not set these. A file named with +# --config or BATCHVAULT_CONFIG is trusted. +RESTRICTED_OPTIONS = frozenset( + { + "seven_zip", + "notify", + "delete_source", + "yes", + "public_key", + "sign_key", + "password_file", + "log_file", + } +) + + +def _tool_section(pyproject: Path) -> dict | None: + try: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + return None + section = data.get("tool", {}).get("batchvault") + return section if isinstance(section, dict) else None + + +def find_config(start: Path | None = None) -> Path | None: + """Look for a config file in ``start`` and then each parent folder.""" + folder = (start or Path.cwd()).resolve() + for candidate in (folder, *folder.parents): + if (candidate / CONFIG_FILE).is_file(): + return candidate / CONFIG_FILE + pyproject = candidate / PYPROJECT + if pyproject.is_file() and _tool_section(pyproject) is not None: + return pyproject + return None + + +def load_config(path: Path) -> dict: + try: + text = path.read_text(encoding="utf-8") + data = tomllib.loads(text) + except OSError as exc: + raise ConfigError(f"Cannot read config file {path}: {exc}") from exc + except tomllib.TOMLDecodeError as exc: + raise ConfigError(f"Config file {path} is not valid TOML: {exc}") from exc + + if path.name == PYPROJECT: + data = data.get("tool", {}).get("batchvault", {}) + + unknown = sorted(set(data) - KNOWN_TABLES) + if unknown: + raise ConfigError( + f"{path}: unknown section(s): {', '.join(unknown)}. " + f"Known: {', '.join(sorted(KNOWN_TABLES))}." + ) + for name, table in data.items(): + if not isinstance(table, dict): + raise ConfigError(f"{path}: [{name}] must be a table") + return data + + +def reject_restricted(config: dict, path: Path) -> None: + """Refuse restricted options in a config file that was found automatically.""" + for table_name, table in config.items(): + mapping = SECTION_KEYS.get(table_name) + for key in table: + dest = mapping.get(key, key) if mapping else key.replace("-", "_") + if dest in RESTRICTED_OPTIONS: + raise ConfigError( + f"{path} sets '{key}' in [{table_name}]. A config file found " + "automatically cannot set options that run programs, send data, " + "choose trusted keys, read passwords or delete files. If you " + f"trust this file, run with: --config {path}" + ) + + +def _check_value(dest: str, value, action, where: str) -> None: + if action.choices is not None and value not in action.choices: + choices = ", ".join(str(choice) for choice in action.choices) + raise ConfigError(f"{where}: '{dest}' must be one of: {choices}") + + if action.nargs == 0: # on/off flag + if not isinstance(value, bool): + raise ConfigError(f"{where}: '{dest}' must be true or false") + elif action.type is int: + if isinstance(value, bool) or not isinstance(value, int): + raise ConfigError(f"{where}: '{dest}' must be a whole number") + elif action.default is not None and isinstance(action.default, list): + if not isinstance(value, list) or not all(isinstance(v, str) for v in value): + raise ConfigError(f"{where}: '{dest}' must be a list of strings") + elif not isinstance(value, str | int | float) or isinstance(value, bool): + raise ConfigError(f"{where}: '{dest}' has an unsupported value") + + +def defaults_for(config: dict, command: str, actions: dict) -> dict: + """Option defaults for ``command`` (``actions`` maps option name to Action).""" + merged: dict = {} + + for key, value in config.get("defaults", {}).items(): + dest = key.replace("-", "_") + if dest in actions: + _check_value(dest, value, actions[dest], "[defaults]") + merged[dest] = value + + for section, mapping in SECTION_KEYS.items(): + for key, value in config.get(section, {}).items(): + dest = mapping.get(key) + if dest is None: + raise ConfigError( + f"Unknown key '{key}' in [{section}]. " + f"Known: {', '.join(mapping)}." + ) + if dest in actions: + _check_value(dest, value, actions[dest], f"[{section}]") + merged[dest] = value + + for key, value in config.get(command, {}).items(): + dest = key.replace("-", "_") + if dest not in actions: + raise ConfigError(f"Unknown option '{key}' in [{command}].") + _check_value(dest, value, actions[dest], f"[{command}]") + merged[dest] = value + + return merged diff --git a/src/batchvault/crypto.py b/src/batchvault/crypto.py new file mode 100644 index 0000000..48ae7f7 --- /dev/null +++ b/src/batchvault/crypto.py @@ -0,0 +1,193 @@ +"""Optional encryption for the native engine. + +Needs the ``cryptography`` package (``pip install "batchvault[crypto]"``). + +An encrypted archive is a header followed by one record per compressed block +and a final terminator record. Each record is AES-256-GCM encrypted with a +unique nonce; the header, the record number and a "last record" flag are +authenticated, so reordering, dropping or truncating records is detected. The +key comes from the password with scrypt and a random per-archive salt. +""" + +from __future__ import annotations + +import hashlib +import io +import os +import struct + +from batchvault.errors import EngineError, EngineNotFoundError + +MAGIC = b"BVENC1\x00\x00" +FORMAT_IDS = {"xz": 1, "gzip": 2, "bzip2": 3, "store": 4} +_ID_FORMATS = {value: key for key, value in FORMAT_IDS.items()} +SALT_SIZE = 16 +PREFIX_SIZE = 4 +HEADER_SIZE = len(MAGIC) + 2 + SALT_SIZE + PREFIX_SIZE + +# scrypt cost: N = 2**log2_n (about 128 MiB of memory at the default), r = 8, p = 1. +# The value used is stored in the archive header so the default can rise later, +# and header values outside this range are refused: a damaged or hostile header +# must not be able to ask for terabytes of memory. +DEFAULT_LOG2_N = 17 +KDF_LOG2_N_RANGE = (14, 18) +_SCRYPT_R = 8 +_SCRYPT_P = 1 + +_LENGTH = struct.Struct(">I") +_INDEX = struct.Struct(">Q") +# A record holds one compressed block (16 MiB by default); anything near this +# size is damage, and the limit stops a bad length from allocating gigabytes. +_MAX_RECORD = 128 * 1024 * 1024 + +INSTALL_HINT = ( + "Encryption and signing need the optional 'cryptography' package: " + 'pip install "batchvault[crypto]"' +) + + +def require_cryptography() -> None: + try: + import cryptography # noqa: F401 + except ImportError as exc: + raise EngineNotFoundError(INSTALL_HINT) from exc + + +def make_header(fmt: str) -> bytes: + return ( + MAGIC + + bytes([FORMAT_IDS[fmt], DEFAULT_LOG2_N]) + + os.urandom(SALT_SIZE + PREFIX_SIZE) + ) + + +def is_encrypted_header(data: bytes) -> bool: + return data.startswith(MAGIC) + + +def parse_header(data: bytes) -> tuple[str, int, bytes, bytes]: + """Return ``(format, log2_n, salt, nonce_prefix)`` from an archive header.""" + if len(data) < HEADER_SIZE or not is_encrypted_header(data): + raise EngineError("Not an encrypted BatchVault archive.") + fmt = _ID_FORMATS.get(data[len(MAGIC)]) + if fmt is None: + raise EngineError("Unknown format in encrypted archive header.") + + log2_n = data[len(MAGIC) + 1] + low, high = KDF_LOG2_N_RANGE + if not low <= log2_n <= high: + raise EngineError("Unsupported key setting in the encrypted archive header.") + + start = len(MAGIC) + 2 + salt = data[start : start + SALT_SIZE] + return fmt, log2_n, salt, data[start + SALT_SIZE : HEADER_SIZE] + + +class BlockCipher: + """Encrypts and decrypts numbered records for one archive.""" + + def __init__(self, password: str, header: bytes): + require_cryptography() + from cryptography.hazmat.primitives.ciphers.aead import AESGCM + + _, log2_n, salt, prefix = parse_header(header) + n = 1 << log2_n + key = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=n, + r=_SCRYPT_R, + p=_SCRYPT_P, + maxmem=130 * n * _SCRYPT_R, # scrypt needs about 128 * n * r bytes + dklen=32, + ) + self._aes = AESGCM(key) + self._prefix = prefix + self._header = header + + def _nonce(self, index: int) -> bytes: + return self._prefix + _INDEX.pack(index) + + def _aad(self, index: int, last: bool) -> bytes: + return self._header + _INDEX.pack(index) + (b"\x01" if last else b"\x00") + + def encrypt(self, index: int, data: bytes, last: bool = False) -> bytes: + """Return a length-prefixed encrypted record.""" + sealed = self._aes.encrypt(self._nonce(index), data, self._aad(index, last)) + return _LENGTH.pack(len(sealed)) + sealed + + def decrypt(self, index: int, sealed: bytes) -> tuple[bytes, bool]: + """Return ``(data, is_last)``; raise EngineError on a wrong password.""" + from cryptography.exceptions import InvalidTag + + for last in (False, True): + try: + data = self._aes.decrypt( + self._nonce(index), sealed, self._aad(index, last) + ) + except InvalidTag: + continue + return data, last + raise EngineError("Wrong password, or the encrypted archive is damaged.") + + +def _read_exact(source, count: int) -> bytes: + chunks = [] + while count: + chunk = source.read(count) + if not chunk: + break + chunks.append(chunk) + count -= len(chunk) + return b"".join(chunks) + + +class DecryptingReader(io.RawIOBase): + """Read-only stream of the decrypted (still compressed) block data.""" + + def __init__(self, source, cipher: BlockCipher): + super().__init__() + self._source = source + self._cipher = cipher + self._buffer = memoryview(b"") + self._index = 0 + self._done = False + + def readable(self) -> bool: + return True + + def _next_record(self) -> None: + head = _read_exact(self._source, _LENGTH.size) + if len(head) < _LENGTH.size: + raise EngineError("Encrypted archive is truncated (no end marker).") + + (length,) = _LENGTH.unpack(head) + if length > _MAX_RECORD: + raise EngineError("Encrypted archive is damaged (bad record size).") + + sealed = _read_exact(self._source, length) + if len(sealed) < length: + raise EngineError("Encrypted archive is truncated.") + + data, self._done = self._cipher.decrypt(self._index, sealed) + self._index += 1 + self._buffer = memoryview(data) + + def readinto(self, buffer) -> int: + view = memoryview(buffer).cast("B") + while not len(self._buffer): + if self._done: + return 0 + self._next_record() + + count = min(len(view), len(self._buffer)) + view[:count] = self._buffer[:count] + self._buffer = self._buffer[count:] + return count + + +def open_decrypted(source, password: str) -> tuple[str, DecryptingReader]: + """Read the header from ``source``; return the inner format and a reader.""" + header = _read_exact(source, HEADER_SIZE) + fmt = parse_header(header)[0] + return fmt, DecryptingReader(source, BlockCipher(password, header)) diff --git a/src/batchvault/doctor.py b/src/batchvault/doctor.py new file mode 100644 index 0000000..bbb75ab --- /dev/null +++ b/src/batchvault/doctor.py @@ -0,0 +1,67 @@ +"""Checks that BatchVault can run on this machine.""" + +from __future__ import annotations + +import os +import platform +import sys +import tempfile +from dataclasses import dataclass + +from batchvault.engines.sevenzip import find_7zip +from batchvault.errors import EngineNotFoundError +from batchvault.logs import default_log_dir + + +@dataclass(frozen=True) +class Check: + name: str + ok: bool + detail: str + required: bool = False # a failed required check makes doctor exit non-zero + + +def _module(name: str, label: str, required: bool = False) -> Check: + try: + __import__(name) + except ImportError: + return Check(label, False, "not available in this Python build", required) + return Check(label, True, "available", required) + + +def _log_dir_check() -> Check: + folder = default_log_dir() + try: + folder.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=folder): + pass + except OSError as exc: + return Check("Log folder", False, f"{folder} is not writable: {exc}") + return Check("Log folder", True, str(folder)) + + +def run_checks(seven_zip: str | None = None) -> list[Check]: + checks = [ + Check( + "Python", + sys.version_info >= (3, 11, 4), # safe tar extraction filters + f"{platform.python_version()} on {platform.system()}", + required=True, + ), + _module("zlib", "zlib (gzip, store)", required=True), + _module("lzma", "lzma (xz format)"), + _module("bz2", "bz2 (bzip2 format)"), + ] + + try: + checks.append(Check("7-Zip (optional)", True, str(find_7zip(seven_zip)))) + except EngineNotFoundError as exc: + checks.append(Check("7-Zip (optional)", False, str(exc))) + + checks.append(_log_dir_check()) + checks.append(Check("CPU threads", True, str(os.cpu_count() or 1))) + return checks + + +def passed(checks: list[Check]) -> bool: + return all(check.ok for check in checks if check.required) diff --git a/src/batchvault/engines/__init__.py b/src/batchvault/engines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/batchvault/engines/base.py b/src/batchvault/engines/base.py new file mode 100644 index 0000000..cf82cb4 --- /dev/null +++ b/src/batchvault/engines/base.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from pathlib import Path + +from batchvault.options import PackOptions + + +@dataclass(frozen=True) +class Capabilities: + """What an engine supports, used to warn about unsupported settings.""" + + levels: range + threads_per_file: bool + archive_suffix: str + + +class Engine(ABC): + """Common interface implemented by every compression engine.""" + + name: str + capabilities: Capabilities + + @abstractmethod + def is_available(self) -> bool: + """True if the engine can run on this machine.""" + + @abstractmethod + def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]: + """Archive ``source`` into volumes inside ``dest_dir``; return the volumes. + + Encrypts when ``options.password`` is set. Raises OutputExistsError if + volumes already exist and ``options.force`` is false. Existing volumes + are only removed once the new set is complete. + """ + + @abstractmethod + def unpack( + self, + volume: Path, + output_dir: Path, + *, + force: bool = False, + password: str | None = None, + max_bytes: int | None = None, + ) -> list[Path]: + """Restore the files in the volume set containing ``volume``. + + Raises OutputExistsError if a destination exists and ``force`` is false. + With ``max_bytes``, an archive that would expand beyond that many bytes + is refused before anything is written. + """ + + @abstractmethod + def test(self, volume: Path, *, password: str | None = None) -> None: + """Check the volume set's integrity; raise EngineError if it fails.""" diff --git a/src/batchvault/engines/native.py b/src/batchvault/engines/native.py new file mode 100644 index 0000000..3f88d0f --- /dev/null +++ b/src/batchvault/engines/native.py @@ -0,0 +1,237 @@ +from __future__ import annotations + +import os +import tarfile +import tempfile +import zlib +from pathlib import Path, PurePosixPath + +from batchvault import crypto +from batchvault.blocks import ( + BLOCK_SIZE, + BlockWriter, + check_format, + open_decompressed, + sniff_archive, +) +from batchvault.engines.base import Capabilities, Engine +from batchvault.errors import EngineError, OutputExistsError +from batchvault.names import unsafe_on_windows as _unsafe_on_windows +from batchvault.options import MIB, PackOptions +from batchvault.volumes import ( + VolumeReader, + VolumeWriter, + existing_volumes, + find_volumes, + install_volumes, +) + +try: + import lzma + + _LZMA_ERRORS: tuple[type[Exception], ...] = (lzma.LZMAError,) +except ImportError: + _LZMA_ERRORS = () + +# Errors raised while reading damaged, truncated or unreadable archive data. +_READ_ERRORS = (tarfile.TarError, EOFError, zlib.error, OSError, *_LZMA_ERRORS) + +_STREAM_BUFFER = MIB +_CHUNK = MIB +# Data allowed after the tar end marker; a real archive has only padding. +_MAX_TRAILING = 4 * MIB + + +def _normalize(info: tarfile.TarInfo) -> tarfile.TarInfo: + """Drop ownership so archives do not depend on the user who made them.""" + info.uid = info.gid = 0 + info.uname = info.gname = "" + return info + + +class _Budget: + """Caps the total size of the files restored from one archive.""" + + def __init__(self, limit: int | None): + self.remaining = limit + + def charge(self, member: tarfile.TarInfo) -> None: + if self.remaining is None: + return + if member.size > self.remaining: + raise EngineError( + f"{member.name!r} would expand beyond the allowed size; " + "the archive was not restored." + ) + self.remaining -= member.size + + +def _check_member(member: tarfile.TarInfo, windows: bool | None = None) -> None: + """Accept only a plain, single-part regular file name that is safe to write.""" + if windows is None: + windows = os.name == "nt" + + path = PurePosixPath(member.name) + if ( + not member.isfile() + or path.is_absolute() + or len(path.parts) != 1 + or path.name in ("", ".", "..") + or "\\" in member.name + or (windows and _unsafe_on_windows(member.name)) + ): + raise EngineError(f"Unsupported or unsafe entry in archive: {member.name!r}") + + +class NativeEngine(Engine): + """Standard-library engine: tar stream, block-parallel compression.""" + + name = "native" + capabilities = Capabilities( + levels=range(0, 10), threads_per_file=True, archive_suffix=".bv" + ) + + def __init__(self, block_size: int = BLOCK_SIZE): + self._block_size = block_size + + def is_available(self) -> bool: + return True + + def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]: + if not source.is_file(): + raise FileNotFoundError(f"Source file not found: {source}") + check_format(options.format) + + base = f"{source.name}{self.capabilities.archive_suffix}" + existing = existing_volumes(dest_dir, base) + if existing and not options.force: + raise OutputExistsError( + f"Archive volumes already exist in {dest_dir}. Use --force to replace." + ) + + header = cipher = None + if options.password: + header = crypto.make_header(options.format) + cipher = crypto.BlockCipher(options.password, header) + + dest_dir.mkdir(parents=True, exist_ok=True) + + # Build in a temporary folder so a failed run never damages what exists. + with tempfile.TemporaryDirectory(dir=dest_dir, prefix=".batchvault-") as tmp: + with VolumeWriter(Path(tmp), base, options.volume_size) as volumes: + with BlockWriter( + volumes, + options.format, + options.level, + self._block_size, + options.jobs, + cipher=cipher, + header=header or b"", + ) as blocks: + with tarfile.open( + fileobj=blocks, + mode="w|", + format=tarfile.PAX_FORMAT, + bufsize=_STREAM_BUFFER, + ) as tar: + tar.add(source, arcname=source.name, filter=_normalize) + + return install_volumes(volumes.volumes, dest_dir, existing) + + def unpack( + self, + volume: Path, + output_dir: Path, + *, + force: bool = False, + password: str | None = None, + max_bytes: int | None = None, + ) -> list[Path]: + volumes = find_volumes(volume) + output_dir.mkdir(parents=True, exist_ok=True) + budget = _Budget(max_bytes) + + def handle(member, tar) -> None: + budget.charge(member) # refuse a decompression bomb before writing + self._extract(member, tar, staging) + + with tempfile.TemporaryDirectory(dir=output_dir, prefix=".batchvault-") as tmp: + staging = Path(tmp) + self._read(volumes, handle, password) + + extracted = sorted(staging.iterdir()) + if not extracted: + raise EngineError(f"No files found in archive {volumes[0]}") + + conflicts = [ + output_dir / item.name + for item in extracted + if (output_dir / item.name).exists() + ] + if conflicts and not force: + names = ", ".join(path.name for path in conflicts) + raise OutputExistsError( + f"Destination already exists: {names}. Use --force to overwrite." + ) + + restored = [] + for item in extracted: + target = output_dir / item.name + os.replace(item, target) + restored.append(target) + + return restored + + def test(self, volume: Path, *, password: str | None = None) -> None: + self._read(find_volumes(volume), self._drain_member, password) + + @staticmethod + def _extract(member, tar, staging: Path) -> None: + # Belt and braces next to _check_member and the "data" filter: whatever + # the name looks like, the file must end up inside the staging folder. + root = staging.resolve() + if root not in (root / member.name).resolve().parents: + raise EngineError(f"Unsafe entry in archive: {member.name!r}") + tar.extract(member, staging, filter="data") + + @staticmethod + def _drain_member(member, tar) -> None: + handle = tar.extractfile(member) + while handle.read(_CHUNK): + pass + + @staticmethod + def _read(volumes: list[Path], handle_member, password: str | None) -> None: + """Stream the archive and call ``handle_member(member, tar)`` for each file.""" + fmt, encrypted = sniff_archive(volumes[0]) + if encrypted and not password: + raise EngineError( + f"{volumes[0].name} is encrypted. A password is required." + ) + + try: + with VolumeReader(volumes) as reader: + source = reader + if encrypted: + fmt, source = crypto.open_decrypted(reader, password) + + with open_decompressed(fmt, source) as stream: + with tarfile.open(fileobj=stream, mode="r|") as tar: + for member in tar: + _check_member(member) + handle_member(member, tar) + + # Read to the very end so a truncated stream is detected. A + # real archive has almost nothing after the tar end marker. + trailing = 0 + while chunk := stream.read(_CHUNK): + trailing += len(chunk) + if trailing > _MAX_TRAILING: + raise EngineError( + f"{volumes[0].name} has unexpected data after the " + "end of the archive." + ) + except _READ_ERRORS as exc: + raise EngineError( + f"Archive is corrupt or incomplete ({volumes[0].name}): {exc}" + ) from exc diff --git a/src/batchvault/engines/registry.py b/src/batchvault/engines/registry.py new file mode 100644 index 0000000..e270332 --- /dev/null +++ b/src/batchvault/engines/registry.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from pathlib import Path + +from batchvault.engines.base import Engine +from batchvault.engines.native import NativeEngine +from batchvault.engines.sevenzip import SevenZipEngine, find_7zip +from batchvault.errors import InputError +from batchvault.volumes import split_volume_name + +ENGINE_CHOICES = ("auto", "native", "7zip") + + +def get_engine(name: str = "auto", seven_zip: str | Path | None = None) -> Engine: + """Return the engine to create archives with. + + ``auto`` uses 7-Zip when it is found (or when a path to it was given) and + falls back to the native engine. + """ + if name == "native": + return NativeEngine() + + if name in ("7zip", "7z"): + find_7zip(seven_zip) # raises EngineNotFoundError with install help + return SevenZipEngine(seven_zip) + + if name == "auto": + if seven_zip: + find_7zip(seven_zip) + return SevenZipEngine(seven_zip) + sevenzip = SevenZipEngine() + return sevenzip if sevenzip.is_available() else NativeEngine() + + choices = ", ".join(ENGINE_CHOICES) + raise InputError(f"Unknown engine {name!r}. Choose from: {choices}.") + + +def engine_for_volume(volume: Path, seven_zip: str | Path | None = None) -> Engine: + """Pick the engine that can read a volume, from its file name.""" + parsed = split_volume_name(volume.name) + if parsed is None: + raise InputError(f"Not an archive volume: {volume}") + if parsed[0].lower().endswith(".7z"): + find_7zip(seven_zip) + return SevenZipEngine(seven_zip) + return NativeEngine() + + +def option_warnings( + engine: Engine, format_given: str | None, level: int +) -> list[str]: + """Settings the chosen engine cannot honour exactly.""" + warnings = [] + if engine.name == "7zip" and format_given not in (None, "xz"): + warnings.append( + f"The 7-Zip engine always writes 7z (LZMA2); --format {format_given} " + "is ignored." + ) + if engine.name == "native" and format_given == "bzip2" and level == 0: + warnings.append("bzip2 has no level 0; level 1 was used.") + return warnings diff --git a/src/batchvault/engines/sevenzip.py b/src/batchvault/engines/sevenzip.py new file mode 100644 index 0000000..cca8e48 --- /dev/null +++ b/src/batchvault/engines/sevenzip.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +from batchvault.engines.base import Capabilities, Engine +from batchvault.errors import EngineError, EngineNotFoundError, OutputExistsError +from batchvault.options import PackOptions +from batchvault.volumes import existing_volumes, find_volumes, install_volumes + +ENV_VAR = "BATCHVAULT_7Z" +COMMANDS = ("7zz", "7z", "7za", "7zr") +WINDOWS_PATHS = ( + Path(r"C:\Program Files\7-Zip\7z.exe"), + Path(r"C:\Program Files (x86)\7-Zip\7z.exe"), +) +OUTPUT_TAIL_LINES = 15 +LIMIT_POLL_SECONDS = 0.05 + + +def _search_path(command: str) -> Path | None: + """Find ``command`` in the folders listed in ``PATH``. + + Unlike ``shutil.which``, this never looks in the current folder (Windows does + that first) and ignores empty and relative ``PATH`` entries, so a program + planted in the folder you happen to be working in is never run. + """ + cwd = Path.cwd().resolve() + if sys.platform == "win32": + extensions = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";") + names = [command + extension.lower() for extension in extensions if extension] + else: + names = [command] + + for entry in os.environ.get("PATH", "").split(os.pathsep): + folder = Path(entry) if entry else None + if folder is None or not folder.is_absolute(): + continue + try: + if folder.resolve() == cwd: + continue + except OSError: + continue + + for name in names: + candidate = folder / name + if candidate.is_file() and os.access(candidate, os.X_OK): + return candidate + return None + + +def find_7zip(explicit: str | Path | None = None) -> Path: + """Locate a 7-Zip command-line executable. + + Order: explicit path, ``BATCHVAULT_7Z``, ``PATH`` (never the current folder), + then well-known Windows install folders. + """ + configured = explicit or os.environ.get(ENV_VAR) + if configured: + path = Path(configured).expanduser() + if path.is_file(): + return path + raise EngineNotFoundError(f"7-Zip executable not found: {configured}") + + for command in COMMANDS: + found = _search_path(command) + if found: + return found + + if sys.platform == "win32": + for candidate in WINDOWS_PATHS: + if candidate.is_file(): + return candidate + + raise EngineNotFoundError( + "Could not find 7-Zip. Install it from https://www.7-zip.org/ (7zz on " + f"Linux and macOS), or set the path with --7z or {ENV_VAR}." + ) + + +def build_pack_command( + executable: Path, archive: Path, source: Path, options: PackOptions +) -> list[str]: + command = [ + str(executable), + "a", + "-t7z", + f"-mx={options.level}", + "-m0=LZMA2", + f"-mmt={max(options.jobs, 1)}", + f"-v{options.volume_size}b", + "-y", + # A file name containing * or ? is a name, not a pattern; without this + # 7-Zip would archive every neighbour it matches. + "-spd", + ] + if options.password: + # 7-Zip only takes the password on its command line, where other users + # of this machine could see it in the process list while it runs. + command += [f"-p{options.password}", "-mhe=on"] + return [*command, str(archive), str(source)] + + +def _password_args(password: str | None) -> list[str]: + return [f"-p{password}"] if password else [] + + +def build_unpack_command( + executable: Path, first_volume: Path, target: Path, password: str | None = None +): + return [ + str(executable), + "e", + str(first_volume), + f"-o{target}", + "-y", + "-spd", + *_password_args(password), + ] + + +def build_test_command( + executable: Path, first_volume: Path, password: str | None = None +) -> list[str]: + return [ + str(executable), "t", str(first_volume), "-spd", *_password_args(password) + ] # fmt: skip + + +def folder_size(folder: Path) -> int: + """Total size of the files below ``folder`` (files that vanish are ignored).""" + total = 0 + for root, _, names in os.walk(folder): + for name in names: + try: + total += (Path(root) / name).stat().st_size + except OSError: + continue + return total + + +class SevenZipEngine(Engine): + name = "7zip" + capabilities = Capabilities( + levels=range(0, 10), threads_per_file=True, archive_suffix=".7z" + ) + + def __init__(self, executable: str | Path | None = None): + self._explicit = executable + self._executable: Path | None = None + + @property + def executable(self) -> Path: + if self._executable is None: + self._executable = find_7zip(self._explicit) + return self._executable + + def is_available(self) -> bool: + try: + find_7zip(self._explicit) + except EngineNotFoundError: + return False + return True + + @staticmethod + def _failure(code: int, output: str) -> EngineError: + tail = "\n".join(output.strip().splitlines()[-OUTPUT_TAIL_LINES:]) + return EngineError(f"7-Zip failed (exit code {code}).\n{tail}".rstrip()) + + def _run(self, command: list[str]) -> str: + """Run 7-Zip; return its standard output, raise EngineError on failure.""" + result = subprocess.run( + command, + capture_output=True, + stdin=subprocess.DEVNULL, # never wait for a password prompt + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + if result.returncode != 0: + raise self._failure(result.returncode, result.stderr or result.stdout) + return result.stdout + + def _run_limited(self, command: list[str], watch: Path, max_bytes: int) -> str: + """Like ``_run``, but stop the program if ``watch`` grows past ``max_bytes``. + + 7-Zip writes as it extracts, so a decompression bomb is cut off while it + is still small instead of filling the disk. Output goes to a temporary + file so a full pipe can never stall the program. + """ + too_big = EngineError( + f"The archive would expand beyond the allowed size ({max_bytes} bytes); " + "extraction was stopped." + ) + + with tempfile.TemporaryFile() as output: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=output, + stderr=subprocess.STDOUT, + ) + try: + while True: + try: + code = process.wait(timeout=LIMIT_POLL_SECONDS) + break + except subprocess.TimeoutExpired: + if folder_size(watch) > max_bytes: + raise too_big from None + finally: + if process.poll() is None: + process.kill() + process.wait() + + output.seek(0) + text = output.read().decode("utf-8", errors="replace") + + if folder_size(watch) > max_bytes: # it finished between two checks + raise too_big + if code != 0: + raise self._failure(code, text) + return text + + def pack(self, source: Path, dest_dir: Path, options: PackOptions) -> list[Path]: + if not source.is_file(): + raise FileNotFoundError(f"Source file not found: {source}") + + base = f"{source.name}{self.capabilities.archive_suffix}" + existing = existing_volumes(dest_dir, base) + if existing and not options.force: + raise OutputExistsError( + f"Archive volumes already exist in {dest_dir}. Use --force to replace." + ) + + dest_dir.mkdir(parents=True, exist_ok=True) + + # Build in a temporary folder so a failed run never damages what exists. + with tempfile.TemporaryDirectory(dir=dest_dir, prefix=".batchvault-") as tmp: + staging = Path(tmp) + self._run( + build_pack_command(self.executable, staging / base, source, options) + ) + + built = existing_volumes(staging, base) + if not built: + raise EngineError(f"7-Zip produced no volumes for {source}") + + return install_volumes(built, dest_dir, existing) + + def unpack( + self, + volume: Path, + output_dir: Path, + *, + force: bool = False, + password: str | None = None, + max_bytes: int | None = None, + ) -> list[Path]: + first = find_volumes(volume)[0] + output_dir.mkdir(parents=True, exist_ok=True) + + with tempfile.TemporaryDirectory(dir=output_dir, prefix=".batchvault-") as tmp: + staging = Path(tmp) + command = build_unpack_command(self.executable, first, staging, password) + if max_bytes is None: + self._run(command) + else: + self._run_limited(command, staging, max_bytes) + + # Never accept links or other special files from an archive. + if any(p.is_symlink() or not p.is_file() for p in staging.iterdir()): + raise EngineError( + f"{first.name} contains a link or special file, which is refused." + ) + + extracted = sorted(staging.iterdir()) + if not extracted: + raise EngineError(f"7-Zip extracted no files from {first}") + + conflicts = [output_dir / p.name for p in extracted] + conflicts = [path for path in conflicts if path.exists()] + if conflicts and not force: + names = ", ".join(path.name for path in conflicts) + raise OutputExistsError( + f"Destination already exists: {names}. Use --force to overwrite." + ) + + restored = [] + for item in extracted: + target = output_dir / item.name + os.replace(item, target) + restored.append(target) + + return restored + + def test(self, volume: Path, *, password: str | None = None) -> None: + first = find_volumes(volume)[0] + self._run(build_test_command(self.executable, first, password)) diff --git a/src/batchvault/errors.py b/src/batchvault/errors.py new file mode 100644 index 0000000..8880235 --- /dev/null +++ b/src/batchvault/errors.py @@ -0,0 +1,38 @@ +class BatchVaultError(Exception): + """Base class for all BatchVault errors.""" + + +class EngineNotFoundError(BatchVaultError): + """The requested compression engine is not available.""" + + +class OptionsError(BatchVaultError, ValueError): + """An option value is invalid.""" + + +class ConfigError(BatchVaultError): + """The configuration file is unreadable or contains invalid settings.""" + + +class SignatureError(BatchVaultError): + """A manifest signature is missing, invalid or cannot be created.""" + + +class InputError(BatchVaultError): + """An input file, list or path is missing or unusable.""" + + +class EngineError(BatchVaultError): + """A compression engine failed to run or reported an error.""" + + +class OutputExistsError(BatchVaultError): + """The destination already exists and replacing it was not requested.""" + + +class ManifestError(BatchVaultError): + """A manifest is missing, malformed or fails validation.""" + + +class VolumeError(BatchVaultError): + """Archive volumes are missing, incomplete or corrupt.""" diff --git a/src/batchvault/eta.py b/src/batchvault/eta.py new file mode 100644 index 0000000..a1410d0 --- /dev/null +++ b/src/batchvault/eta.py @@ -0,0 +1,39 @@ +import time + +# Wait for this much real run time before trusting the measured speed. +MIN_SECONDS = 5.0 + + +def format_duration(seconds): + seconds = max(0, int(seconds)) + hours, remainder = divmod(seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" + + +class EtaEstimator: + """ETA from the average speed of the whole run so far. + + Speed is bytes completed divided by the time since the first file started, + so per-file overhead is included. + """ + + def __init__(self, clock=time.monotonic): + self.clock = clock + self.started_at = None + + def start(self): + if self.started_at is None: + self.started_at = self.clock() + + def estimate(self, done_bytes, total_bytes): + """Seconds remaining, or None while there is not enough data yet.""" + if self.started_at is None or done_bytes <= 0: + return None + + elapsed = self.clock() - self.started_at + if elapsed < MIN_SECONDS: + return None + + remaining = max(0, total_bytes - done_bytes) + return remaining / (done_bytes / elapsed) diff --git a/src/batchvault/filters.py b/src/batchvault/filters.py new file mode 100644 index 0000000..8555fb9 --- /dev/null +++ b/src/batchvault/filters.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path + +import pathspec + +from batchvault.errors import InputError + + +def load_patterns(values: Iterable[str]) -> list[str]: + """Expand ``@file`` references into the patterns listed in that file.""" + patterns = [] + for value in values: + if value.startswith("@"): + path = Path(value[1:]).expanduser() + try: + lines = path.read_text(encoding="utf-8-sig").splitlines() + except OSError as exc: + raise InputError(f"Cannot read pattern file {path}: {exc}") from exc + patterns.extend( + line.strip() + for line in lines + if line.strip() and not line.lstrip().startswith("#") + ) + elif value.strip(): + patterns.append(value.strip()) + return patterns + + +def _spec(patterns: list[str]): + return pathspec.GitIgnoreSpec.from_lines(patterns) if patterns else None + + +class PathFilter: + """Include/exclude filter using gitignore-style patterns (``*``, ``**``, ``/``). + + Exclude wins. If any include pattern is given, only matching paths are kept. + """ + + def __init__(self, include: Iterable[str] = (), exclude: Iterable[str] = ()): + self._include = _spec(load_patterns(include)) + self._exclude = _spec(load_patterns(exclude)) + + def matches(self, path: str) -> bool: + path = path.replace("\\", "/").lstrip("/") + if self._exclude and self._exclude.match_file(path): + return False + if self._include and not self._include.match_file(path): + return False + return True diff --git a/src/batchvault/inputs.py b/src/batchvault/inputs.py new file mode 100644 index 0000000..02bbd4c --- /dev/null +++ b/src/batchvault/inputs.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import json +import os +import sys +from collections.abc import Iterable +from pathlib import Path + +from batchvault.errors import InputError +from batchvault.filters import PathFilter +from batchvault.manifest import MANIFEST_NAME +from batchvault.volumes import split_volume_name + +# A text list written by ``list files`` starts with this comment. It records the +# folder the files were listed from, so ``pack`` keeps that folder's name. +ROOT_HEADER = "# batchvault-root:" + +ListItem = tuple[str, str, str | None] # (label, path, root the list recorded) + + +def read_list_items(list_path: Path) -> list[ListItem]: + """Read a list file as ``(label, path, root)`` items. + + Text lists have one path per line; blank lines and ``#`` comments are + ignored, except the ``# batchvault-root:`` header written by ``list files``. + JSON lists are an array of paths or ``{"path": ..., "root": ...}`` objects (as + printed by ``batchvault list files``), or an object with such an array under + ``"files"``. ``root`` is the folder the file was listed from, if recorded. + """ + if str(list_path) == "-": + list_path = Path("") + text = sys.stdin.read().lstrip("") + else: + try: + text = list_path.read_text(encoding="utf-8-sig") + except OSError as exc: + raise InputError(f"Cannot read list file {list_path}: {exc}") from exc + + if list_path.suffix.lower() == ".json" or text.lstrip().startswith(("[", "{")): + return _json_items(list_path, text) + + items = [] + root = None + for number, line in enumerate(text.splitlines(), start=1): + entry = line.strip() + if entry.startswith(ROOT_HEADER): + root = entry[len(ROOT_HEADER) :].strip() or None + elif entry and not entry.startswith("#"): + items.append((f"{list_path}:{number}", entry, root)) + return items + + +def read_list_entries(list_path: Path) -> list[tuple[str, str]]: + """Read a list file as ``(label, path)`` pairs (see ``read_list_items``).""" + return [(label, entry) for label, entry, _ in read_list_items(list_path)] + + +def _json_items(list_path: Path, text: str) -> list[ListItem]: + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise InputError(f"{list_path} is not valid JSON: {exc}") from exc + + default_root = None + if isinstance(data, dict): + default_root = data.get("root") if isinstance(data.get("root"), str) else None + data = data.get("files") + if not isinstance(data, list): + raise InputError(f"{list_path}: expected a JSON array of paths") + + items = [] + for index, item in enumerate(data, start=1): + value = item.get("path") if isinstance(item, dict) else item + if not isinstance(value, str) or not value.strip(): + raise InputError(f"{list_path}: item {index} has no path") + root = item.get("root") if isinstance(item, dict) else None + if not isinstance(root, str) or not root.strip(): + root = default_root + items.append((f"{list_path}[{index}]", value.strip(), root)) + return items + + +def relative_folder(path: Path, root: Path) -> str: + """The folder of ``path`` relative to ``root``, with forward slashes. + + Returns ``""`` when the file sits directly in ``root``. + """ + try: + relative = path.parent.relative_to(root) + except ValueError: + raise InputError( + f"{path} is not inside the root folder {root}. Choose a root that " + "contains all the files, or leave --root out." + ) from None + return relative.as_posix() if relative.parts else "" + + +def assign_folders( + items: list[tuple[Path, Path | None]], root: Path | None = None +) -> dict[Path, str]: + """Decide the folder each file is stored under (``""`` = not in a subfolder). + + - With an explicit ``root``, every file is stored relative to it. + - A file from a ``list files`` list is stored relative to the folder that + listing recorded, so the listed folder's own name is kept. + - Other files are stored relative to the deepest folder they all share, so the + folders below it are kept and files directly in it are not nested. + """ + if root is not None: + return {path: relative_folder(path, root) for path, _ in items} + + folders: dict[Path, str] = {} + unlisted = [] + for path, recorded in items: + if recorded is not None: + try: + folders[path] = relative_folder(path, recorded) + continue + except InputError: + pass # the list was edited since it was made; use the shared folder + unlisted.append(path) + + if unlisted: + try: + shared = Path(os.path.commonpath([str(p.parent) for p in unlisted])) + except ValueError: + raise InputError( + "These files are on different drives, so there is no folder they " + "share. Archive each drive separately, or pass --root." + ) from None + for path in unlisted: + folders[path] = relative_folder(path, shared) + return folders + + +def looks_like_archive(path: Path) -> bool: + return split_volume_name(path.name) is not None or path.name == MANIFEST_NAME + + +def gather_sources( + files: Iterable[str | Path] = (), + list_file: str | Path | None = None, + *, + path_filter: PathFilter | None = None, + allow_archives: bool = False, + root: Path | None = None, +) -> list[tuple[Path, str]]: + """Turn arguments and/or a list file into unique, existing source files. + + Returns ``(file, folder)`` pairs, where ``folder`` is the folder the file is + stored under (see ``assign_folders``). Relative paths are resolved from the + current directory. Files with the same name are fine when their folders + differ. + """ + if root is not None: + root = Path(root).expanduser().resolve() + + items: list[ListItem] = [("argument", str(file), None) for file in files] + if list_file is not None: + items += read_list_items( + Path(list_file) if str(list_file) == "-" else Path(list_file).expanduser() + ) + + sources: dict[str, tuple[Path, Path | None]] = {} + + for label, entry, recorded in items: + path = Path(entry).expanduser() + if not path.is_absolute(): + path = Path.cwd() / path + if not path.is_file(): + raise InputError(f"{label}: source file not found: {entry}") + + path = path.resolve() + if looks_like_archive(path) and not allow_archives: + raise InputError( + f"{label}: {path.name} looks like an archive volume or manifest. " + "Use allow_archives / --allow-archives to archive it anyway." + ) + if path_filter and not path_filter.matches(path.as_posix()): + continue + + key = os.path.normcase(str(path)) + if key not in sources: + hint = Path(recorded).expanduser().resolve() if recorded else None + sources[key] = (path, hint) + + if not sources: + raise InputError("No input files to archive.") + + folders = assign_folders(list(sources.values()), root) + + # Each file gets its own archive folder named after it, inside its stored + # folder; two files may not end up in the same place. + taken: dict[tuple[str, str], Path] = {} + for path, _ in sources.values(): + place = (folders[path].casefold(), path.name.casefold()) + if place in taken: + raise InputError( + "Two source files would use the same archive folder: " + f"{taken[place]} and {path}. Archive them separately, or keep " + "their folders apart with --root." + ) + taken[place] = path + + return [(path, folders[path]) for path, _ in sources.values()] + + +def collect_sources( + files: Iterable[str | Path] = (), + list_file: str | Path | None = None, + *, + path_filter: PathFilter | None = None, + allow_archives: bool = False, + root: Path | None = None, +) -> list[Path]: + """Like ``gather_sources``, returning only the files.""" + return [ + path + for path, _ in gather_sources( + files, + list_file, + path_filter=path_filter, + allow_archives=allow_archives, + root=root, + ) + ] diff --git a/src/batchvault/levels.py b/src/batchvault/levels.py new file mode 100644 index 0000000..dab2af6 --- /dev/null +++ b/src/batchvault/levels.py @@ -0,0 +1,169 @@ +"""Pick a compression level for each file from its type (``--level auto``). + +The choice uses only the file's signature (its first bytes) and extension, so it +is instant and predictable. Level 0 means "do not try": the file is already +compressed, so a stronger level would only cost time and memory. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path + +from batchvault.errors import OptionsError + +AUTO = "auto" +DEFAULT_LEVEL = 5 + +TEXT_LEVEL = 6 +MEDIA_LEVEL = 6 +BINARY_LEVEL = 5 +PDF_LEVEL = 3 +INCOMPRESSIBLE_LEVEL = 0 + + +@dataclass(frozen=True) +class LevelChoice: + level: int + kind: str # a short reason, shown next to the result + + +def parse_level(value: int | str) -> int | str: + """Validate a level: 0 to 9, or ``"auto"``.""" + if isinstance(value, str): + text = value.strip().lower() + if text == AUTO: + return AUTO + try: + value = int(text) + except ValueError: + raise OptionsError( + f"Invalid compression level {value!r}. Use 0-9 or auto." + ) from None + + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 9: + raise OptionsError("Compression level must be between 0 and 9, or auto.") + return value + + +def level_argument(text: str) -> int | str: + """argparse type for ``--level``; the range is checked by ``choices``.""" + value = text.strip().lower() + if value == AUTO: + return AUTO + try: + return int(value) + except ValueError: + raise argparse.ArgumentTypeError( + f"invalid level {text!r}: use 0-9 or auto" + ) from None + + +def _extensions(text: str) -> frozenset[str]: + return frozenset(text.split()) + + +# Formats that are already compressed (or encrypted): nothing left to squeeze. +INCOMPRESSIBLE = _extensions( + """ + zip 7z rar gz tgz xz txz bz2 tbz2 zst lz4 lzma z cab jar war ear apk ipa + docx xlsx pptx odt ods odp epub + jpg jpeg png gif webp heic heif avif cr2 nef arw dng orf rw2 + mp4 m4v mov mkv webm avi wmv flv mpg mpeg m2ts 3gp + mp3 aac m4a ogg oga opus flac wma wv + """ +) + +# Plain text and similar: very repetitive, so a strong level pays off. +TEXT = _extensions( + """ + txt log csv tsv json jsonl ndjson xml html htm xhtml css scss sass less + js mjs jsx tsx md markdown rst yaml yml toml ini cfg conf properties sql + py pyi java kt c h cc cpp hpp cs go rs rb php swift sh bash zsh bat cmd ps1 + tex bib srt vtt ass svg rtf eml ics vcf gpx kml diff patch gradle tf + ipynb r pl lua dart scala vue + """ +) + +# Uncompressed pictures and sound. +UNCOMPRESSED_MEDIA = _extensions( + "wav aiff aif bmp tif tiff ppm pgm pbm tga dds pcm psd" +) + +# Programs, disk images, databases and containers: compressible, but the +# strongest levels gain little for the memory they need. +BINARY = _extensions( + """ + exe dll so dylib o a lib class pyc wasm iso img vhd vhdx vmdk qcow2 vdi + db sqlite sqlite3 mdb bak dump pak tar + """ +) + +PDF = _extensions("pdf") + +# (offset, bytes) at the start of a file that mark an already compressed format. +_SIGNATURES = ( + (0, b"PK\x03\x04"), # zip, docx, jar, apk, epub, ... + (0, b"PK\x05\x06"), + (0, b"\x1f\x8b"), # gzip + (0, b"\xfd7zXZ\x00"), # xz + (0, b"7z\xbc\xaf\x27\x1c"), + (0, b"Rar!\x1a\x07"), + (0, b"\x28\xb5\x2f\xfd"), # zstd + (0, b"\x04\x22\x4d\x18"), # lz4 + (0, b"MSCF"), # cab + (0, b"\xff\xd8\xff"), # jpeg + (0, b"\x89PNG\r\n\x1a\n"), + (0, b"GIF87a"), + (0, b"GIF89a"), + (4, b"ftyp"), # mp4, mov, heic, m4a + (0, b"\x1a\x45\xdf\xa3"), # matroska, webm + (0, b"OggS"), + (0, b"fLaC"), + (0, b"\x00\x00\x01\xba"), # mpeg program stream +) + + +def _has_compressed_signature(head: bytes) -> bool: + if any(head[at : at + len(mark)] == mark for at, mark in _SIGNATURES): + return True + if head[:4] == b"RIFF" and head[8:12] == b"WEBP": + return True + # bzip2: "BZh", a block size digit and the block magic. The longer check keeps + # a text file that happens to start with "BZh" from being misjudged. + if head[:3] == b"BZh" and head[3:4] in b"123456789" and head[4:10] == b"1AY&SY": + return True + # ID3v2 tag (mp3): version, flags, then a size whose bytes are below 0x80. + return ( + head[:3] == b"ID3" + and head[3:4] in (b"\x02", b"\x03", b"\x04") + and head[4:5] == b"\x00" + and len(head) >= 10 + and all(byte < 0x80 for byte in head[6:10]) + ) + + +def choose_level(path: Path) -> LevelChoice: + """The best compression level for this file, judged by its type.""" + try: + with path.open("rb") as handle: + head = handle.read(16) + except OSError: + head = b"" + + if _has_compressed_signature(head): + return LevelChoice(INCOMPRESSIBLE_LEVEL, "already compressed") + + extension = path.suffix.lower().lstrip(".") + if extension in INCOMPRESSIBLE: + return LevelChoice(INCOMPRESSIBLE_LEVEL, "already compressed") + if extension in TEXT: + return LevelChoice(TEXT_LEVEL, "text") + if extension in UNCOMPRESSED_MEDIA: + return LevelChoice(MEDIA_LEVEL, "uncompressed media") + if extension in BINARY: + return LevelChoice(BINARY_LEVEL, "binary data") + if extension in PDF: + return LevelChoice(PDF_LEVEL, "pdf") + return LevelChoice(DEFAULT_LEVEL, "unknown type") diff --git a/src/batchvault/locate.py b/src/batchvault/locate.py new file mode 100644 index 0000000..430829b --- /dev/null +++ b/src/batchvault/locate.py @@ -0,0 +1,140 @@ +"""Work out which archives a path given to unpack, verify or list refers to.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from batchvault.errors import InputError, ManifestError +from batchvault.inputs import read_list_entries +from batchvault.manifest import MANIFEST_NAME, Manifest, ManifestEntry +from batchvault.volumes import split_volume_name + + +@dataclass +class Target: + """One archive to process. ``entry`` is None when no manifest is available.""" + + volume: Path # first volume (or the given one when there is no manifest) + entry: ManifestEntry | None = None + base_dir: Path | None = None # folder the entry's volume paths are relative to + manifest: Path | None = None # the manifest file the entry came from + + @property + def name(self) -> str: + if self.entry: + return self.entry.name + base = split_volume_name(self.volume.name)[0] + return base.rsplit(".", 1)[0] + + +def _same(a: Path, b: Path) -> bool: + try: + return a.resolve() == b.resolve() + except OSError: + return False + + +def _find_manifest(*folders: Path) -> Path | None: + for folder in folders: + candidate = folder / MANIFEST_NAME + if candidate.is_file(): + return candidate + return None + + +Loader = Callable[[Path], Manifest] + + +def _manifest_targets( + manifest_path: Path, under: Path | None = None, load: Loader = Manifest.load +) -> list[Target]: + manifest = load(manifest_path) + base = manifest_path.parent + + targets = [] + for entry in manifest.entries.values(): + first = base / entry.volumes[0].file + if under is None or _is_under(first, under): + targets.append(Target(first, entry, base, manifest_path)) + return targets + + +def _is_under(path: Path, folder: Path) -> bool: + try: + return path.resolve().is_relative_to(folder.resolve()) + except OSError: + return False + + +def _volume_target(volume: Path, load: Loader = Manifest.load) -> Target: + """Target for one volume path, using a nearby manifest when it knows it.""" + if split_volume_name(volume.name) is None: + raise InputError(f"Not an archive volume: {volume}") + if not volume.is_file(): + raise InputError(f"Archive volume not found: {volume}") + + manifest_path = _find_manifest(volume.parent, volume.parent.parent) + if manifest_path: + base = manifest_path.parent + for entry in load(manifest_path).entries.values(): + if any(_same(base / v.file, volume) for v in entry.volumes): + return Target(base / entry.volumes[0].file, entry, base, manifest_path) + + return Target(volume) + + +def resolve_targets( + source: str | Path, load: Loader = Manifest.load +) -> list[Target]: + """Archives referred to by a manifest, folder, volume file or list file. + + Every manifest is read through ``load``, so a caller can verify a manifest's + signature on the very bytes that are then parsed. + """ + path = Path(source).expanduser() + if not path.exists(): + raise InputError(f"Path not found: {path}") + + if path.is_dir(): + manifest_path = _find_manifest(path, path.parent) + if manifest_path: + under = None if manifest_path.parent == path else path + targets = _manifest_targets(manifest_path, under, load) + else: + targets = [ + Target(found) + for found in sorted(path.rglob("*")) + if found.is_file() + and (parsed := split_volume_name(found.name)) + and parsed[1] == 1 + ] + if not targets: + raise InputError(f"No archives found in {path}") + return targets + + if split_volume_name(path.name): + return [_volume_target(path, load)] + + if path.name == MANIFEST_NAME or path.suffix.lower() == ".json": + try: + return _manifest_targets(path, None, load) + except ManifestError: + # A JSON list of volume paths is also accepted; fall through. + if path.name == MANIFEST_NAME: + raise + + targets = [] + for label, entry in read_list_entries(path): + volume = Path(entry).expanduser() + if not volume.is_absolute(): + volume = path.parent / volume + try: + targets.append(_volume_target(volume, load)) + except InputError as exc: + raise InputError(f"{label}: {exc}") from exc + + if not targets: + raise InputError(f"No archive paths found in {path}") + return targets diff --git a/src/batchvault/logs.py b/src/batchvault/logs.py new file mode 100644 index 0000000..267804b --- /dev/null +++ b/src/batchvault/logs.py @@ -0,0 +1,86 @@ +import logging +import os +import sys +from datetime import datetime +from pathlib import Path + +from batchvault.security import redact, restrict_permissions + +LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR") +DEFAULT_LEVEL = "INFO" +LOGGER_NAME = "batchvault" +ENV_LOG_DIR = "BATCHVAULT_LOG_DIR" +KEEP_LOGS = 20 + + +class RedactingFormatter(logging.Formatter): + """Hides credentials embedded in URLs, including in tracebacks.""" + + def format(self, record): + return redact(super().format(record)) + + +def default_log_dir() -> Path: + """The conventional per-user log folder for the current OS.""" + override = os.environ.get(ENV_LOG_DIR) + if override: + return Path(override).expanduser() + + home = Path.home() + + if sys.platform == "win32": + base = Path(os.environ.get("LOCALAPPDATA") or home / "AppData" / "Local") + return base / "BatchVault" / "Logs" + if sys.platform == "darwin": + return home / "Library" / "Logs" / "BatchVault" + + state = os.environ.get("XDG_STATE_HOME") + return (Path(state) if state else home / ".local" / "state") / "batchvault" + + +def prune_logs(directory: Path, keep: int = KEEP_LOGS) -> None: + """Delete the oldest per-run log files beyond ``keep``.""" + try: + logs = sorted(directory.glob("batchvault-*.log")) + for old in logs[:-keep] if keep else logs: + old.unlink(missing_ok=True) + except OSError: + pass + + +def setup_logging(level=DEFAULT_LEVEL, log_file=None): + """Log to a file for this run. Returns its path, or None if it can't be opened.""" + logger = logging.getLogger(LOGGER_NAME) + logger.setLevel(getattr(logging, level.upper())) + close_logging() + + if log_file: + path = Path(log_file).expanduser() + else: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + path = default_log_dir() / f"batchvault-{stamp}-{os.getpid()}.log" + + try: + path.parent.mkdir(parents=True, exist_ok=True) + handler = logging.FileHandler(path, encoding="utf-8") + except OSError: + logger.addHandler(logging.NullHandler()) + return None + + restrict_permissions(path) + handler.setFormatter( + RedactingFormatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s") + ) + logger.addHandler(handler) + + if not log_file: + prune_logs(path.parent) + return path.resolve() + + +def close_logging() -> None: + logger = logging.getLogger(LOGGER_NAME) + for handler in list(logger.handlers): + handler.close() + logger.removeHandler(handler) + logger.addHandler(logging.NullHandler()) diff --git a/src/batchvault/manifest.py b/src/batchvault/manifest.py new file mode 100644 index 0000000..3cf878a --- /dev/null +++ b/src/batchvault/manifest.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import hashlib +import json +import os +import tempfile +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path + +from batchvault import __version__ +from batchvault.errors import ManifestError +from batchvault.names import validate_folder + +SCHEMA_VERSION = 1 +MANIFEST_NAME = "manifest.json" +_CHUNK = 1024 * 1024 + + +def hash_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(_CHUNK): + digest.update(chunk) + return digest.hexdigest() + + +def _require(data: dict, key: str, kind: type, where: str): + value = data.get(key) + # bool is an int subclass; never accept it as a number. + if not isinstance(value, kind) or isinstance(value, bool) and kind is int: + raise ManifestError(f"{where}: '{key}' is missing or not a {kind.__name__}") + return value + + +def _folder(value, where: str) -> str: + if not isinstance(value, str): + raise ManifestError(f"{where}: 'folder' must be a string") + try: + return validate_folder(value) + except ValueError as exc: + raise ManifestError(f"{where}: {exc}") from exc + + +@dataclass(frozen=True) +class VolumeInfo: + file: str # relative to the manifest folder, with forward slashes + size: int + sha256: str + + @classmethod + def from_dict(cls, data, where: str) -> VolumeInfo: + if not isinstance(data, dict): + raise ManifestError(f"{where}: volume entry must be an object") + return cls( + file=_require(data, "file", str, where), + size=_require(data, "size", int, where), + sha256=_require(data, "sha256", str, where), + ) + + +@dataclass +class ManifestEntry: + name: str + size: int + sha256: str + engine: str + format: str + level: int + volume_size: int + volumes: list[VolumeInfo] = field(default_factory=list) + encrypted: bool = False + mtime_ns: int = 0 # source modification time when archived; 0 if unknown + folder: str = "" # where the file lived, relative to --root ("" = flat) + + @property + def path(self) -> str: + """The file's path relative to the root, with forward slashes.""" + return f"{self.folder}/{self.name}" if self.folder else self.name + + def to_dict(self) -> dict: + return { + "name": self.name, + "size": self.size, + "sha256": self.sha256, + "engine": self.engine, + "format": self.format, + "level": self.level, + "volume_size": self.volume_size, + "encrypted": self.encrypted, + "mtime_ns": self.mtime_ns, + "folder": self.folder, + "volumes": [vars(volume) for volume in self.volumes], + } + + @classmethod + def from_dict(cls, data, index: int) -> ManifestEntry: + where = f"entries[{index}]" + if not isinstance(data, dict): + raise ManifestError(f"{where}: must be an object") + + name = _require(data, "name", str, where) + if not name or "/" in name or "\\" in name or name in (".", ".."): + raise ManifestError(f"{where}: unsafe file name {name!r}") + + raw_volumes = _require(data, "volumes", list, where) + if not raw_volumes: + raise ManifestError(f"{where}: no volumes listed") + + return cls( + name=name, + size=_require(data, "size", int, where), + sha256=_require(data, "sha256", str, where), + engine=_require(data, "engine", str, where), + format=_require(data, "format", str, where), + level=_require(data, "level", int, where), + volume_size=_require(data, "volume_size", int, where), + volumes=[VolumeInfo.from_dict(v, where) for v in raw_volumes], + encrypted=bool(data.get("encrypted", False)), + mtime_ns=int(data.get("mtime_ns", 0) or 0), + folder=_folder(data.get("folder", ""), where), + ) + + def check_volumes(self, base_dir: Path) -> list[str]: + """Problems found with the volumes on disk; empty if all are intact.""" + problems = [] + for volume in self.volumes: + path = base_dir / volume.file + if not path.is_file(): + problems.append(f"missing volume {volume.file}") + elif path.stat().st_size != volume.size: + problems.append(f"volume {volume.file} has the wrong size") + elif hash_file(path) != volume.sha256: + problems.append(f"volume {volume.file} fails its checksum") + return problems + + +@dataclass +class Manifest: + entries: dict[str, ManifestEntry] = field(default_factory=dict) + created: str = "" + + def to_dict(self) -> dict: + return { + "schema_version": SCHEMA_VERSION, + "created": self.created or datetime.now(UTC).isoformat(timespec="seconds"), + "batchvault_version": __version__, + "entries": [entry.to_dict() for entry in self.entries.values()], + } + + @classmethod + def from_dict(cls, data) -> Manifest: + if not isinstance(data, dict): + raise ManifestError("Manifest must be a JSON object") + + version = data.get("schema_version") + if version != SCHEMA_VERSION: + raise ManifestError( + f"Unsupported manifest schema_version {version!r} " + f"(this version reads {SCHEMA_VERSION})." + ) + + entries = {} + for index, raw in enumerate(_require(data, "entries", list, "manifest")): + entry = ManifestEntry.from_dict(raw, index) + if entry.path in entries: + raise ManifestError(f"Duplicate entry for {entry.path!r}") + entries[entry.path] = entry + + return cls(entries=entries, created=str(data.get("created", ""))) + + @classmethod + def from_bytes(cls, raw: bytes, source: Path) -> Manifest: + """Parse manifest bytes already read from ``source``.""" + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ManifestError(f"Manifest {source} is not valid JSON: {exc}") from exc + + try: + return cls.from_dict(data) + except ManifestError as exc: + raise ManifestError(f"{source}: {exc}") from exc + + @classmethod + def load(cls, path: Path) -> Manifest: + try: + raw = path.read_bytes() + except OSError as exc: + raise ManifestError(f"Cannot read manifest {path}: {exc}") from exc + return cls.from_bytes(raw, path) + + def save(self, path: Path) -> None: + """Write atomically so an interrupted run cannot leave a half-written file.""" + path.parent.mkdir(parents=True, exist_ok=True) + text = json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n" + + # A fresh, unpredictable, owner-only file (not a fixed ".tmp" name that + # someone could pre-create as a link); the manifest lists file names. + fd, name = tempfile.mkstemp( + dir=path.parent, prefix=path.name + ".", suffix=".tmp" + ) + temp = Path(name) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + os.replace(temp, path) + except BaseException: + temp.unlink(missing_ok=True) + raise diff --git a/src/batchvault/models.py b/src/batchvault/models.py new file mode 100644 index 0000000..f8c2e60 --- /dev/null +++ b/src/batchvault/models.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + + +class Status(StrEnum): + SUCCESS = "success" + FAILED = "failed" + SKIPPED = "skipped" + PARTIAL = "partial" + + +@dataclass +class FileResult: + """Outcome of one file in a pack, unpack or verify run. + + For ``pack`` the ``source`` is the original file and ``archive`` its first + volume. For ``unpack`` and ``verify`` the ``source`` is the archive (its first + volume) and, for ``unpack``, ``restored`` lists the files written. + """ + + source: Path + status: Status + message: str = "" + size: int = 0 + volumes: list[Path] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + archive: Path | None = None # first volume of the archive (pack) + stored_size: int = 0 # total size of the volumes (pack) + restored: list[Path] = field(default_factory=list) # files written (unpack) + folder: str = "" # folder of the file under the root ("" = flat) + name: str = "" # the original file's name (pack, unpack and verify) + level: int | None = None # compression level used (pack) + + def to_dict(self) -> dict: + return { + "source": str(self.source), + "name": self.name, + "level": self.level, + "folder": self.folder, + "status": self.status.value, + "message": self.message, + "size": self.size, + "volumes": [str(volume) for volume in self.volumes], + "archive": str(self.archive) if self.archive else None, + "stored_size": self.stored_size, + "restored": [str(path) for path in self.restored], + "warnings": self.warnings, + } + + +@dataclass +class RunResult: + """Structured outcome of a whole run.""" + + entries: list[FileResult] = field(default_factory=list) + manifest: Path | None = None + warnings: list[str] = field(default_factory=list) + engine: str = "" + + @property + def status(self) -> Status: + statuses = {entry.status for entry in self.entries} + if not statuses or statuses == {Status.SUCCESS}: + return Status.SUCCESS + if statuses == {Status.SKIPPED}: + return Status.SKIPPED + if Status.SUCCESS in statuses: + return Status.PARTIAL + return Status.FAILED + + def count(self, status: Status) -> int: + return sum(1 for entry in self.entries if entry.status == status) + + def to_dict(self) -> dict: + return { + "status": self.status.value, + "engine": self.engine, + "manifest": str(self.manifest) if self.manifest else None, + "warnings": self.warnings, + "entries": [entry.to_dict() for entry in self.entries], + } + + +@dataclass(frozen=True) +class FileEntry: + """A file found by ``list_files`` or an archive found by ``list_archives``.""" + + path: Path + size: int | None = None + root: Path | None = None # folder a file was listed from (``list files``) + + def to_dict(self) -> dict: + data = {"path": str(self.path), "size": self.size} + if self.root is not None: + # Lets ``pack`` keep the listed folder's own name in the archives. + data["root"] = str(self.root) + return data + + +@dataclass(frozen=True) +class ProgressEvent: + """Progress report. Callbacks run on worker threads and must be thread-safe. + + ``kind`` is ``"plan"`` (once, before work: ``size`` is the total bytes if + known and ``count`` the number of files), then ``"start"`` and ``"finish"`` + for each file. + """ + + kind: str + source: Path + size: int = 0 + status: Status | None = None + count: int = 0 + + def to_dict(self) -> dict: + return { + "event": self.kind, + "source": str(self.source), + "size": self.size, + "status": self.status.value if self.status else None, + "count": self.count, + } + + +@dataclass(frozen=True) +class InspectEntry: + """What the manifest (or, without one, the file names) say about an archive.""" + + name: str + path: Path # first volume + engine: str + format: str | None = None + level: int | None = None + encrypted: bool | None = None + size: int | None = None # original size + stored_size: int | None = None # total size of the volumes + volumes: int = 0 + complete: bool | None = None # every listed volume is present + folder: str = "" # folder under the root the file came from ("" = flat) + + @property + def ratio(self) -> float | None: + if self.size and self.stored_size is not None: + return self.stored_size / self.size + return None + + def to_dict(self) -> dict: + return { + "name": self.name, + "folder": self.folder, + "path": str(self.path), + "engine": self.engine, + "format": self.format, + "level": self.level, + "encrypted": self.encrypted, + "size": self.size, + "stored_size": self.stored_size, + "ratio": self.ratio, + "volumes": self.volumes, + "complete": self.complete, + } diff --git a/src/batchvault/names.py b/src/batchvault/names.py new file mode 100644 index 0000000..36f8fea --- /dev/null +++ b/src/batchvault/names.py @@ -0,0 +1,36 @@ +"""Checks on file and folder names before they are written to disk.""" + +from __future__ import annotations + +_WINDOWS_RESERVED = { + "con", "prn", "aux", "nul", + *(f"com{n}" for n in range(1, 10)), + *(f"lpt{n}" for n in range(1, 10)), +} # fmt: skip +_WINDOWS_BAD_CHARACTERS = '<>:"|?*' + + +def unsafe_on_windows(name: str) -> bool: + """Names Windows treats specially: drives (``C:x``), streams (``a:b``), + devices (``NUL``, ``con.txt``), and names ending in a dot or space.""" + stem = name.split(".")[0].rstrip(" ").lower() + return ( + any(char in name for char in _WINDOWS_BAD_CHARACTERS) + or name != name.rstrip(" .") + or stem in _WINDOWS_RESERVED + ) + + +def validate_folder(folder: str) -> str: + """Check a stored folder path (forward slashes, relative, no ``..``). + + Returns it unchanged; raises ValueError if it could point outside the folder + it is restored into. + """ + if folder == "": + return folder + if "\\" in folder or "\x00" in folder or folder.startswith("/"): + raise ValueError(f"unsafe folder {folder!r}") + if any(part in ("", ".", "..") for part in folder.split("/")): + raise ValueError(f"unsafe folder {folder!r}") + return folder diff --git a/src/batchvault/notify.py b/src/batchvault/notify.py new file mode 100644 index 0000000..9c63523 --- /dev/null +++ b/src/batchvault/notify.py @@ -0,0 +1,73 @@ +"""Webhook notifications. Used by the command-line interface only.""" + +from __future__ import annotations + +import json +import platform +import urllib.error +import urllib.request +from urllib.parse import urlparse + +from batchvault.errors import OptionsError +from batchvault.models import RunResult, Status + +NOTIFY_ON = ("failure", "success", "always") +TIMEOUT_SECONDS = 10 + + +def should_notify(when: str, result: RunResult) -> bool: + if when == "always": + return True + ok = result.status == Status.SUCCESS + return ok if when == "success" else not ok + + +def validate_url(url: str) -> str: + parsed = urlparse(url) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + raise OptionsError("The notification URL must start with http:// or https://") + return url + + +def is_insecure(url: str) -> bool: + """True for plain http:// to anything other than this machine.""" + parsed = urlparse(url) + local = parsed.hostname in ("localhost", "127.0.0.1", "::1") + return parsed.scheme == "http" and not local + + +def build_payload(command: str, result: RunResult) -> dict: + counts = ", ".join( + f"{result.count(status)} {status.value}" + for status in (Status.SUCCESS, Status.SKIPPED, Status.FAILED) + if result.count(status) + ) + text = ( + f"BatchVault {command} on {platform.node()}: " + f"{result.status.value} ({counts or 'nothing to do'})" + ) + # "text" is read by Slack and Teams, "content" by Discord. + return { + "text": text, + "content": text, + "command": command, + "host": platform.node(), + "result": result.to_dict(), + } + + +def send(url: str, payload: dict) -> str | None: + """POST the payload as JSON. Returns an error message, or None on success.""" + request = urllib.request.Request( + validate_url(url), + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=TIMEOUT_SECONDS): + return None + except urllib.error.HTTPError as exc: + return f"the server answered {exc.code}" + except (urllib.error.URLError, OSError, ValueError) as exc: + return str(getattr(exc, "reason", exc)) diff --git a/src/batchvault/options.py b/src/batchvault/options.py new file mode 100644 index 0000000..aa27e34 --- /dev/null +++ b/src/batchvault/options.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from batchvault.errors import OptionsError + +KIB = 1024 +MIB = 1024 * KIB +GIB = 1024 * MIB + +_SIZE_UNITS = {"": 1, "k": KIB, "m": MIB, "g": GIB} +_SIZE_RE = re.compile(r"^\s*(\d+)\s*([kmg]?)b?\s*$", re.IGNORECASE) + + +# Compression formats of the native engine. The 7-Zip engine always writes 7z. +FORMATS = ("xz", "gzip", "bzip2", "store") + + +def parse_size(value: str | int) -> int: + """Convert a size such as ``10m`` or ``1048576`` to bytes.""" + if isinstance(value, int): + size = value + else: + match = _SIZE_RE.match(value) + if not match: + raise OptionsError( + f"Invalid size {value!r}. Use a number with an optional " + "k, m or g suffix, e.g. 500k, 10m, 2g." + ) + size = int(match.group(1)) * _SIZE_UNITS[match.group(2).lower()] + + if size < 1: + raise OptionsError("Size must be at least 1 byte.") + return size + + +def format_size(size: int | None) -> str: + """Human-readable size such as ``12.3 MiB``; ``-`` when unknown.""" + if size is None: + return "-" + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB"): + if value < 1024: + return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} TiB" + + +@dataclass(frozen=True) +class PackOptions: + """Settings for creating an archive.""" + + level: int = 5 # a fixed level; "auto" is resolved per file before this + volume_size: int = 10 * MIB + jobs: int = 0 + force: bool = False + format: str = "xz" + password: str | None = field(default=None, repr=False) + + def __post_init__(self): + if self.format not in FORMATS: + raise OptionsError( + f"Unknown format {self.format!r}. Choose from: {', '.join(FORMATS)}." + ) + if not 0 <= self.level <= 9: + raise OptionsError("Compression level must be between 0 and 9.") + if self.volume_size < 1: + raise OptionsError("Volume size must be at least 1 byte.") + if self.jobs < 0: + raise OptionsError("Jobs cannot be negative.") diff --git a/src/batchvault/py.typed b/src/batchvault/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/src/batchvault/security.py b/src/batchvault/security.py new file mode 100644 index 0000000..84afe0c --- /dev/null +++ b/src/batchvault/security.py @@ -0,0 +1,40 @@ +import os +import re +from pathlib import Path + +# Credentials embedded in a URL: scheme://user:password@host +_URL_CREDENTIALS = re.compile(r"(?<=://)[^/\s:@]+(:[^/\s@]*)?@") + + +def redact(text: str) -> str: + """Hide credentials embedded in URLs.""" + return _URL_CREDENTIALS.sub("***@", text) + + +SECRET_OPTIONS = ("--notify",) + + +def redact_arguments(argv: list[str]) -> list[str]: + """Hide the values of options that carry secrets (webhook URLs contain one).""" + hidden = [] + mask_next = False + for arg in argv: + if mask_next: + hidden.append("***") + mask_next = False + elif arg in SECRET_OPTIONS: + hidden.append(arg) + mask_next = True + elif any(arg.startswith(f"{option}=") for option in SECRET_OPTIONS): + hidden.append(arg.split("=", 1)[0] + "=***") + else: + hidden.append(redact(arg)) + return hidden + + +def restrict_permissions(path: Path) -> None: + """Make a file readable by its owner only (no effect on Windows).""" + try: + os.chmod(path, 0o600) + except OSError: + pass diff --git a/src/batchvault/signing.py b/src/batchvault/signing.py new file mode 100644 index 0000000..84ae693 --- /dev/null +++ b/src/batchvault/signing.py @@ -0,0 +1,152 @@ +"""Ed25519 signatures for manifests (needs ``batchvault[crypto]``). + +The signature is stored next to the manifest as ``.sig`` and covers +the exact bytes of the manifest file. Anyone with the public key can verify it; +only the holder of the private key can create it. +""" + +from __future__ import annotations + +import base64 +import os +from pathlib import Path + +from batchvault.crypto import require_cryptography +from batchvault.errors import InputError, SignatureError + +SIGNATURE_SUFFIX = ".sig" + + +def signature_path(manifest_path: Path) -> Path: + return manifest_path.with_name(manifest_path.name + SIGNATURE_SUFFIX) + + +def _write_private(path: Path, data: bytes) -> None: + # Create the file readable by its owner only, before any key is written. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "wb") as handle: + handle.write(data) + + +def generate_keypair( + private_path: Path, + public_path: Path, + *, + password: str | None = None, + overwrite: bool = False, +) -> None: + """Create an Ed25519 key pair as PEM files. + + The private key is encrypted with ``password`` when one is given. + """ + require_cryptography() + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + for path in (private_path, public_path): + if path.exists() and not overwrite: + raise InputError(f"{path} already exists. Use --force to replace it.") + + key = Ed25519PrivateKey.generate() + encryption = ( + serialization.BestAvailableEncryption(password.encode("utf-8")) + if password + else serialization.NoEncryption() + ) + _write_private( + private_path, + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + encryption, + ), + ) + public_path.write_bytes( + key.public_key().public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + ) + + +def load_private_key(path: Path, password: str | None = None): + require_cryptography() + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + try: + key = serialization.load_pem_private_key( + path.read_bytes(), password.encode("utf-8") if password else None + ) + except OSError as exc: + raise SignatureError(f"Cannot read private key {path}: {exc}") from exc + except (ValueError, TypeError) as exc: + raise SignatureError( + f"Cannot load private key {path} (wrong password or not a key): {exc}" + ) from exc + + if not isinstance(key, Ed25519PrivateKey): + raise SignatureError(f"{path} is not an Ed25519 private key.") + return key + + +def load_public_key(path: Path): + require_cryptography() + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + + try: + key = serialization.load_pem_public_key(path.read_bytes()) + except OSError as exc: + raise SignatureError(f"Cannot read public key {path}: {exc}") from exc + except (ValueError, TypeError) as exc: + raise SignatureError(f"Cannot load public key {path}: {exc}") from exc + + if not isinstance(key, Ed25519PublicKey): + raise SignatureError(f"{path} is not an Ed25519 public key.") + return key + + +def sign_manifest(manifest_path: Path, private_key) -> Path: + """Sign the manifest with a loaded private key; return the signature file.""" + signature = private_key.sign(manifest_path.read_bytes()) + target = signature_path(manifest_path) + target.write_text(base64.b64encode(signature).decode("ascii") + "\n") + return target + + +def verify_manifest_bytes( + data: bytes, manifest_path: Path, public_key_path: Path +) -> None: + """Raise SignatureError unless ``data`` carries a valid signature. + + ``data`` is the manifest content, already read from ``manifest_path``; the + signature file sits next to it. Checking the bytes that are then parsed + leaves no gap in which the file could be swapped. + """ + from cryptography.exceptions import InvalidSignature + + public_key = load_public_key(public_key_path) + sig_file = signature_path(manifest_path) + if not sig_file.is_file(): + raise SignatureError( + f"{manifest_path.name} is not signed (no {sig_file.name})." + ) + + try: + signature = base64.b64decode(sig_file.read_text().strip(), validate=True) + public_key.verify(signature, data) + except (ValueError, InvalidSignature) as exc: + raise SignatureError( + f"Signature check failed for {manifest_path.name}: the manifest was " + "changed or signed with a different key." + ) from exc + + +def verify_manifest_signature(manifest_path: Path, public_key_path: Path) -> None: + """Raise SignatureError unless the manifest has a valid signature.""" + try: + data = manifest_path.read_bytes() + except OSError as exc: + raise SignatureError(f"Cannot read {manifest_path}: {exc}") from exc + verify_manifest_bytes(data, manifest_path, public_key_path) diff --git a/src/batchvault/ui.py b/src/batchvault/ui.py new file mode 100644 index 0000000..154e715 --- /dev/null +++ b/src/batchvault/ui.py @@ -0,0 +1,298 @@ +"""Rich terminal output. Only the command-line interface imports this module.""" + +from __future__ import annotations + +import threading +from pathlib import Path + +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel +from rich.progress import ( + BarColumn, + Progress, + ProgressColumn, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, +) +from rich.table import Table +from rich.text import Text + +from batchvault.doctor import Check +from batchvault.eta import EtaEstimator, format_duration +from batchvault.models import InspectEntry, ProgressEvent, RunResult, Status +from batchvault.options import format_size + +STATUS_STYLE = { + Status.SUCCESS: ("OK", "green"), + Status.SKIPPED: ("SKIPPED", "yellow"), + Status.FAILED: ("FAILED", "bold red"), + Status.PARTIAL: ("PARTIAL", "yellow"), +} + + +class EtaColumn(ProgressColumn): + """Shows the run's own ETA for tasks created with an ``estimator`` field.""" + + def render(self, task): + estimator = task.fields.get("estimator") + if estimator is None: + return Text("") + + remaining = estimator.estimate(task.completed, task.total or 0) + if remaining is None: + return Text("ETA estimating…", style="dim") + return Text(f"ETA {format_duration(remaining)}", style="progress.remaining") + + +def status_text(status: Status) -> Text: + label, style = STATUS_STYLE[status] + return Text(label, style=style) + + +class RunProgress: + """Progress callback that draws an overall bar and one line per active file. + + Use as a context manager and pass the instance as ``progress=``. With + ``enabled=False`` it does nothing. + """ + + def __init__(self, console: Console, label: str, enabled: bool = True): + self._enabled = enabled + self._label = label + self._lock = threading.Lock() + self._eta = EtaEstimator() + self._by_bytes = False + self._overall = None + self._tasks: dict[Path, int] = {} + self._progress = Progress( + SpinnerColumn(), + TextColumn("[bold cyan]{task.description}"), + BarColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + EtaColumn(), + console=console, + transient=True, + disable=not enabled, + ) + + def __enter__(self): + if self._enabled: + self._progress.start() + return self + + def __exit__(self, *exc_info): + self._progress.stop() + + def pause(self) -> None: + """Stop drawing, e.g. before asking the user a question.""" + self._progress.stop() + + def __call__(self, event: ProgressEvent) -> None: + if not self._enabled: + return + + with self._lock: + if event.kind == "plan": + self._by_bytes = event.size > 0 + total = event.size if self._by_bytes else event.count + self._overall = self._progress.add_task( + self._label, total=total, estimator=self._eta + ) + self._eta.start() + elif event.kind == "start": + task = self._progress.add_task(escape(event.source.name), total=None) + self._tasks[event.source] = task + elif event.kind == "finish": + self._finish(event) + + def _finish(self, event: ProgressEvent) -> None: + task = self._tasks.pop(event.source, None) + if task is not None: + self._progress.remove_task(task) + if self._overall is not None: + self._progress.advance( + self._overall, event.size if self._by_bytes else 1 + ) + + label, style = STATUS_STYLE.get(event.status, ("", "")) + self._progress.console.print( + Text.assemble((f"{label:<8}", style), event.source.name) + ) + + +def print_run_info(console: Console, title: str, rows: list[tuple[str, str]]) -> None: + info = Table.grid(padding=(0, 2)) + info.add_column(style="bold cyan") + info.add_column() + for key, value in rows: + info.add_row(key, Text(value)) + console.print(Panel(info, title=title, border_style="cyan")) + + +# Terms used on screen: a SOURCE is an original file; an ARCHIVE is the set of +# volumes made from it. Each table names its columns accordingly. +SOURCE_STYLE = "cyan" +ARCHIVE_STYLE = "magenta" + +SUMMARY_TITLES = { + "pack": "Pack: source files -> archives", + "unpack": "Unpack: archives -> restored files", + "verify": "Verify: archives", +} + + +def _shown(folder: str, name: str) -> str: + return f"{folder}/{name}" if folder else name + + +def _detail_text(entry) -> Text: + style = "dim" if entry.status == Status.SUCCESS else "" + text = Text(entry.message, style=style) + for warning in entry.warnings: + text.append(f"\n{warning}", style="yellow") + return text + + +def print_summary( + console: Console, result: RunResult, title: str | None = None, kind: str = "pack" +) -> None: + """Print one row per file. ``kind`` is pack, unpack or verify.""" + table = Table(title=title or SUMMARY_TITLES.get(kind, ""), show_lines=False) + + # The same columns as `inspect`: the original file, its archive, the outcome + # and an explanation. + table.add_column("Source", style=SOURCE_STYLE, overflow="fold") + table.add_column("Archive", style=ARCHIVE_STYLE, overflow="fold") + table.add_column("Status") + table.add_column("Details", overflow="fold") + + for entry in result.entries: + # pack: the source is the original file and the archive its first volume. + # unpack and verify: the "source" is the archive's first volume. + if kind == "pack": + archive = entry.archive.name if entry.archive else "-" + original = entry.name or entry.source.name + else: + archive = entry.source.name + original = entry.name or "-" + table.add_row( + Text(_shown(entry.folder, original)), + Text(archive), + status_text(entry.status), + _detail_text(entry), + ) + console.print(table) + + for warning in result.warnings: + console.print(Text(f"Warning: {warning}", style="yellow")) + + counts = ", ".join( + f"{result.count(status)} {label.lower()}" + for status, (label, _) in STATUS_STYLE.items() + if status != Status.PARTIAL and result.count(status) + ) + console.print(f"[bold]Summary:[/bold] {escape(counts or 'nothing to do')}") + if result.manifest and any(e.status == Status.SUCCESS for e in result.entries): + console.print(f"Manifest: {escape(str(result.manifest))}") + + +def _archive_status(entry: InspectEntry) -> Text: + if entry.complete is True: + return Text("OK", style="green") + if entry.complete is False: + return Text("MISSING", style="bold red") + return Text("-", style="dim") # no manifest, so completeness is unknown + + +def _inspect_details(entry: InspectEntry) -> Text: + notes = [] + if entry.encrypted: + notes.append(("encrypted", "yellow")) + if entry.complete is False: + notes.append(("some volumes are missing", "bold red")) + if entry.complete is None: + notes.append(("no manifest: sizes and checks unknown", "dim")) + + text = Text() + for index, (note, style) in enumerate(notes): + if index: + text.append(", ") + text.append(note, style=style) + return text + + +def print_inspect( + console: Console, entries: list[InspectEntry], source: str | None = None +) -> None: + """Describe archives with the same layout and colours as the result tables.""" + original = sum(e.size or 0 for e in entries) + stored = sum(e.stored_size or 0 for e in entries) + missing = sum(1 for e in entries if e.complete is False) + encrypted = sum(1 for e in entries if e.encrypted) + + rows = [] + if source: + rows.append(("Archives", source)) + rows.append(("Files", str(len(entries)))) + if original: + sizes = f"{format_size(original)} -> {format_size(stored)}" + rows.append(("Size", f"{sizes} ({stored / original:.0%})")) + else: + rows.append(("Stored size", format_size(stored))) + if encrypted: + rows.append(("Encrypted", f"{encrypted} of {len(entries)}")) + print_run_info(console, "BatchVault inspect", rows) + + table = Table(title="Inspect: archives", show_lines=False) + table.add_column("Source", style=SOURCE_STYLE, overflow="fold") + table.add_column("Archive", style=ARCHIVE_STYLE, overflow="fold") + table.add_column("Status") + table.add_column("Engine") + table.add_column("Format") + table.add_column("Level", justify="right") + table.add_column("Original", justify="right") + table.add_column("Stored", justify="right") + table.add_column("Ratio", justify="right") + table.add_column("Volumes", justify="right") + table.add_column("Details", overflow="fold") + + for entry in entries: + ratio = f"{entry.ratio:.0%}" if entry.ratio is not None else "-" + table.add_row( + Text(_shown(entry.folder, entry.name)), + Text(entry.path.name), + _archive_status(entry), + entry.engine, + entry.format or "-", + "-" if entry.level is None else str(entry.level), + format_size(entry.size), + format_size(entry.stored_size), + ratio, + str(entry.volumes), + _inspect_details(entry), + ) + console.print(table) + + summary = f"{len(entries)} archive{'' if len(entries) == 1 else 's'}" + if missing: + summary += f", {missing} with missing volumes" + console.print(f"[bold]Summary:[/bold] {escape(summary)}") + + +def print_checks(console: Console, checks: list[Check]) -> None: + table = Table(title="BatchVault doctor") + table.add_column("Check") + table.add_column("Result") + table.add_column("Details", overflow="fold") + + for check in checks: + if check.ok: + result = Text("OK", style="green") + else: + result = Text("FAILED" if check.required else "MISSING", style="red") + table.add_row(check.name, result, Text(check.detail)) + console.print(table) diff --git a/src/batchvault/volumes.py b/src/batchvault/volumes.py new file mode 100644 index 0000000..5161dc0 --- /dev/null +++ b/src/batchvault/volumes.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import io +import os +import re +import shutil +import time +from pathlib import Path + +from batchvault.errors import VolumeError +from batchvault.security import restrict_permissions + +VOLUME_RE = re.compile(r"^(?P.+\.(?:7z|bv))\.(?P\d{3,})$", re.IGNORECASE) + + +def split_volume_name(name: str) -> tuple[str, int] | None: + """Return ``(base, number)`` for ``intro.mp4.7z.003``, or None if not a volume.""" + match = VOLUME_RE.match(name) + if not match: + return None + return match.group("base"), int(match.group("number")) + + +def existing_volumes(directory: Path, base: str) -> list[Path]: + """Volumes of ``base`` already present in ``directory``, in volume order.""" + if not directory.is_dir(): + return [] + + found = [] + for path in directory.iterdir(): + parsed = split_volume_name(path.name) + if path.is_file() and parsed and parsed[0].casefold() == base.casefold(): + found.append((parsed[1], path)) + return [path for _, path in sorted(found)] + + +def find_volumes(path: Path) -> list[Path]: + """Find every volume of the set that ``path`` belongs to. + + Any volume of the set may be given. Raises VolumeError if the path is not a + volume or if a volume inside the numbered range is missing. A missing + *trailing* volume cannot be detected without a manifest. + """ + parsed = split_volume_name(path.name) + if parsed is None: + raise VolumeError(f"Not an archive volume: {path}") + + volumes = existing_volumes(path.parent, parsed[0]) + numbers = [split_volume_name(volume.name)[1] for volume in volumes] + + missing = sorted(set(range(1, max(numbers) + 1)) - set(numbers)) + if missing: + listed = ", ".join(f"{number:03d}" for number in missing) + raise VolumeError( + f"Missing volume(s) {listed} for {parsed[0]} in {path.parent}" + ) + + return volumes + + +STAGING_PREFIX = ".batchvault-" + + +def cleanup_staging(root: Path, max_age: float = 86400.0) -> int: + """Remove staging folders left behind by a killed run; return how many. + + Looks through the whole tree under ``root``. Only folders untouched for + ``max_age`` seconds are removed, so a run in progress is never disturbed. + """ + if not root.is_dir(): + return 0 + + removed = 0 + now = time.time() + for current, dirnames, _ in os.walk(root): # does not follow links + for name in list(dirnames): + if not name.startswith(STAGING_PREFIX): + continue + dirnames.remove(name) # never look inside a staging folder + item = Path(current) / name + try: + if not item.is_symlink() and now - item.stat().st_mtime > max_age: + shutil.rmtree(item) + removed += 1 + except OSError: + continue + return removed + + +def install_volumes(built: list[Path], dest_dir: Path, existing: list[Path]): + """Move a finished volume set into ``dest_dir``, replacing ``existing``. + + Old volumes are removed only here, after the new set was built completely. + """ + for old in existing: + old.unlink() + + installed = [] + for volume in built: + # Archives may hold private data; do not let umask make them readable by + # other users (no effect on Windows, where folder permissions apply). + restrict_permissions(volume) + target = dest_dir / volume.name + os.replace(volume, target) + installed.append(target) + return installed + + +class VolumeWriter: + """Write-only file object that rolls over to a new volume every N bytes.""" + + def __init__(self, directory: Path, base: str, volume_size: int): + self._directory = directory + self._base = base + self._volume_size = volume_size + self._file = None + self._room = 0 + self.volumes: list[Path] = [] + + def write(self, data) -> int: + view = memoryview(data) + total = len(view) + + while len(view): + if self._room == 0: + self._next_volume() + chunk = view[: self._room] + self._file.write(chunk) + self._room -= len(chunk) + view = view[len(chunk) :] + + return total + + def _next_volume(self) -> None: + self._close_current() + number = len(self.volumes) + 1 + path = self._directory / f"{self._base}.{number:03d}" + self._file = path.open("wb") + self.volumes.append(path) + self._room = self._volume_size + + def _close_current(self) -> None: + if self._file is not None: + self._file.close() + self._file = None + + def close(self) -> None: + self._close_current() + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close() + + +class VolumeReader(io.RawIOBase): + """Read-only file object that reads a volume set as one continuous stream.""" + + def __init__(self, paths): + super().__init__() + self._paths = list(paths) + self._index = 0 + self._file = None + + def readable(self) -> bool: + return True + + def readinto(self, buffer) -> int: + view = memoryview(buffer).cast("B") + if not len(view): + return 0 + + while True: + if self._file is None: + if self._index >= len(self._paths): + return 0 + self._file = self._paths[self._index].open("rb") + self._index += 1 + + count = self._file.readinto(view) + if count: + return count + + self._file.close() + self._file = None + + def close(self) -> None: + if self._file is not None: + self._file.close() + self._file = None + super().close() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..97501c0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +import pytest + + +@pytest.fixture(autouse=True) +def isolated_log_dir(tmp_path_factory, monkeypatch): + """Keep test runs out of the user's real log folder.""" + monkeypatch.setenv("BATCHVAULT_LOG_DIR", str(tmp_path_factory.mktemp("logs"))) + + +@pytest.fixture(autouse=True) +def fast_key_derivation(monkeypatch): + """Use a cheap password hash so encryption tests stay quick. + + The cost is stored in each archive's header, so archives still open; one test + checks that the real default is strong. + """ + from batchvault import crypto + + monkeypatch.setattr(crypto, "DEFAULT_LOG2_N", 14) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py new file mode 100644 index 0000000..d4fab0b --- /dev/null +++ b/tests/e2e/conftest.py @@ -0,0 +1,38 @@ +"""Shared setup for the end-to-end tests. + +Everything under tests/e2e is marked ``e2e`` automatically, so the normal unit +test run (``-m "not integration and not e2e"``) skips it. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +HERE = Path(__file__).parent +REPO = HERE.parent.parent + +sys.path.insert(0, str(HERE / "scripts")) + +from make_data import make_data # noqa: E402 + + +def pytest_collection_modifyitems(items): + for item in items: + if HERE in Path(str(item.path)).parents: + item.add_marker(pytest.mark.e2e) + + +@pytest.fixture +def repo_root() -> Path: + return REPO + + +@pytest.fixture +def data_dir(tmp_path) -> Path: + """A tree of test files; the folder is returned.""" + root = tmp_path / "data" + make_data(root) + return root diff --git a/tests/e2e/scripts/compare.py b/tests/e2e/scripts/compare.py new file mode 100644 index 0000000..562bc2d --- /dev/null +++ b/tests/e2e/scripts/compare.py @@ -0,0 +1,48 @@ +"""Check that restored files are byte-for-byte identical to the originals. + + python tests/e2e/scripts/compare.py SOURCE_TREE RESTORED_FOLDER [NAME ...] + +Every file under SOURCE_TREE (or only the NAMEs given) must exist in +RESTORED_FOLDER at the same relative path with identical content. Exits 1 on any +difference. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def main(argv: list[str]) -> int: + source, restored = Path(argv[0]), Path(argv[1]) + wanted = set(argv[2:]) + + originals = [ + path + for path in sorted(source.rglob("*")) + if path.is_file() and (not wanted or path.name in wanted) + ] + if not originals: + print(f"No files to compare under {source}") + return 1 + + problems = 0 + for original in originals: + # Folders are kept, so a file comes back at the same path below the folder. + relative = original.relative_to(source) + copy = restored / relative + if not copy.is_file(): + print(f"MISSING {copy}") + problems += 1 + elif copy.read_bytes() != original.read_bytes(): + print(f"DIFFERS {copy}") + problems += 1 + else: + print(f"OK {relative.as_posix()}") + + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.stdout.reconfigure(encoding="utf-8") + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/e2e/scripts/make_data.py b/tests/e2e/scripts/make_data.py new file mode 100644 index 0000000..726353e --- /dev/null +++ b/tests/e2e/scripts/make_data.py @@ -0,0 +1,37 @@ +"""Create a small tree of test files: binary, text, Unicode names, spaces, empty. + +Used by the end-to-end tests and by the GitHub Action workflow: + + python tests/e2e/scripts/make_data.py data +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + + +def make_data(root: Path) -> list[Path]: + root.mkdir(parents=True, exist_ok=True) + + contents = { + "report.bin": os.urandom(2 * 1024 * 1024), + "notes.txt": ("hello world\n" * 5000).encode("utf-8"), + "ünï cödé.txt": ("Unicode ✓ 日本語\n" * 1000).encode("utf-8"), + "empty.dat": b"", + "sub dir/nested file.csv": b"a,b,c\n" * 2000, + } + + paths = [] + for name, data in contents.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + paths.append(path) + return paths + + +if __name__ == "__main__": + for created in make_data(Path(sys.argv[1])): + print(created) diff --git a/tests/e2e/test_cli_e2e.py b/tests/e2e/test_cli_e2e.py new file mode 100644 index 0000000..4823b37 --- /dev/null +++ b/tests/e2e/test_cli_e2e.py @@ -0,0 +1,335 @@ +"""End-to-end tests of the installed ``batchvault`` command. + +Each test runs the real program in a separate process, the way a user would: +real arguments, real files, real exit codes, real standard output. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from batchvault.engines.sevenzip import SevenZipEngine + +try: + import cryptography # noqa: F401 + + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +needs_crypto = pytest.mark.skipif(not HAS_CRYPTO, reason="cryptography not installed") +ENGINES = [ + "native", + pytest.param( + "7zip", + marks=pytest.mark.skipif( + not SevenZipEngine().is_available(), reason="7-Zip is not installed" + ), + ), +] + + +def _command() -> list[str]: + script = shutil.which("batchvault") + return [script] if script else [sys.executable, "-m", "batchvault"] + + +@pytest.fixture +def cli(tmp_path): + """Run batchvault in ``tmp_path``; returns the finished process.""" + env = {k: v for k, v in os.environ.items() if not k.startswith("BATCHVAULT_")} + env.update( + PYTHONUTF8="1", + NO_COLOR="1", + COLUMNS="200", + BATCHVAULT_LOG_DIR=str(tmp_path / "logs"), + ) + + def run(*args, cwd=None, input=None, extra_env=None): + return subprocess.run( + [*_command(), *map(str, args)], + cwd=cwd or tmp_path, + env={**env, **(extra_env or {})}, + input=input, + capture_output=True, + text=True, + encoding="utf-8", + check=False, + ) + + return run + + +def code(process) -> int: + return process.returncode + + +def assert_restored(data_dir: Path, restored: Path, names: set[str] | None = None): + """Every original is back at the same path below ``restored``.""" + checked = 0 + for original in data_dir.rglob("*"): + if original.is_file() and (names is None or original.name in names): + copy = restored / original.relative_to(data_dir) + assert copy.read_bytes() == original.read_bytes() + checked += 1 + assert checked + + +def manifest_entries(archives: Path) -> list[dict]: + text = (archives / "manifest.json").read_text(encoding="utf-8") + return json.loads(text)["entries"] + + +def test_version_help_and_module_entry_point(cli): + version = cli("--version") + assert code(version) == 0 and "batchvault" in version.stdout + + for command in ("pack", "unpack", "verify", "inspect", "list", "doctor"): + assert code(cli(command, "--help")) == 0 + + module = subprocess.run( + [sys.executable, "-m", "batchvault", "--version"], + capture_output=True, + text=True, + check=False, + ) + assert code(module) == 0 and module.stdout == version.stdout + + +@pytest.mark.parametrize("engine", ENGINES) +def test_full_journey(cli, tmp_path, data_dir, engine): + listing = tmp_path / "files.json" + assert code(cli("list", "files", data_dir, "-o", listing)) == 0 + assert len(json.loads(listing.read_text(encoding="utf-8"))) == 5 + + packed = cli( + "pack", "--list", listing, "-o", "archives", "--engine", engine, + "--volume-size", "256k", "--verify", "--json", + ) # fmt: skip + assert code(packed) == 0, packed.stderr + result = json.loads(packed.stdout) + assert result["status"] == "success" + assert result["engine"] == engine + volumes = {Path(e["source"]).name: e["volumes"] for e in result["entries"]} + assert len(volumes["report.bin"]) > 1 + + inspected = json.loads(cli("inspect", "archives", "--json").stdout) + assert len(inspected) == 5 and all(e["complete"] for e in inspected) + + checked = cli("verify", "archives", "--against", data_dir, "--json") + assert code(checked) == 0, checked.stdout + assert json.loads(checked.stdout)["status"] == "success" + + # The folder that was listed ("data") and the sub folder inside it are kept. + assert (tmp_path / "archives/data/report.bin").is_dir() + assert (tmp_path / "archives/data/sub dir/nested file.csv").is_dir() + + restored = tmp_path / "restored" + assert code(cli("unpack", "archives", "-o", restored, "-q")) == 0 + assert_restored(data_dir, restored / "data") + + +@pytest.mark.parametrize("engine", ENGINES) +def test_update_only_repacks_what_changed(cli, tmp_path, data_dir, engine): + notes = data_dir / "notes.txt" + base = [ + "pack", notes, data_dir / "report.bin", "-o", "a", + "--engine", engine, "--json", + ] # fmt: skip + assert code(cli(*base)) == 0 + assert code(cli(*base)) == 1 # already there: skipped + + unchanged = json.loads(cli(*base, "--update").stdout) + assert {e["message"] for e in unchanged["entries"]} == {"Up to date."} + + notes.write_text("changed\n" * 1000, encoding="utf-8") + later = notes.stat().st_mtime + 30 + os.utime(notes, (later, later)) + + changed = json.loads(cli(*base, "--update").stdout) + statuses = {Path(e["source"]).name: e["status"] for e in changed["entries"]} + assert statuses == {"notes.txt": "success", "report.bin": "skipped"} + + assert code(cli("unpack", "a", "-o", "restored", "-q")) == 0 + assert (tmp_path / "restored" / "notes.txt").read_bytes() == notes.read_bytes() + + +def test_pipeline_through_standard_input(cli, data_dir): + listed = cli("list", "files", data_dir, "--exclude", "*.dat") + assert code(listed) == 0 + + packed = cli( + "pack", "--list", "-", "-o", "a", "--engine", "native", "--json", + input=listed.stdout, + ) # fmt: skip + assert code(packed) == 0, packed.stderr + names = {Path(e["source"]).name for e in json.loads(packed.stdout)["entries"]} + assert names == {"report.bin", "notes.txt", "ünï cödé.txt", "nested file.csv"} + + +def test_json_events_are_one_object_per_line(cli, data_dir): + run = cli( + "pack", data_dir / "notes.txt", "-o", "a", "--engine", "native", + "--json-events", + ) # fmt: skip + assert code(run) == 0 + events = [json.loads(line) for line in run.stdout.splitlines()] + assert [e["event"] for e in events] == ["plan", "start", "finish", "result"] + + +def test_exit_codes(cli, tmp_path, data_dir): + notes = data_dir / "notes.txt" + pack = ["pack", notes, "-o", "a", "--engine", "native", "-q"] + assert code(cli(*pack)) == 0 + assert code(cli(*pack)) == 1 + assert code(cli("pack", tmp_path / "nope.txt", "-o", "b", "-q")) == 2 + assert code(cli("pack", "-o", "b", "-q")) == 2 + assert code(cli("unpack", tmp_path / "nowhere", "-q")) == 2 + assert code(cli("not-a-command")) == 2 + + +@pytest.mark.parametrize("engine", ENGINES) +def test_damaged_archives_are_never_restored(cli, tmp_path, data_dir, engine): + packed = cli( + "pack", data_dir / "report.bin", "-o", "a", "--engine", engine, + "--volume-size", "256k", "-q", + ) # fmt: skip + assert code(packed) == 0 + + volume = sorted((tmp_path / "a" / "report.bin").glob("*.00*"))[2] + damaged = bytearray(volume.read_bytes()) + damaged[len(damaged) // 2] ^= 0xFF + volume.write_bytes(bytes(damaged)) + + assert code(cli("verify", "a", "-q")) == 1 + assert code(cli("unpack", "a", "-o", "restored", "-q")) == 1 + assert not (tmp_path / "restored" / "report.bin").exists() + + +def test_restore_from_one_volume_without_a_manifest(cli, tmp_path, data_dir): + packed = cli( + "pack", data_dir / "report.bin", "-o", "a", "--engine", "native", + "--volume-size", "256k", "-q", + ) # fmt: skip + assert code(packed) == 0 + (tmp_path / "a" / "manifest.json").unlink() + + volume = sorted((tmp_path / "a" / "report.bin").glob("*.bv.*"))[3] + restored = cli("unpack", volume, "-o", "restored", "--json") + assert code(restored) == 0 + assert json.loads(restored.stdout)["warnings"] # checksums not verified + assert_restored(data_dir, tmp_path / "restored", {"report.bin"}) + + +def test_settings_come_from_a_config_file(cli, tmp_path, data_dir): + (tmp_path / "batchvault.toml").write_text( + '[pack]\nengine = "native"\nlevel = 7\nvolume-size = "64k"\n', encoding="utf-8" + ) + assert code(cli("pack", data_dir / "report.bin", "-o", "a", "-q")) == 0 + + entry = manifest_entries(tmp_path / "a")[0] + assert entry["level"] == 7 and len(entry["volumes"]) > 1 + + override = cli("pack", data_dir / "notes.txt", "-o", "b", "--level", "2", "-q") + assert code(override) == 0 + assert manifest_entries(tmp_path / "b")[0]["level"] == 2 # the command line wins + + +def test_many_files_in_parallel(cli, tmp_path): + folder = tmp_path / "many" + folder.mkdir() + for number in range(12): + (folder / f"file {number:02}.bin").write_bytes(os.urandom(40_000)) + + files = tmp_path / "many.txt" + assert code(cli("list", "files", folder, "-o", files)) == 0 + + packed = cli( + "pack", "--list", files, "-o", "a", "--engine", "native", "-j", "4", + "--volume-size", "16k", "--verify", "-q", + ) # fmt: skip + assert code(packed) == 0 + assert code(cli("verify", "a", "--against", folder, "-q")) == 0 + assert code(cli("unpack", "a", "-o", "restored", "-q", "-j", "4")) == 0 + assert_restored(folder, tmp_path / "restored" / "many") + + +def test_a_log_file_is_written(cli, tmp_path, data_dir): + run = cli("pack", data_dir / "notes.txt", "-o", "a", "--engine", "native", "-q") + assert code(run) == 0 + + logs = list((tmp_path / "logs").glob("batchvault-*.log")) + assert len(logs) == 1 + assert "notes.txt" in logs[0].read_text(encoding="utf-8") + + +def test_delete_source_only_after_proof(cli, tmp_path, data_dir): + notes = data_dir / "notes.txt" + original = notes.read_bytes() + + base = ["pack", notes, "-o", "a", "--engine", "native", "--delete-source", "-q"] + assert code(cli(*base)) == 0 + assert notes.exists() # no terminal and no --yes: nothing deleted + + assert code(cli(*base, "--yes", "--force")) == 0 + assert not notes.exists() + + assert code(cli("unpack", "a", "-o", "restored", "-q")) == 0 + assert (tmp_path / "restored" / "notes.txt").read_bytes() == original + + +def test_doctor(cli): + report = cli("doctor", "--json") + assert code(report) == 0 + assert {c["name"] for c in json.loads(report.stdout)} >= {"Python", "Log folder"} + + +@needs_crypto +def test_encrypted_and_signed_archives(cli, tmp_path, data_dir): + notes = data_dir / "notes.txt" + password = {"BATCHVAULT_PASSWORD": "correct horse"} + wrong = {"BATCHVAULT_PASSWORD": "wrong"} + + keygen = cli("keygen", "--private", "k.key", "--public", "k.pub", "-q") + assert code(keygen) == 0 + + packed = cli( + "pack", notes, "-o", "a", "--engine", "native", "--encrypt", + "--sign-key", "k.key", "-q", extra_env=password, + ) # fmt: skip + assert code(packed) == 0 + + # Missing, wrong and right password. + assert code(cli("verify", "a", "-q")) == 1 + assert code(cli("verify", "a", "-q", extra_env=wrong)) == 1 + signed = cli("verify", "a", "--public-key", "k.pub", "-q", extra_env=password) + assert code(signed) == 0 + + # The contents are not readable in the volumes. + volumes = list((tmp_path / "a" / "notes.txt").glob("*.bv.*")) + assert b"hello world" not in b"".join(v.read_bytes() for v in volumes) + + restored = cli( + "unpack", "a", "-o", "restored", "--public-key", "k.pub", "-q", + extra_env=password, + ) # fmt: skip + assert code(restored) == 0 + assert (tmp_path / "restored" / "notes.txt").read_bytes() == notes.read_bytes() + + # A changed manifest is refused before anything is restored. + manifest = tmp_path / "a" / "manifest.json" + text = manifest.read_text(encoding="utf-8") + manifest.write_text(text.replace("notes.txt", "other.txt", 1), encoding="utf-8") + refused = cli( + "unpack", "a", "-o", "again", "--public-key", "k.pub", "-q", + extra_env=password, + ) # fmt: skip + assert code(refused) == 2 + assert not (tmp_path / "again").exists() diff --git a/tests/e2e/test_docker_e2e.py b/tests/e2e/test_docker_e2e.py new file mode 100644 index 0000000..1f03d0a --- /dev/null +++ b/tests/e2e/test_docker_e2e.py @@ -0,0 +1,199 @@ +"""End-to-end tests of the Docker images. + +The images are built from the Dockerfile in this repository, so the tests cover +exactly what would be published. Set ``BATCHVAULT_E2E_IMAGE`` (and +``BATCHVAULT_E2E_IMAGE_7ZIP``) to test images that are already built or pulled. +The tests are skipped when Docker is not available. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +from batchvault import __version__ + +pytestmark = pytest.mark.docker + +REPO = Path(__file__).resolve().parents[2] + + +def _docker_available() -> bool: + if not shutil.which("docker"): + return False + probe = subprocess.run(["docker", "info"], capture_output=True, check=False) + return probe.returncode == 0 + + +if not _docker_available(): + pytest.skip("Docker is not available", allow_module_level=True) + + +def _build(tag: str, *build_args: str) -> str: + command = ["docker", "build", "--quiet", "-t", tag] + for arg in build_args: + command += ["--build-arg", arg] + process = subprocess.run( + [*command, str(REPO)], capture_output=True, text=True, check=False + ) + assert process.returncode == 0, process.stderr + return tag + + +@pytest.fixture(scope="module") +def image() -> str: + return os.environ.get("BATCHVAULT_E2E_IMAGE") or _build("batchvault-e2e:native") + + +@pytest.fixture(scope="module") +def image_7zip() -> str: + return os.environ.get("BATCHVAULT_E2E_IMAGE_7ZIP") or _build( + "batchvault-e2e:7zip", "WITH_7ZIP=true" + ) + + +@pytest.fixture +def docker(tmp_path): + """Run a container with ``tmp_path`` mounted at /data.""" + + def run(image, *args, env=None, entrypoint=None, mount=True): + command = ["docker", "run", "--rm"] + if mount: + command += ["-v", f"{tmp_path}:/data"] + if hasattr(os, "getuid"): # keep file ownership on Linux hosts + command += ["--user", f"{os.getuid()}:{os.getgid()}"] + for key, value in (env or {}).items(): + command += ["-e", f"{key}={value}"] + if entrypoint: + command += ["--entrypoint", entrypoint] + command += [image, *map(str, args)] + return subprocess.run( + command, capture_output=True, text=True, encoding="utf-8", check=False + ) + + return run + + +def write_files(folder: Path) -> dict[str, bytes]: + folder.mkdir(parents=True, exist_ok=True) + files = { + "report.bin": os.urandom(300_000), + "notes.txt": b"hello world\n" * 2000, + "ünï cödé.txt": "Unicode ✓ 日本語\n".encode() * 100, + } + for name, data in files.items(): + (folder / name).write_bytes(data) + return files + + +def test_version_and_help(docker, image): + version = docker(image, "--version", mount=False) + assert version.returncode == 0 and __version__ in version.stdout + + default = docker(image, mount=False) # the image's default command is --help + assert default.returncode == 0 and "usage: batchvault" in default.stdout + assert docker(image, "pack", "--help", mount=False).returncode == 0 + assert docker(image, "nonsense", mount=False).returncode == 2 + + +def test_runs_as_a_non_root_user_by_default(image): + process = subprocess.run( + ["docker", "run", "--rm", "--entrypoint", "id", image, "-u"], + capture_output=True, + text=True, + check=False, + ) + assert process.stdout.strip() == "1000" + + +def test_pack_and_unpack_in_a_container(docker, image, tmp_path): + files = write_files(tmp_path / "input") + + packed = docker( + image, "pack", *[f"input/{name}" for name in files], "-o", "archives", + "--volume-size", "64k", "--engine", "native", "--verify", "--json", + "--no-log", + ) # fmt: skip + assert packed.returncode == 0, packed.stderr + result = json.loads(packed.stdout) + assert result["status"] == "success" and result["engine"] == "native" + assert (tmp_path / "archives" / "manifest.json").is_file() + + restored = docker(image, "unpack", "archives", "-o", "restored", "-q", "--no-log") + assert restored.returncode == 0, restored.stderr + for name, data in files.items(): + assert (tmp_path / "restored" / name).read_bytes() == data + + +def test_exit_code_reaches_the_host(docker, image, tmp_path): + write_files(tmp_path / "input") + args = ["pack", "input/notes.txt", "-o", "a", "--engine", "native", "-q"] + assert docker(image, *args, "--no-log").returncode == 0 + assert docker(image, *args, "--no-log").returncode == 1 # skipped + assert docker(image, "unpack", "missing", "-q", "--no-log").returncode == 2 + + +def test_list_files_prints_json(docker, image, tmp_path): + write_files(tmp_path / "input") + listing = docker(image, "list", "files", "input", "--no-log") + assert listing.returncode == 0 + assert len(json.loads(listing.stdout)) == 3 + + +def test_encryption_is_available_in_the_image(docker, image, tmp_path): + files = write_files(tmp_path / "input") + env = {"BATCHVAULT_PASSWORD": "correct horse"} + + packed = docker( + image, "pack", "input/notes.txt", "-o", "vault", "--engine", "native", + "--encrypt", "-q", "--no-log", env=env, + ) # fmt: skip + assert packed.returncode == 0, packed.stderr + + assert docker(image, "verify", "vault", "-q", "--no-log").returncode == 1 + restored = docker(image, "unpack", "vault", "-o", "out", "-q", "--no-log", env=env) + assert restored.returncode == 0, restored.stderr + assert (tmp_path / "out" / "notes.txt").read_bytes() == files["notes.txt"] + + +def test_config_file_in_the_mounted_folder_is_used(docker, image, tmp_path): + write_files(tmp_path / "input") + (tmp_path / "batchvault.toml").write_text( + '[pack]\nengine = "native"\nlevel = 8\n', encoding="utf-8" + ) + + packed = docker(image, "pack", "input/notes.txt", "-o", "a", "-q", "--no-log") + assert packed.returncode == 0, packed.stderr + + manifest = json.loads((tmp_path / "a" / "manifest.json").read_text("utf-8")) + assert manifest["entries"][0]["level"] == 8 + + +def test_native_image_has_no_7zip(docker, image): + report = json.loads(docker(image, "doctor", "--json", "--no-log").stdout) + sevenzip = next(c for c in report if c["name"].startswith("7-Zip")) + assert sevenzip["ok"] is False + + +def test_7zip_image_uses_7zip(docker, image_7zip, tmp_path): + files = write_files(tmp_path / "input") + + report = json.loads(docker(image_7zip, "doctor", "--json", "--no-log").stdout) + assert next(c for c in report if c["name"].startswith("7-Zip"))["ok"] is True + + packed = docker( + image_7zip, "pack", *[f"input/{name}" for name in files], "-o", "a", + "--volume-size", "64k", "--json", "--no-log", + ) # fmt: skip + assert packed.returncode == 0, packed.stderr + assert json.loads(packed.stdout)["engine"] == "7zip" # auto picks 7-Zip + + restored = docker(image_7zip, "unpack", "a", "-o", "out", "-q", "--no-log") + assert restored.returncode == 0, restored.stderr + for name, data in files.items(): + assert (tmp_path / "out" / name).read_bytes() == data diff --git a/tests/e2e/test_package_e2e.py b/tests/e2e/test_package_e2e.py new file mode 100644 index 0000000..b36b461 --- /dev/null +++ b/tests/e2e/test_package_e2e.py @@ -0,0 +1,240 @@ +"""End-to-end tests of the built package, installed into a clean virtual env. + +These catch problems the normal tests cannot: files missing from the wheel, +a broken entry point, an undeclared dependency, or the optional ``crypto`` +extra not behaving as documented. They need network access (pip) and take a +minute or so. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import time +import zipfile +from pathlib import Path + +import pytest + +from batchvault import __version__ + +pytestmark = pytest.mark.package + +REPO = Path(__file__).resolve().parents[2] + + +def run(command, **kwargs) -> subprocess.CompletedProcess: + return subprocess.run( + [str(part) for part in command], + capture_output=True, + text=True, + encoding="utf-8", + check=False, + **kwargs, + ) + + +def checked(command, **kwargs) -> subprocess.CompletedProcess: + process = run(command, **kwargs) + assert process.returncode == 0, f"{command}\n{process.stdout}\n{process.stderr}" + return process + + +def venv_python(venv: Path) -> Path: + folder = "Scripts" if sys.platform == "win32" else "bin" + return venv / folder / ("python.exe" if sys.platform == "win32" else "python") + + +def venv_script(venv: Path, name: str) -> Path: + folder = "Scripts" if sys.platform == "win32" else "bin" + return venv / folder / (f"{name}.exe" if sys.platform == "win32" else name) + + +@pytest.fixture(scope="module") +def wheel(tmp_path_factory) -> Path: + out = tmp_path_factory.mktemp("wheel") + uv = shutil.which("uv") + if uv: + checked([uv, "build", "--wheel", "--out-dir", out], cwd=REPO) + else: + checked( + [sys.executable, "-m", "pip", "wheel", ".", "--no-deps", "-w", out], + cwd=REPO, + ) + wheels = list(out.glob("batchvault-*.whl")) + assert len(wheels) == 1 + return wheels[0] + + +def make_venv(tmp_path_factory, wheel: Path, extra: str = "") -> Path: + venv = tmp_path_factory.mktemp("venv") + uv = shutil.which("uv") + + # uv is quicker and more reliable than the pip bundled with a new venv. + if uv: + checked([uv, "venv", "--quiet", "--python", sys.executable, venv]) + install = [uv, "pip", "install", "--quiet", "--python", venv_python(venv)] + else: + checked([sys.executable, "-m", "venv", venv]) + install = [ + venv_python(venv), "-m", "pip", "install", "--quiet", + "--disable-pip-version-check", + ] # fmt: skip + command = [*install, f"{wheel}{extra}"] + + # Downloading dependencies can fail for reasons that have nothing to do with + # the package (a flaky connection or proxy), so try a few times. + for attempt in range(1, 3): + if run(command).returncode == 0: + return venv + time.sleep(3 * attempt) + checked(command) # last try; shows the real error if it still fails + return venv + + +@pytest.fixture(scope="module") +def plain_venv(tmp_path_factory, wheel) -> Path: + return make_venv(tmp_path_factory, wheel) + + +@pytest.fixture(scope="module") +def crypto_venv(tmp_path_factory, wheel) -> Path: + return make_venv(tmp_path_factory, wheel, "[crypto]") + + +def test_wheel_contents(wheel): + with zipfile.ZipFile(wheel) as archive: + names = set(archive.namelist()) + + assert "batchvault/py.typed" in names + for module in ("api", "cli", "ui", "engines/native", "engines/sevenzip"): + assert f"batchvault/{module}.py" in names + assert not [n for n in names if n.startswith(("tests/", ".temp/", "docs/"))] + + +def test_metadata_and_entry_point(plain_venv): + script = ( + "import importlib.metadata as md, json\n" + "dist = md.distribution('batchvault')\n" + "print(json.dumps({\n" + " 'version': dist.version,\n" + " 'python': dist.metadata['Requires-Python'],\n" + " 'requires': dist.requires,\n" + " 'scripts': [e.name for e in dist.entry_points\n" + " if e.group == 'console_scripts'],\n" + "}))\n" + ) + info = json.loads(checked([venv_python(plain_venv), "-c", script]).stdout) + + assert info["version"] == __version__ + assert info["python"] == ">=3.11.4" + assert info["scripts"] == ["batchvault"] + requires = " ".join(info["requires"]) + assert "rich" in requires and "pathspec" in requires + + # cryptography is only required through the optional "crypto" extra. + crypto = [r for r in info["requires"] if r.startswith("cryptography")] + assert len(crypto) == 1 and "extra" in crypto[0] and "crypto" in crypto[0] + + +def test_installed_command_and_module(plain_venv): + command = checked([venv_script(plain_venv, "batchvault"), "--version"]) + assert __version__ in command.stdout + + module = checked([venv_python(plain_venv), "-m", "batchvault", "--version"]) + assert module.stdout == command.stdout + + +API_SCRIPT = """ +import json, os, sys +from pathlib import Path +import batchvault as bv + +work = Path(sys.argv[1]) +source = work / "données 1.bin" +source.write_bytes(os.urandom(300_000)) # incompressible, so it needs many volumes + +packed = bv.pack([source], work / "a", engine="native", volume_size="64k") +unpacked = bv.unpack(work / "a", work / "out") +checked = bv.verify(work / "a", against=work) + +print(json.dumps({ + "packed": packed.status.value, + "unpacked": unpacked.status.value, + "verified": checked.status.value, + "same": (work / "out" / source.name).read_bytes() == source.read_bytes(), + "volumes": len(packed.entries[0].volumes), + "rich_imported": "rich" in sys.modules, +})) +""" + + +def test_python_api_in_a_clean_environment(plain_venv, tmp_path): + process = checked([venv_python(plain_venv), "-c", API_SCRIPT, tmp_path]) + result = json.loads(process.stdout) + + assert result["packed"] == result["unpacked"] == result["verified"] == "success" + assert result["same"] and result["volumes"] > 1 + # The library must not pull in the terminal UI. + assert result["rich_imported"] is False + + +def test_missing_crypto_extra_gives_a_clear_message(plain_venv, tmp_path): + source = tmp_path / "secret.txt" + source.write_text("secret") + + script = ( + "import sys, batchvault as bv\n" + "try:\n" + " bv.pack([sys.argv[1]], sys.argv[2], engine='native', password='x')\n" + "except bv.EngineNotFoundError as exc:\n" + " print(exc)\n" + ) + message = checked( + [venv_python(plain_venv), "-c", script, source, tmp_path / "a"] + ).stdout + assert "batchvault[crypto]" in message + assert not (tmp_path / "a" / "secret.txt").exists() + + command = [ + venv_script(plain_venv, "batchvault"), + "pack", + source, + "-o", + tmp_path / "b", + "--engine", + "native", + "--encrypt", + "--no-log", + ] + cli = run(command, env={**os.environ, "BATCHVAULT_PASSWORD": "x"}) + assert cli.returncode == 2 + assert "batchvault[crypto]" in cli.stderr + + +def test_crypto_extra_enables_encryption_and_signing(crypto_venv, tmp_path): + script = ( + "import sys, json\n" + "from pathlib import Path\n" + "import batchvault as bv\n" + "from batchvault import signing\n" + "work = Path(sys.argv[1])\n" + "source = work / 'notes.txt'\n" + "source.write_text('hello ' * 1000)\n" + "signing.generate_keypair(work / 'k.key', work / 'k.pub')\n" + "bv.pack([source], work / 'a', engine='native', password='pw',\n" + " sign_key=work / 'k.key')\n" + "good = bv.unpack(work / 'a', work / 'out', password='pw',\n" + " public_key=work / 'k.pub')\n" + "bad = bv.unpack(work / 'a', work / 'out2', password='nope')\n" + "print(json.dumps({'good': good.status.value, 'bad': bad.status.value,\n" + " 'same': (work / 'out' / 'notes.txt').read_text()\n" + " == source.read_text()}))\n" + ) + result = json.loads( + checked([venv_python(crypto_venv), "-c", script, tmp_path]).stdout + ) + assert result == {"good": "success", "bad": "failed", "same": True} diff --git a/tests/integration/test_sevenzip_api.py b/tests/integration/test_sevenzip_api.py new file mode 100644 index 0000000..8292b1d --- /dev/null +++ b/tests/integration/test_sevenzip_api.py @@ -0,0 +1,154 @@ +"""End-to-end runs through the public API and CLI with a real 7-Zip.""" + +import json +import os + +import pytest + +import batchvault as bv +from batchvault.cli import main +from batchvault.engines.sevenzip import SevenZipEngine + +pytestmark = pytest.mark.integration + +if not SevenZipEngine().is_available(): + pytest.skip("7-Zip is not installed", allow_module_level=True) + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "données 1.bin").write_bytes(os.urandom(20_000)) + (folder / "notes.txt").write_text("hello " * 2000, encoding="utf-8") + return folder + + +def pack(media, tmp_path, **kwargs): + kwargs.setdefault("volume_size", "4k") + return bv.pack( + [media / "données 1.bin", media / "notes.txt"], + tmp_path / "archives", + engine="7zip", + **kwargs, + ) + + +def test_pack_unpack_verify(tmp_path, media): + result = pack(media, tmp_path, jobs=2) + assert result.status == bv.Status.SUCCESS + assert result.engine == "7zip" + + manifest = json.loads(result.manifest.read_text(encoding="utf-8")) + assert {e["engine"] for e in manifest["entries"]} == {"7zip"} + assert {e["format"] for e in manifest["entries"]} == {"7z"} + + assert bv.verify(tmp_path / "archives", against=media).status == bv.Status.SUCCESS + + restored = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert restored.status == bv.Status.SUCCESS + for name in ("données 1.bin", "notes.txt"): + assert (tmp_path / "out" / name).read_bytes() == (media / name).read_bytes() + + +def test_unpack_from_any_volume_without_manifest(tmp_path, media): + pack(media, tmp_path) + archives = tmp_path / "archives" + (archives / "manifest.json").unlink() + + volume = sorted((archives / "données 1.bin").glob("*.7z.*"))[2] + result = bv.unpack(volume, tmp_path / "out") + assert result.status == bv.Status.SUCCESS + assert result.warnings + assert (tmp_path / "out" / "données 1.bin").read_bytes() == ( + media / "données 1.bin" + ).read_bytes() + + +def test_missing_last_volume_is_caught(tmp_path, media): + pack(media, tmp_path) + folder = tmp_path / "archives" / "données 1.bin" + sorted(folder.glob("*.7z.*"))[-1].unlink() + + assert bv.verify(tmp_path / "archives").count(bv.Status.FAILED) == 1 + + +def test_update_only_repacks_changed_files(tmp_path, media): + pack(media, tmp_path) + assert pack(media, tmp_path, update=True).count(bv.Status.SKIPPED) == 2 + + (media / "notes.txt").write_text("changed " * 3000, encoding="utf-8") + later = (media / "notes.txt").stat().st_mtime + 30 + os.utime(media / "notes.txt", (later, later)) + + result = pack(media, tmp_path, update=True) + assert {e.source.name: e.status for e in result.entries} == { + "données 1.bin": bv.Status.SKIPPED, + "notes.txt": bv.Status.SUCCESS, + } + + +def test_encrypted_roundtrip(tmp_path, media): + pack(media, tmp_path, password="s3cret", verify=True) + archives = tmp_path / "archives" + + assert all(e.encrypted for e in bv.inspect(archives)) + assert bv.verify(archives).count(bv.Status.FAILED) == 2 # no password + assert bv.verify(archives, password="wrong").count(bv.Status.FAILED) == 2 + assert bv.verify(archives, password="s3cret").status == bv.Status.SUCCESS + + bv.unpack(archives, tmp_path / "out", password="s3cret") + assert (tmp_path / "out" / "notes.txt").read_bytes() == ( + media / "notes.txt" + ).read_bytes() + + +def test_delete_source_after_roundtrip(tmp_path, media): + original = (media / "notes.txt").read_bytes() + bv.pack( + [media / "notes.txt"], tmp_path / "a", engine="7zip", delete_source=True + ) + assert not (media / "notes.txt").exists() + + bv.unpack(tmp_path / "a", tmp_path / "out") + assert (tmp_path / "out" / "notes.txt").read_bytes() == original + + +def test_native_and_7zip_archives_in_one_manifest(tmp_path, media): + archives = tmp_path / "archives" + bv.pack([media / "notes.txt"], archives, engine="native") + bv.pack([media / "données 1.bin"], archives, engine="7zip", volume_size="4k") + + engines = {e.name: e.engine for e in bv.inspect(archives)} + assert engines == {"notes.txt": "native", "données 1.bin": "7zip"} + + assert bv.unpack(archives, tmp_path / "out").status == bv.Status.SUCCESS + + +def test_cli_with_7zip(tmp_path, media, capsys): + code = main([ + "pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "7zip", "--volume-size", "4k", "--json", "-q", "--no-log", + ]) # fmt: skip + assert code == 0 + assert json.loads(capsys.readouterr().out)["engine"] == "7zip" + + assert main(["unpack", str(tmp_path / "a"), "-o", str(tmp_path / "o"), "-q", + "--no-log"]) == 0 # fmt: skip + + +def test_folder_tree_is_kept_with_7zip(tmp_path): + tree = tmp_path / "tree" + (tree / "Resources" / "intro").mkdir(parents=True) + first = tree / "Resources" / "intro" / "notes.txt" + second = tree / "Resources" / "notes.txt" # same name, other folder + first.write_text("intro " * 500) + second.write_text("resources " * 500) + + result = bv.pack([first, second], tmp_path / "a", engine="7zip", root=tree) + assert result.status == bv.Status.SUCCESS + assert (tmp_path / "a/Resources/intro/notes.txt/notes.txt.7z.001").is_file() + + bv.unpack(tmp_path / "a", tmp_path / "out") + assert (tmp_path / "out/Resources/intro/notes.txt").read_text() == first.read_text() + assert (tmp_path / "out/Resources/notes.txt").read_text() == second.read_text() diff --git a/tests/integration/test_sevenzip_roundtrip.py b/tests/integration/test_sevenzip_roundtrip.py new file mode 100644 index 0000000..c02f0c7 --- /dev/null +++ b/tests/integration/test_sevenzip_roundtrip.py @@ -0,0 +1,66 @@ +import os + +import pytest + +from batchvault.engines.sevenzip import SevenZipEngine +from batchvault.errors import EngineError, OutputExistsError +from batchvault.options import PackOptions + +pytestmark = pytest.mark.integration + +engine = SevenZipEngine() + +if not engine.is_available(): + pytest.skip("7-Zip is not installed", allow_module_level=True) + + +@pytest.fixture +def source(tmp_path): + path = tmp_path / "données 1.bin" + path.write_bytes(os.urandom(20_000)) + return path + + +def test_roundtrip_with_split_volumes(tmp_path, source): + archives = tmp_path / "archives" + volumes = engine.pack(source, archives, PackOptions(volume_size=4096, jobs=2)) + + assert len(volumes) > 1 + assert volumes[0].name == "données 1.bin.7z.001" + + engine.test(volumes[0]) + + restored = engine.unpack(volumes[2], tmp_path / "restored") + assert [p.name for p in restored] == [source.name] + assert restored[0].read_bytes() == source.read_bytes() + + +def test_existing_archive_is_not_replaced_without_force(tmp_path, source): + archives = tmp_path / "archives" + engine.pack(source, archives, PackOptions()) + + with pytest.raises(OutputExistsError): + engine.pack(source, archives, PackOptions()) + + engine.pack(source, archives, PackOptions(force=True)) + + +def test_unpack_does_not_overwrite_without_force(tmp_path, source): + volumes = engine.pack(source, tmp_path / "archives", PackOptions()) + output = tmp_path / "restored" + engine.unpack(volumes[0], output) + + with pytest.raises(OutputExistsError): + engine.unpack(volumes[0], output) + + engine.unpack(volumes[0], output, force=True) + + +def test_corrupt_volume_fails_test(tmp_path, source): + volumes = engine.pack(source, tmp_path / "archives", PackOptions()) + data = bytearray(volumes[0].read_bytes()) + data[len(data) // 2] ^= 0xFF + volumes[0].write_bytes(bytes(data)) + + with pytest.raises(EngineError): + engine.test(volumes[0]) diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..3647d8f --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,249 @@ +import json +import os + +import pytest + +import batchvault as bv +from batchvault import Status +from batchvault.errors import InputError, ManifestError + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "données 1.bin").write_bytes(os.urandom(5000)) + (folder / "notes.txt").write_text("hello " * 500, encoding="utf-8") + (folder / "skip.log").write_text("log") + return folder + + +def pack(media, tmp_path, **kwargs): + kwargs.setdefault("volume_size", "1k") + kwargs.setdefault("engine", "native") + return bv.pack( + [media / "données 1.bin", media / "notes.txt"], tmp_path / "archives", **kwargs + ) + + +def test_pack_writes_volumes_and_manifest(tmp_path, media): + result = pack(media, tmp_path, jobs=2) + + assert result.status == Status.SUCCESS + assert result.manifest == (tmp_path / "archives" / "manifest.json").resolve() + + manifest = json.loads(result.manifest.read_text(encoding="utf-8")) + entries = {e["name"]: e for e in manifest["entries"]} + assert set(entries) == {"données 1.bin", "notes.txt"} + big = entries["données 1.bin"] + assert big["engine"] == "native" and big["format"] == "xz" + assert len(big["volumes"]) > 1 + assert big["volumes"][0]["file"] == "données 1.bin/données 1.bin.bv.001" + + +def test_existing_archives_are_skipped_then_forced(tmp_path, media): + pack(media, tmp_path) + + again = pack(media, tmp_path) + assert again.status == Status.SKIPPED + assert again.count(Status.SKIPPED) == 2 + + assert pack(media, tmp_path, force=True).status == Status.SUCCESS + + +def test_manifest_is_merged_across_runs(tmp_path, media): + archives = tmp_path / "archives" + bv.pack([media / "notes.txt"], archives, engine="native") + bv.pack([media / "données 1.bin"], archives, engine="native") + + manifest = json.loads((archives / "manifest.json").read_text(encoding="utf-8")) + assert len(manifest["entries"]) == 2 + + +def test_failed_file_does_not_stop_others(tmp_path, media): + source = media / "notes.txt" + result = bv.pack([source, media / "données 1.bin"], tmp_path / "a", engine="native") + assert result.status == Status.SUCCESS + + # A directory in place of an output folder makes just that file fail. + (tmp_path / "b").mkdir() + (tmp_path / "b" / "notes.txt").write_text("blocks the output folder") + result = bv.pack([source, media / "données 1.bin"], tmp_path / "b", engine="native") + assert result.status == Status.PARTIAL + assert result.count(Status.FAILED) == 1 + + +@pytest.mark.parametrize("how", ["manifest", "folder", "volume", "list"]) +def test_unpack_sources(tmp_path, media, how): + pack(media, tmp_path) + archives = tmp_path / "archives" + + if how == "manifest": + source = archives / "manifest.json" + elif how == "folder": + source = archives + elif how == "volume": + source = archives / "notes.txt" / "notes.txt.bv.001" + else: + source = tmp_path / "unarchive.txt" + source.write_text("archives/notes.txt/notes.txt.bv.001\n", encoding="utf-8") + + result = bv.unpack(source, tmp_path / "restored") + assert result.status == Status.SUCCESS + assert (tmp_path / "restored" / "notes.txt").read_text( + encoding="utf-8" + ) == (media / "notes.txt").read_text(encoding="utf-8") + assert not [p for p in (tmp_path / "restored").iterdir() if p.name.startswith(".")] + + +def test_unpack_one_subfolder_only_restores_that_archive(tmp_path, media): + pack(media, tmp_path) + result = bv.unpack(tmp_path / "archives" / "notes.txt", tmp_path / "restored") + + assert [e.source.name for e in result.entries] == ["notes.txt.bv.001"] + assert not (tmp_path / "restored" / "données 1.bin").exists() + + +def test_unpack_restores_all_content(tmp_path, media): + pack(media, tmp_path, jobs=3) + result = bv.unpack(tmp_path / "archives", tmp_path / "restored") + + assert result.status == Status.SUCCESS + for name in ("données 1.bin", "notes.txt"): + restored = (tmp_path / "restored" / name).read_bytes() + assert restored == (media / name).read_bytes() + + +def test_unpack_skips_existing_unless_forced(tmp_path, media): + pack(media, tmp_path) + source = tmp_path / "archives" + bv.unpack(source, tmp_path / "restored") + + assert bv.unpack(source, tmp_path / "restored").status == Status.SKIPPED + assert bv.unpack(source, tmp_path / "restored", force=True).status == Status.SUCCESS + + +def test_corrupt_volume_fails_manifest_check(tmp_path, media): + pack(media, tmp_path) + volume = tmp_path / "archives" / "données 1.bin" / "données 1.bin.bv.002" + data = bytearray(volume.read_bytes()) + data[10] ^= 0xFF + volume.write_bytes(bytes(data)) + + result = bv.unpack(tmp_path / "archives", tmp_path / "restored") + assert result.status == Status.PARTIAL + failed = next(e for e in result.entries if e.status == Status.FAILED) + assert "checksum" in failed.message + assert not (tmp_path / "restored" / "données 1.bin").exists() + + +def test_missing_last_volume_is_caught_by_manifest(tmp_path, media): + pack(media, tmp_path) + folder = tmp_path / "archives" / "données 1.bin" + sorted(folder.glob("*.bv.*"))[-1].unlink() + + result = bv.verify(tmp_path / "archives" / "manifest.json") + assert result.count(Status.FAILED) == 1 + assert "missing volume" in next( + e.message for e in result.entries if e.status == Status.FAILED + ) + + +def test_unpack_without_manifest_warns(tmp_path, media): + pack(media, tmp_path) + (tmp_path / "archives" / "manifest.json").unlink() + + result = bv.unpack(tmp_path / "archives", tmp_path / "restored") + assert result.status == Status.SUCCESS + assert result.warnings + + +def test_verify_against_originals(tmp_path, media): + pack(media, tmp_path) + source = tmp_path / "archives" + + assert bv.verify(source, against=media).status == Status.SUCCESS + + (media / "notes.txt").write_text("changed") + result = bv.verify(source, against=media) + assert result.count(Status.FAILED) == 1 + + +def test_damaged_manifest_stops_pack_before_any_work(tmp_path, media): + archives = tmp_path / "archives" + archives.mkdir() + (archives / "manifest.json").write_text("{broken", encoding="utf-8") + + with pytest.raises(ManifestError): + bv.pack([media / "notes.txt"], archives, engine="native") + assert not (archives / "notes.txt").exists() + + +def test_progress_events(tmp_path, media): + events = [] + pack(media, tmp_path, progress=events.append) + kinds = sorted(e.kind for e in events) + assert kinds == ["finish", "finish", "plan", "start", "start"] + plan = next(e for e in events if e.kind == "plan") + assert plan.count == 2 + assert plan.size == sum( + (media / name).stat().st_size for name in ("données 1.bin", "notes.txt") + ) + + +def test_list_files_with_filters(media): + found = bv.list_files(media, exclude=["*.log"]) + assert [f.path.name for f in found] == ["données 1.bin", "notes.txt"] + assert found[1].size == (media / "notes.txt").stat().st_size + assert [f.path.name for f in bv.list_files(media, include=["*.log"])] == [ + "skip.log" + ] + + +def test_list_archives(tmp_path, media): + pack(media, tmp_path) + found = bv.list_archives(tmp_path / "archives") + assert sorted(f.path.name for f in found) == [ + "données 1.bin.bv.001", + "notes.txt.bv.001", + ] + assert bv.list_archives(media) == [] + + +def test_unknown_engine(tmp_path, media): + with pytest.raises(InputError, match="Unknown engine"): + bv.pack([media / "notes.txt"], tmp_path / "a", engine="zip") + + +def test_result_is_json_serialisable(tmp_path, media): + result = pack(media, tmp_path) + assert json.loads(json.dumps(result.to_dict()))["status"] == "success" + + +def test_dry_run_writes_nothing(tmp_path, media): + result = pack(media, tmp_path, dry_run=True, level=5) + + assert result.status == Status.SUCCESS + assert [e.message for e in result.entries] == ["Would archive."] * 2 + assert result.manifest is None + assert not (tmp_path / "archives").exists() + + +def test_dry_run_reports_existing_archives(tmp_path, media): + pack(media, tmp_path) + result = pack(media, tmp_path, dry_run=True) + assert result.count(Status.SKIPPED) == 2 + assert pack(media, tmp_path, dry_run=True, force=True).status == Status.SUCCESS + + +def test_verify_against_finds_originals_in_subfolders(tmp_path): + tree = tmp_path / "tree" + (tree / "sub dir").mkdir(parents=True) + nested = tree / "sub dir" / "nested.txt" + nested.write_text("nested " * 100) + + bv.pack([nested], tmp_path / "a", engine="native") + assert bv.verify(tmp_path / "a", against=tree).status == Status.SUCCESS + + nested.write_text("changed") + assert bv.verify(tmp_path / "a", against=tree).status == Status.FAILED diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..868b7a3 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,170 @@ +import json +import os + +import pytest + +from batchvault import __version__ +from batchvault.cli import main + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "données 1.bin").write_bytes(os.urandom(3000)) + (folder / "notes.txt").write_text("hello " * 200, encoding="utf-8") + return folder + + +def run(*args): + return main([*args, "--no-log"]) + + +def test_version(capsys): + with pytest.raises(SystemExit) as exc: + main(["--version"]) + assert exc.value.code == 0 + assert __version__ in capsys.readouterr().out + + +def test_command_is_required(): + with pytest.raises(SystemExit) as exc: + main([]) + assert exc.value.code == 2 + + +def test_pack_json_output_and_exit_code(tmp_path, media, capsys): + code = run( + "pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--json", "-q", + ) # fmt: skip + assert code == 0 + + result = json.loads(capsys.readouterr().out) + assert result["status"] == "success" + assert result["engine"] == "native" + + +def test_pack_skipped_exits_1(tmp_path, media): + args = ["pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), "-q"] + assert run(*args, "--engine", "native") == 0 + assert run(*args, "--engine", "native") == 1 + + +def test_pack_without_inputs_is_a_usage_error(tmp_path): + assert run("pack", "-o", str(tmp_path / "a")) == 2 + + +def test_pack_missing_file_exits_2(tmp_path): + assert run("pack", str(tmp_path / "nope.txt"), "-o", str(tmp_path / "a")) == 2 + + +def test_pack_from_list_file_then_unpack_and_verify(tmp_path, media): + listing = tmp_path / "list.txt" + listing.write_text(f"# files\n{media / 'données 1.bin'}\n", encoding="utf-8") + archives = tmp_path / "archives" + + assert run("pack", "--list", str(listing), "-o", str(archives), "--engine", + "native", "--volume-size", "1k", "-q") == 0 # fmt: skip + assert run("verify", str(archives), "--against", str(media), "-q") == 0 + assert run("unpack", str(archives), "-o", str(tmp_path / "out"), "-q") == 0 + + restored = (tmp_path / "out" / "données 1.bin").read_bytes() + assert restored == (media / "données 1.bin").read_bytes() + + +def test_pack_include_exclude(tmp_path, media, capsys): + code = run( + "pack", str(media / "notes.txt"), str(media / "données 1.bin"), + "-o", str(tmp_path / "a"), "--engine", "native", + "--exclude", "*.bin", "--json", "-q", + ) # fmt: skip + assert code == 0 + entries = json.loads(capsys.readouterr().out)["entries"] + assert [e["source"].endswith("notes.txt") for e in entries] == [True] + + +def test_strict_turns_warning_into_error(tmp_path, media): + code = run( + "pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--format", "bzip2", "--level", "0", + "--strict", "-q", + ) # fmt: skip + assert code == 2 + + +def test_unpack_missing_source_exits_2(tmp_path): + assert run("unpack", str(tmp_path / "nope"), "-q") == 2 + + +def test_list_files_prints_json(media, capsys): + assert run("list", "files", str(media), "--exclude", "*.bin") == 0 + listing = json.loads(capsys.readouterr().out) + assert [item["path"].endswith("notes.txt") for item in listing] == [True] + + +def test_list_files_to_text_and_json_files(tmp_path, media): + assert run("list", "files", str(media), "-o", str(tmp_path / "l.txt"), "-q") == 0 + lines = (tmp_path / "l.txt").read_text(encoding="utf-8").splitlines() + assert lines[0].startswith("# batchvault-root:") + assert len([line for line in lines if not line.startswith("#")]) == 2 + + assert run("list", "files", str(media), "-o", str(tmp_path / "l.json"), "-q") == 0 + assert len(json.loads((tmp_path / "l.json").read_text(encoding="utf-8"))) == 2 + + +def test_list_files_output_feeds_pack(tmp_path, media): + listing = tmp_path / "l.json" + run("list", "files", str(media), "-o", str(listing), "-q") + assert run("pack", "--list", str(listing), "-o", str(tmp_path / "a"), + "--engine", "native", "-q") == 0 # fmt: skip + + +def test_list_archives(tmp_path, media, capsys): + run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q") # fmt: skip + capsys.readouterr() + + assert run("list", "archives", str(tmp_path / "a")) == 0 + listing = json.loads(capsys.readouterr().out) + assert listing[0]["path"].endswith("notes.txt.bv.001") + + +def test_list_files_warns_on_archive_folder(tmp_path, media, capsys): + run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q") # fmt: skip + capsys.readouterr() + + run("list", "files", str(tmp_path / "a")) + assert "list archives" in capsys.readouterr().err + + +def test_list_archives_warns_on_plain_folder(media, capsys): + run("list", "archives", str(media)) + captured = capsys.readouterr() + assert json.loads(captured.out) == [] + assert "list files" in captured.err + + +def test_doctor_json(capsys): + assert run("doctor", "--json") == 0 + checks = {c["name"]: c for c in json.loads(capsys.readouterr().out)} + assert checks["Python"]["ok"] + + +def test_log_file_is_written(tmp_path, media): + log = tmp_path / "run.log" + code = main(["pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q", "--log-file", str(log)]) # fmt: skip + assert code == 0 + assert "notes.txt" in log.read_text(encoding="utf-8") + + +def test_pack_dry_run(tmp_path, media, capsys): + code = run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--dry-run", "--level", "5", "--json") # fmt: skip + assert code == 0 + assert json.loads(capsys.readouterr().out)["entries"][0]["message"] == ( + "Would archive." + ) + assert not (tmp_path / "a").exists() diff --git a/tests/test_cli_features.py b/tests/test_cli_features.py new file mode 100644 index 0000000..01b84d8 --- /dev/null +++ b/tests/test_cli_features.py @@ -0,0 +1,177 @@ +import json +import os +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from batchvault.cli import main + +try: + import cryptography # noqa: F401 + + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +needs_crypto = pytest.mark.skipif(not HAS_CRYPTO, reason="cryptography not installed") + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "notes.txt").write_text("hello " * 300, encoding="utf-8") + (folder / "data.bin").write_bytes(os.urandom(3000)) + return folder + + +def run(*args): + return main([*args, "--no-log"]) + + +def pack_args(media, tmp_path, *extra): + return ["pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q", *extra] # fmt: skip + + +def test_json_events_stream(tmp_path, media, capsys): + code = run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--json-events") # fmt: skip + assert code == 0 + + events = [json.loads(line) for line in capsys.readouterr().out.splitlines()] + assert [e["event"] for e in events] == ["plan", "start", "finish", "result"] + assert events[-1]["status"] == "success" + assert events[2]["status"] == "success" + + +def test_json_and_json_events_conflict(tmp_path, media): + assert run(*pack_args(media, tmp_path, "--json", "--json-events")) == 2 + + +def test_inspect_command(tmp_path, media, capsys): + run(*pack_args(media, tmp_path)) + capsys.readouterr() + + assert run("inspect", str(tmp_path / "a"), "--json") == 0 + entries = json.loads(capsys.readouterr().out) + assert entries[0]["name"] == "notes.txt" and entries[0]["complete"] is True + + assert run("inspect", str(tmp_path / "a")) == 0 + + +def test_unpack_and_pack_update_flags(tmp_path, media, capsys): + run(*pack_args(media, tmp_path)) + assert run(*pack_args(media, tmp_path, "--update")) == 1 # everything skipped + + assert run("unpack", str(tmp_path / "a"), "-o", str(tmp_path / "o"), "-q") == 0 + assert run("unpack", str(tmp_path / "a"), "-o", str(tmp_path / "o"), "-q", + "--update") == 1 # already up to date # fmt: skip + + +def test_delete_source_needs_an_explicit_yes(tmp_path, media): + assert run(*pack_args(media, tmp_path, "--delete-source")) == 0 + assert (media / "notes.txt").exists() # not a terminal, no --yes: kept + + assert run(*pack_args(media, tmp_path, "--delete-source", "--yes", "--force")) == 0 + assert not (media / "notes.txt").exists() + + +def test_delete_source_dry_run_deletes_nothing(tmp_path, media): + extra = ("--delete-source", "--yes", "--dry-run") + assert run(*pack_args(media, tmp_path, *extra)) == 0 + assert (media / "notes.txt").exists() + + +@needs_crypto +def test_encrypt_flow_with_password_from_environment(tmp_path, media, monkeypatch): + monkeypatch.setenv("BATCHVAULT_PASSWORD", "pw") + assert run(*pack_args(media, tmp_path, "--encrypt")) == 0 + + out = str(tmp_path / "o") + monkeypatch.delenv("BATCHVAULT_PASSWORD") + assert run("unpack", str(tmp_path / "a"), "-o", out, "-q") == 1 + + monkeypatch.setenv("BATCHVAULT_PASSWORD", "wrong") + assert run("unpack", str(tmp_path / "a"), "-o", out, "-q") == 1 + + password_file = tmp_path / "pw.txt" + password_file.write_text("pw\n") + assert run("unpack", str(tmp_path / "a"), "-o", out, "-q", + "--password-file", str(password_file)) == 0 # fmt: skip + assert (tmp_path / "o" / "notes.txt").exists() + + +def test_encrypt_without_a_password_is_a_usage_error(tmp_path, media, monkeypatch): + monkeypatch.delenv("BATCHVAULT_PASSWORD", raising=False) + assert run(*pack_args(media, tmp_path, "--encrypt")) == 2 + + +@needs_crypto +def test_keygen_sign_and_verify_with_public_key(tmp_path, media): + private, public = str(tmp_path / "k.key"), str(tmp_path / "k.pub") + assert run("keygen", "--private", private, "--public", public, "-q") == 0 + assert run("keygen", "--private", private, "--public", public, "-q") == 2 + + assert run(*pack_args(media, tmp_path)) == 0 + manifest = str(tmp_path / "a" / "manifest.json") + + assert run("verify", manifest, "--public-key", public, "-q") == 2 # unsigned + assert run("sign", manifest, "--key", private, "-q") == 0 + assert run("verify", manifest, "--public-key", public, "-q") == 0 + assert run("unpack", manifest, "-o", str(tmp_path / "o"), "--public-key", public, + "-q") == 0 # fmt: skip + + +@needs_crypto +def test_pack_sign_key(tmp_path, media): + private, public = str(tmp_path / "k.key"), str(tmp_path / "k.pub") + run("keygen", "--private", private, "--public", public, "-q") + + assert run(*pack_args(media, tmp_path, "--sign-key", private)) == 0 + assert run("verify", str(tmp_path / "a"), "--public-key", public, "-q") == 0 + + +def test_notify_posts_a_summary(tmp_path, media, monkeypatch): + received = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + received.append(json.loads(body)) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + url = f"http://127.0.0.1:{server.server_port}/hook" + + try: + # Success is silent by default; --notify-on always sends it. + assert run(*pack_args(media, tmp_path, "--notify", url)) == 0 + assert received == [] + + monkeypatch.setenv("BATCHVAULT_NOTIFY_URL", url) + assert run(*pack_args(media, tmp_path, "--force", "--notify-on", "always")) == 0 + assert received[0]["result"]["status"] == "success" + + # A failure notifies by default. + assert run(*pack_args(media, tmp_path)) == 1 + assert received[1]["result"]["status"] == "skipped" + finally: + server.shutdown() + server.server_close() + + +def test_notification_failure_does_not_change_the_exit_code(tmp_path, media): + code = run(*pack_args(media, tmp_path, "--notify", "http://127.0.0.1:1/x", + "--notify-on", "always")) # fmt: skip + assert code == 0 + + +def test_invalid_notify_url_is_a_usage_error(tmp_path, media): + assert run(*pack_args(media, tmp_path, "--notify", "ftp://nope")) == 2 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..af42247 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,163 @@ +import json + +import pytest + +from batchvault.cli import main +from batchvault.config import find_config, load_config +from batchvault.errors import ConfigError + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "notes.txt").write_text("hello " * 200) + (folder / "skip.log").write_text("log") + return folder + + +def run(*args): + return main([*args, "--no-log"]) + + +def level_in(manifest): + return json.loads(manifest.read_text(encoding="utf-8"))["entries"][0]["level"] + + +def test_find_config_walks_up(tmp_path, monkeypatch): + config = tmp_path / "batchvault.toml" + config.write_text("[pack]\nlevel = 7\n") + nested = tmp_path / "a" / "b" + nested.mkdir(parents=True) + monkeypatch.chdir(nested) + + assert find_config() == config.resolve() + + +def test_find_config_reads_pyproject_tool_section(tmp_path): + (tmp_path / "pyproject.toml").write_text("[tool.other]\nx = 1\n") + assert find_config(tmp_path) is None + + (tmp_path / "pyproject.toml").write_text("[tool.batchvault.pack]\nlevel = 2\n") + found = find_config(tmp_path) + assert found.name == "pyproject.toml" + assert load_config(found) == {"pack": {"level": 2}} + + +def test_invalid_toml_and_unknown_section(tmp_path): + bad = tmp_path / "batchvault.toml" + bad.write_text("[pack\n") + with pytest.raises(ConfigError, match="not valid TOML"): + load_config(bad) + + bad.write_text("[nonsense]\nx = 1\n") + with pytest.raises(ConfigError, match="unknown section"): + load_config(bad) + + +def test_config_supplies_defaults_and_cli_wins(tmp_path, media): + config = tmp_path / "bv.toml" + config.write_text('[pack]\nlevel = 7\nengine = "native"\nvolume-size = "1k"\n') + + base = ["pack", str(media / "notes.txt"), "-q", "--config", str(config)] + assert run(*base, "-o", str(tmp_path / "a")) == 0 + assert level_in(tmp_path / "a" / "manifest.json") == 7 + + assert run(*base, "-o", str(tmp_path / "b"), "--level", "3") == 0 + assert level_in(tmp_path / "b" / "manifest.json") == 3 + + +def test_defaults_table_and_list_options_combine(tmp_path, media, capsys): + config = tmp_path / "bv.toml" + config.write_text('[defaults]\nexclude = ["*.log"]\n') + + assert run("list", "files", str(media), "--config", str(config)) == 0 + names = [item["path"].rsplit("\\", 1)[-1].rsplit("/", 1)[-1] + for item in json.loads(capsys.readouterr().out)] # fmt: skip + assert names == ["notes.txt"] + + +def test_unknown_option_in_config_is_a_usage_error(tmp_path, media): + config = tmp_path / "bv.toml" + config.write_text("[pack]\nnot-an-option = 1\n") + assert run("pack", str(media / "notes.txt"), "--config", str(config)) == 2 + + +@pytest.mark.parametrize("line", ["level = 12", 'level = "high"', 'engine = "zip"']) +def test_invalid_value_in_config_is_a_usage_error(tmp_path, media, line): + config = tmp_path / "bv.toml" + config.write_text(f"[pack]\n{line}\n") + assert run("pack", str(media / "notes.txt"), "--config", str(config)) == 2 + + +def test_missing_explicit_config(tmp_path): + assert run("doctor", "--config", str(tmp_path / "nope.toml")) == 2 + + +# --- a config file found by searching is not fully trusted ------------------- + + +RESTRICTED = [ + ("defaults", 'seven-zip = "./evil"'), + ("pack", 'seven-zip = "./evil"'), + ("pack", 'notify = "https://attacker.example/hook"'), + ("notify", 'url = "https://attacker.example/hook"'), + ("pack", "delete-source = true"), + ("pack", "yes = true"), + ("unpack", 'public-key = "attacker.pub"'), + ("verify", 'public-key = "attacker.pub"'), + ("pack", 'sign-key = "attacker.key"'), + ("pack", 'password-file = "/etc/passwd"'), + ("log", 'file = "/tmp/overwrite-me"'), +] + + +@pytest.mark.parametrize(("table", "line"), RESTRICTED) +def test_found_config_cannot_set_dangerous_options(tmp_path, monkeypatch, table, line): + (tmp_path / "batchvault.toml").write_text(f"[{table}]\n{line}\n") + monkeypatch.chdir(tmp_path) + + assert run("doctor", "-q") == 2 + + +def test_error_explains_how_to_trust_the_file(tmp_path, monkeypatch, capsys): + (tmp_path / "batchvault.toml").write_text('[pack]\nseven-zip = "./evil"\n') + monkeypatch.chdir(tmp_path) + + run("doctor", "-q") + message = " ".join(capsys.readouterr().err.split()) + assert "seven-zip" in message and "--config" in message + + +def test_pyproject_section_is_restricted_too(tmp_path, monkeypatch): + (tmp_path / "pyproject.toml").write_text( + '[tool.batchvault.pack]\ndelete-source = true\n' + ) + monkeypatch.chdir(tmp_path) + assert run("doctor", "-q") == 2 + + +@pytest.mark.parametrize(("table", "line"), RESTRICTED[:3]) +def test_the_same_file_is_accepted_when_named_explicitly(tmp_path, table, line): + config = tmp_path / "trusted.toml" + config.write_text(f"[{table}]\n{line}\n") + assert run("doctor", "-q", "--config", str(config)) == 0 + + +def test_environment_variable_counts_as_naming_the_file(tmp_path, monkeypatch): + config = tmp_path / "trusted.toml" + config.write_text('[defaults]\nseven-zip = "./whatever"\n') + monkeypatch.setenv("BATCHVAULT_CONFIG", str(config)) + monkeypatch.chdir(tmp_path) + assert run("doctor", "-q") == 0 + + +def test_harmless_options_still_work_from_a_found_config(tmp_path, monkeypatch, media): + (tmp_path / "batchvault.toml").write_text( + '[pack]\nengine = "native"\nlevel = 4\nretries = 1\nupdate = true\n' + ) + monkeypatch.chdir(tmp_path) + + code = run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), "-q") + assert code == 0 + assert level_in(tmp_path / "a" / "manifest.json") == 4 diff --git a/tests/test_crypto.py b/tests/test_crypto.py new file mode 100644 index 0000000..65cd742 --- /dev/null +++ b/tests/test_crypto.py @@ -0,0 +1,145 @@ +import os + +import pytest + +pytest.importorskip("cryptography") + +from batchvault.engines.native import NativeEngine # noqa: E402 +from batchvault.errors import EngineError # noqa: E402 +from batchvault.options import PackOptions # noqa: E402 + +engine = NativeEngine(block_size=4096) +PASSWORD = "correct horse" + + +@pytest.fixture +def source(tmp_path): + path = tmp_path / "secret données.bin" + path.write_bytes(os.urandom(20_000)) + return path + + +def pack(source, tmp_path, **kwargs): + options = PackOptions(volume_size=3000, password=PASSWORD, **kwargs) + return engine.pack(source, tmp_path / "archives", options) + + +@pytest.mark.parametrize("fmt", ["xz", "gzip", "bzip2", "store"]) +def test_encrypted_roundtrip(tmp_path, source, fmt): + volumes = pack(source, tmp_path, format=fmt, jobs=3) + assert len(volumes) > 1 + + engine.test(volumes[0], password=PASSWORD) + restored = engine.unpack(volumes[0], tmp_path / "out", password=PASSWORD) + assert restored[0].read_bytes() == source.read_bytes() + + +def test_plaintext_does_not_appear_in_volumes(tmp_path): + source = tmp_path / "marker.txt" + source.write_text("TOP-SECRET-MARKER " * 500) + volumes = pack(source, tmp_path, format="store") + assert b"TOP-SECRET-MARKER" not in b"".join(v.read_bytes() for v in volumes) + + +def test_wrong_password(tmp_path, source): + volumes = pack(source, tmp_path) + with pytest.raises(EngineError, match="Wrong password"): + engine.test(volumes[0], password="nope") + + +def test_missing_password(tmp_path, source): + volumes = pack(source, tmp_path) + with pytest.raises(EngineError, match="encrypted"): + engine.test(volumes[0]) + + +def test_flipped_byte_is_detected(tmp_path, source): + volumes = pack(source, tmp_path) + data = bytearray(volumes[1].read_bytes()) + data[len(data) // 2] ^= 0xFF + volumes[1].write_bytes(bytes(data)) + + with pytest.raises(EngineError): + engine.test(volumes[0], password=PASSWORD) + + +def test_truncation_is_detected(tmp_path, source): + volumes = pack(source, tmp_path) + last = volumes[-1] + last.write_bytes(last.read_bytes()[:-1]) + + with pytest.raises(EngineError): + engine.test(volumes[0], password=PASSWORD) + + +def test_dropping_whole_records_is_detected(tmp_path, source): + # Removing the end marker (the last record) must not look like a valid archive. + volumes = pack(source, tmp_path, format="store") + joined = b"".join(v.read_bytes() for v in volumes) + marker = 4 + 16 # length prefix + GCM tag of the empty end record + for volume in volumes: + volume.unlink() + volumes[0].write_bytes(joined[:-marker]) + + with pytest.raises(EngineError): + engine.test(volumes[0], password=PASSWORD) + + +# --- key derivation cost and header limits ----------------------------------- + + +def test_real_default_is_a_strong_setting(monkeypatch): + from batchvault import crypto + + monkeypatch.undo() # drop the cheap setting the test suite uses + assert crypto.DEFAULT_LOG2_N >= 17 # at least 128 MiB of scrypt memory + assert crypto.parse_header(crypto.make_header("xz"))[1] == crypto.DEFAULT_LOG2_N + + +def test_archives_open_whatever_cost_they_were_made_with(tmp_path, source, monkeypatch): + from batchvault import crypto + + monkeypatch.setattr(crypto, "DEFAULT_LOG2_N", 14) + volumes = pack(source, tmp_path) + + monkeypatch.setattr(crypto, "DEFAULT_LOG2_N", 16) # the default moved on + restored = engine.unpack(volumes[0], tmp_path / "out", password=PASSWORD) + assert restored[0].read_bytes() == source.read_bytes() + + +@pytest.mark.parametrize("log2_n", [0, 13, 19, 30, 255]) +def test_header_asking_for_absurd_memory_is_refused(tmp_path, source, log2_n): + from batchvault import crypto + + volumes = pack(source, tmp_path) + data = bytearray(volumes[0].read_bytes()) + data[len(crypto.MAGIC) + 1] = log2_n # a hostile or damaged header + volumes[0].write_bytes(bytes(data)) + + with pytest.raises(EngineError, match="key setting"): + engine.test(volumes[0], password=PASSWORD) + + +def test_header_cost_cannot_be_lowered_without_detection(tmp_path, source): + from batchvault import crypto + + volumes = pack(source, tmp_path) + data = bytearray(volumes[0].read_bytes()) + data[len(crypto.MAGIC) + 1] = crypto.DEFAULT_LOG2_N + 1 # in range, but changed + volumes[0].write_bytes(bytes(data)) + + with pytest.raises(EngineError): # a different key, and the header is signed + engine.test(volumes[0], password=PASSWORD) + + +def test_an_absurd_record_length_is_refused_without_allocating(tmp_path): + import io + + from batchvault import crypto + + header = crypto.make_header("store") + stream = io.BytesIO(header + (0xFFFFFFFF).to_bytes(4, "big") + b"x" * 64) + _, reader = crypto.open_decrypted(stream, PASSWORD) + + with pytest.raises(EngineError, match="bad record size"): + reader.read(10) diff --git a/tests/test_eta.py b/tests/test_eta.py new file mode 100644 index 0000000..97e0f86 --- /dev/null +++ b/tests/test_eta.py @@ -0,0 +1,30 @@ +from batchvault.eta import MIN_SECONDS, EtaEstimator, format_duration + + +class FakeClock: + def __init__(self): + self.now = 0.0 + + def __call__(self): + return self.now + + +def test_format_duration(): + assert format_duration(3725) == "1:02:05" + assert format_duration(-5) == "0:00:00" + + +def test_no_estimate_before_warmup(): + clock = FakeClock() + eta = EtaEstimator(clock) + eta.start() + clock.now = MIN_SECONDS - 1 + assert eta.estimate(10, 100) is None + + +def test_estimate_uses_average_speed(): + clock = FakeClock() + eta = EtaEstimator(clock) + eta.start() + clock.now = 10 + assert eta.estimate(50, 100) == 10 diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 0000000..1f9fd55 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,265 @@ +import json +import os +import time + +import pytest + +import batchvault as bv +from batchvault import api +from batchvault.engines.native import NativeEngine +from batchvault.errors import OptionsError +from batchvault.volumes import cleanup_staging + +try: + import cryptography # noqa: F401 + + HAS_CRYPTO = True +except ImportError: + HAS_CRYPTO = False + +needs_crypto = pytest.mark.skipif(not HAS_CRYPTO, reason="cryptography not installed") + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "données 1.bin").write_bytes(os.urandom(5000)) + (folder / "notes.txt").write_text("hello " * 500, encoding="utf-8") + return folder + + +def pack(media, tmp_path, **kwargs): + kwargs.setdefault("volume_size", "1k") + kwargs.setdefault("engine", "native") + return bv.pack( + [media / "données 1.bin", media / "notes.txt"], tmp_path / "archives", **kwargs + ) + + +def touch(path, text): + path.write_text(text, encoding="utf-8") + later = time.time() + 30 + os.utime(path, (later, later)) + + +# --- incremental mode ------------------------------------------------------- + + +def test_update_skips_unchanged_and_repacks_changed(tmp_path, media): + pack(media, tmp_path) + + unchanged = pack(media, tmp_path, update=True) + assert unchanged.count(bv.Status.SKIPPED) == 2 + assert {e.message for e in unchanged.entries} == {"Up to date."} + + touch(media / "notes.txt", "changed " * 300) + result = pack(media, tmp_path, update=True) + by_name = {e.source.name: e for e in result.entries} + assert by_name["données 1.bin"].status == bv.Status.SKIPPED + assert by_name["notes.txt"].status == bv.Status.SUCCESS + + restored = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert restored.status == bv.Status.SUCCESS + assert (tmp_path / "out" / "notes.txt").read_text(encoding="utf-8") == ( + media / "notes.txt" + ).read_text(encoding="utf-8") + + +def test_update_repacks_when_settings_change(tmp_path, media): + pack(media, tmp_path, level=5) + result = pack(media, tmp_path, update=True, level=9) + assert result.count(bv.Status.SUCCESS) == 2 + + +def test_update_dry_run_reports_plan(tmp_path, media): + pack(media, tmp_path) + touch(media / "notes.txt", "changed " * 300) + + result = pack(media, tmp_path, update=True, dry_run=True, level=5) + messages = {e.source.name: e.message for e in result.entries} + assert messages == { + "données 1.bin": "Would skip: up to date.", + "notes.txt": "Would update.", + } + + +def test_unpack_update(tmp_path, media): + pack(media, tmp_path) + source, out = tmp_path / "archives", tmp_path / "out" + bv.unpack(source, out) + + again = bv.unpack(source, out, update=True) + assert {e.message for e in again.entries} == {"Already up to date."} + + (out / "notes.txt").write_text("edited") + assert bv.unpack(source, out).count(bv.Status.SKIPPED) == 2 # no update/force + assert bv.unpack(source, out, update=True).count(bv.Status.SUCCESS) == 1 + assert (out / "notes.txt").read_text(encoding="utf-8") == ( + media / "notes.txt" + ).read_text(encoding="utf-8") + + +# --- retries ---------------------------------------------------------------- + + +@pytest.fixture +def flaky(monkeypatch): + monkeypatch.setattr(api, "RETRY_BASE_DELAY", 0) + original = NativeEngine.pack + state = {"calls": 0, "fail_times": 2, "error": OSError} + + def pack_that_fails(self, source, dest_dir, options): + state["calls"] += 1 + if state["calls"] <= state["fail_times"]: + raise state["error"]("transient failure") + return original(self, source, dest_dir, options) + + monkeypatch.setattr(NativeEngine, "pack", pack_that_fails) + return state + + +def test_retries_recover_from_transient_errors(tmp_path, media, flaky): + result = bv.pack([media / "notes.txt"], tmp_path / "a", engine="native", retries=2) + assert result.status == bv.Status.SUCCESS + assert flaky["calls"] == 3 + + +def test_too_few_retries_still_fails(tmp_path, media, flaky): + result = bv.pack([media / "notes.txt"], tmp_path / "a", engine="native", retries=1) + assert result.status == bv.Status.FAILED + assert flaky["calls"] == 2 + + +def test_missing_files_are_not_retried(tmp_path, media, flaky): + flaky["error"] = FileNotFoundError + result = bv.pack([media / "notes.txt"], tmp_path / "a", engine="native", retries=5) + assert result.status == bv.Status.FAILED + assert flaky["calls"] == 1 + + +def test_negative_retries(tmp_path, media): + with pytest.raises(OptionsError): + bv.pack([media / "notes.txt"], tmp_path / "a", engine="native", retries=-1) + + +# --- inspect ---------------------------------------------------------------- + + +def test_inspect(tmp_path, media): + pack(media, tmp_path) + entries = {e.name: e for e in bv.inspect(tmp_path / "archives")} + + big = entries["données 1.bin"] + assert big.size == 5000 and big.engine == "native" and big.format == "xz" + assert big.volumes > 1 and big.complete is True and big.encrypted is False + assert big.ratio is not None + assert json.dumps(big.to_dict()) + + big.path.with_name(big.path.name[:-3] + "002").unlink() + assert {e.name: e for e in bv.inspect(tmp_path / "archives")}[ + "données 1.bin" + ].complete is False + + +def test_inspect_without_manifest(tmp_path, media): + pack(media, tmp_path) + (tmp_path / "archives" / "manifest.json").unlink() + + entries = {e.name: e for e in bv.inspect(tmp_path / "archives")} + assert entries["notes.txt"].engine == "native" + assert entries["notes.txt"].size is None and entries["notes.txt"].volumes >= 1 + + +# --- safe source cleanup ---------------------------------------------------- + + +def test_delete_source_after_roundtrip(tmp_path, media): + original = (media / "notes.txt").read_bytes() + asked = [] + + def confirm(paths): + asked.append(paths) + return True + + result = bv.pack( + [media / "notes.txt"], tmp_path / "a", engine="native", + delete_source=True, confirm=confirm, + ) # fmt: skip + assert result.status == bv.Status.SUCCESS + assert asked == [[(media / "notes.txt").resolve()]] + assert not (media / "notes.txt").exists() + assert "original deleted" in result.entries[0].message + + bv.unpack(tmp_path / "a", tmp_path / "out") + assert (tmp_path / "out" / "notes.txt").read_bytes() == original + + +def test_declined_confirmation_keeps_originals(tmp_path, media): + result = bv.pack( + [media / "notes.txt"], tmp_path / "a", engine="native", + delete_source=True, confirm=lambda paths: False, + ) # fmt: skip + assert (media / "notes.txt").exists() + assert any("not confirmed" in w for w in result.warnings) + + +def test_failed_roundtrip_keeps_original(tmp_path, media, monkeypatch): + monkeypatch.setattr(api, "_roundtrip_problem", lambda *args: "it does not match") + result = bv.pack( + [media / "notes.txt"], tmp_path / "a", engine="native", delete_source=True + ) + assert (media / "notes.txt").exists() + assert "Original kept" in result.entries[0].warnings[0] + + +def test_nothing_is_deleted_without_the_flag(tmp_path, media): + bv.pack([media / "notes.txt"], tmp_path / "a", engine="native") + assert (media / "notes.txt").exists() + + +# --- leftovers from killed runs --------------------------------------------- + + +def test_stale_staging_folders_are_cleaned(tmp_path): + root = tmp_path / "archives" + stale = root / "file.bin" / ".batchvault-old" + fresh = root / "file.bin" / ".batchvault-new" + stale.mkdir(parents=True) + fresh.mkdir(parents=True) + long_ago = time.time() - 3 * 86400 + os.utime(stale, (long_ago, long_ago)) + + assert cleanup_staging(root) == 1 + assert not stale.exists() and fresh.exists() + + +# --- encryption through the API --------------------------------------------- + + +@needs_crypto +def test_encrypted_pack_unpack_verify_inspect(tmp_path, media): + pack(media, tmp_path, password="pw") + archives = tmp_path / "archives" + + assert all(e.encrypted for e in bv.inspect(archives)) + + assert bv.verify(archives).count(bv.Status.FAILED) == 2 + assert bv.verify(archives, password="pw").status == bv.Status.SUCCESS + assert bv.verify(archives, password="wrong").count(bv.Status.FAILED) == 2 + + missing = bv.unpack(archives, tmp_path / "out") + assert "password is required" in missing.entries[0].message + assert not (tmp_path / "out" / "notes.txt").exists() + + bv.unpack(archives, tmp_path / "out", password="pw") + assert (tmp_path / "out" / "données 1.bin").read_bytes() == ( + media / "données 1.bin" + ).read_bytes() + + +@needs_crypto +def test_update_repacks_when_encryption_changes(tmp_path, media): + pack(media, tmp_path) + result = pack(media, tmp_path, update=True, password="pw") + assert result.count(bv.Status.SUCCESS) == 2 diff --git a/tests/test_filters.py b/tests/test_filters.py new file mode 100644 index 0000000..0eca12f --- /dev/null +++ b/tests/test_filters.py @@ -0,0 +1,31 @@ +from batchvault.filters import PathFilter, load_patterns + + +def test_no_patterns_matches_everything(): + assert PathFilter().matches("a/b.txt") + + +def test_include_limits_matches(): + path_filter = PathFilter(include=["*.mp4"]) + assert path_filter.matches("media/a.mp4") + assert not path_filter.matches("media/a.txt") + + +def test_exclude_wins_over_include(): + path_filter = PathFilter(include=["*.mp4"], exclude=["tmp/"]) + assert path_filter.matches("a.mp4") + assert not path_filter.matches("tmp/a.mp4") + + +def test_recursive_glob(): + assert PathFilter(include=["docs/**/*.md"]).matches("docs/a/b/c.md") + + +def test_windows_separators_are_normalised(): + assert not PathFilter(exclude=["tmp/"]).matches("C:\\data\\tmp\\a.txt") + + +def test_patterns_from_file(tmp_path): + patterns = tmp_path / "patterns.txt" + patterns.write_text("# skip\n*.log\n\n*.tmp\n", encoding="utf-8") + assert load_patterns([f"@{patterns}", "*.bak"]) == ["*.log", "*.tmp", "*.bak"] diff --git a/tests/test_folders.py b/tests/test_folders.py new file mode 100644 index 0000000..9e4a6ee --- /dev/null +++ b/tests/test_folders.py @@ -0,0 +1,318 @@ +"""Folders are kept by default; --root and list files adjust where they start.""" + +import io +import json +import os +import time + +import pytest +from rich.console import Console + +import batchvault as bv +from batchvault import ui +from batchvault.cli import main +from batchvault.errors import InputError, ManifestError +from batchvault.volumes import cleanup_staging + +FILES = { + "Resources/intro/lesson1.bin": os.urandom(5000), + "Resources/intro/notes.txt": b"intro notes\n" * 400, + "Resources/notes.txt": b"resource notes\n" * 400, # same name, other folder + "top.txt": b"top level\n" * 400, +} + + +@pytest.fixture +def tree(tmp_path): + root = tmp_path / "project" + for name, data in FILES.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return root + + +def sources(tree): + return [tree / name for name in FILES] + + +def pack(tree, tmp_path, **kwargs): + kwargs.setdefault("volume_size", "2k") + return bv.pack(sources(tree), tmp_path / "archives", engine="native", **kwargs) + + +def manifest_entries(tmp_path): + text = (tmp_path / "archives" / "manifest.json").read_text(encoding="utf-8") + return json.loads(text)["entries"] + + +def stored(tmp_path): + return {(e["folder"], e["name"]) for e in manifest_entries(tmp_path)} + + +# --- the default: folders are kept ------------------------------------------------ + + +def test_folders_are_kept_without_any_option(tmp_path, tree): + result = pack(tree, tmp_path) + assert result.status == bv.Status.SUCCESS + + archives = tmp_path / "archives" + assert (archives / "Resources/intro/lesson1.bin/lesson1.bin.bv.001").is_file() + assert (archives / "Resources/notes.txt/notes.txt.bv.001").is_file() + assert (archives / "Resources/intro/notes.txt/notes.txt.bv.001").is_file() + assert (archives / "top.txt/top.txt.bv.001").is_file() # directly in the folder + + assert stored(tmp_path) == { + ("Resources/intro", "lesson1.bin"), + ("Resources/intro", "notes.txt"), + ("Resources", "notes.txt"), + ("", "top.txt"), + } + assert {e.folder for e in result.entries} == {"Resources/intro", "Resources", ""} + + +def test_unpack_recreates_the_tree(tmp_path, tree): + pack(tree, tmp_path) + result = bv.unpack(tmp_path / "archives", tmp_path / "out") + + assert result.status == bv.Status.SUCCESS + for name, data in FILES.items(): + assert (tmp_path / "out" / name).read_bytes() == data + assert not list((tmp_path / "out").rglob(".batchvault-*")) + + +def test_files_in_one_folder_are_not_nested(tmp_path, tree): + both = [tree / "Resources/intro/lesson1.bin", tree / "Resources/intro/notes.txt"] + bv.pack(both, tmp_path / "archives", engine="native") + + assert (tmp_path / "archives/lesson1.bin/lesson1.bin.bv.001").is_file() + assert {e["folder"] for e in manifest_entries(tmp_path)} == {""} + + bv.unpack(tmp_path / "archives", tmp_path / "out") + assert (tmp_path / "out" / "lesson1.bin").is_file() + + +def test_a_single_file_is_stored_without_folders(tmp_path, tree): + bv.pack([tree / "Resources/intro/lesson1.bin"], tmp_path / "archives", + engine="native") # fmt: skip + assert stored(tmp_path) == {("", "lesson1.bin")} + + +def test_same_name_in_two_folders_just_works(tmp_path, tree): + result = pack(tree, tmp_path) # two notes.txt, in different folders + assert result.status == bv.Status.SUCCESS + bv.unpack(tmp_path / "archives", tmp_path / "out") + assert (tmp_path / "out/Resources/notes.txt").read_bytes() == ( + FILES["Resources/notes.txt"] + ) + assert (tmp_path / "out/Resources/intro/notes.txt").read_bytes() == ( + FILES["Resources/intro/notes.txt"] + ) + + +def test_files_from_different_folders_keep_what_is_below_the_shared_folder( + tmp_path, tree +): + other = tmp_path / "other" + other.mkdir() + (other / "extra.txt").write_text("extra") + bv.pack([tree / "top.txt", other / "extra.txt"], tmp_path / "archives", + engine="native") # fmt: skip + + # The shared folder is tmp_path itself, so each file keeps its own folder name. + assert stored(tmp_path) == {("project", "top.txt"), ("other", "extra.txt")} + + +# --- list files remembers the listed folder --------------------------------------- + + +@pytest.fixture +def solo(tmp_path): + """Every file sits one level inside Resources: nothing else would show it.""" + root = tmp_path / "solo" + for name in ("a.bin", "b.bin"): + path = root / "Resources" / "intro" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(os.urandom(3000)) + return root + + +def test_the_listed_folders_name_is_kept(tmp_path, solo): + listed = bv.list_files(solo / "Resources") + assert {e.root for e in listed} == {solo.resolve()} + + listing = tmp_path / "files.json" + listing.write_text(json.dumps([e.to_dict() for e in listed]), encoding="utf-8") + bv.pack(list_file=listing, output=tmp_path / "archives", engine="native") + + assert stored(tmp_path) == {("Resources/intro", n) for n in ("a.bin", "b.bin")} + bv.unpack(tmp_path / "archives", tmp_path / "out") + assert (tmp_path / "out/Resources/intro/a.bin").is_file() + + +def test_without_the_listing_only_the_shared_folder_is_known(tmp_path, solo): + files = sorted((solo / "Resources").rglob("*.bin")) + bv.pack(files, tmp_path / "archives", engine="native") + assert stored(tmp_path) == {("", "a.bin"), ("", "b.bin")} + + +def test_text_lists_remember_it_too(tmp_path, solo): + listing = tmp_path / "files.txt" + assert main(["list", "files", str(solo / "Resources"), "-o", str(listing), "-q", + "--no-log"]) == 0 # fmt: skip + first_line = listing.read_text(encoding="utf-8").splitlines()[0] + assert first_line == f"# batchvault-root: {solo.resolve()}" + + bv.pack(list_file=listing, output=tmp_path / "archives", engine="native") + assert stored(tmp_path) == {("Resources/intro", n) for n in ("a.bin", "b.bin")} + + +def test_listing_json_on_the_command_line_has_the_root(tmp_path, solo, capsys): + assert main(["list", "files", str(solo / "Resources"), "--no-log"]) == 0 + items = json.loads(capsys.readouterr().out) + assert {item["root"] for item in items} == {str(solo.resolve())} + + +def test_command_line_default_uses_the_listing(tmp_path, tree): + listing = tmp_path / "files.json" + assert main(["list", "files", str(tree), "-o", str(listing), "-q", "--no-log"]) == 0 + + code = main(["pack", "--list", str(listing), "-o", str(tmp_path / "a"), + "--engine", "native", "-q", "--no-log"]) # fmt: skip + assert code == 0 + first = tmp_path / "a/project/Resources/intro/lesson1.bin/lesson1.bin.bv.001" + assert first.is_file() + + assert main(["unpack", str(tmp_path / "a"), "-o", str(tmp_path / "o"), "-q", + "--no-log"]) == 0 # fmt: skip + restored = (tmp_path / "o/project/Resources/notes.txt").read_bytes() + assert restored == FILES["Resources/notes.txt"] + + +# --- --root overrides --------------------------------------------------------------- + + +def test_an_explicit_root_decides_where_paths_start(tmp_path, tree): + inside = [tree / "Resources/intro/lesson1.bin", tree / "Resources/notes.txt"] + bv.pack(inside, tmp_path / "archives", engine="native", root=tree / "Resources") + + assert stored(tmp_path) == {("intro", "lesson1.bin"), ("", "notes.txt")} + + +def test_an_explicit_root_keeps_a_chosen_top_folder(tmp_path, tree): + pack(tree, tmp_path, root=tree.parent) + assert ("project/Resources/intro", "lesson1.bin") in stored(tmp_path) + + +def test_a_file_outside_an_explicit_root_is_refused(tmp_path, tree): + with pytest.raises(InputError, match="not inside the root"): + pack(tree, tmp_path, root=tree / "Resources/intro") + + +def test_command_line_root_option(tmp_path, tree): + listing = tmp_path / "files.json" + main(["list", "files", str(tree), "-o", str(listing), "-q", "--no-log"]) + + code = main(["pack", "--list", str(listing), "--root", str(tree), + "-o", str(tmp_path / "a"), "--engine", "native", "-q", + "--no-log"]) # fmt: skip + assert code == 0 + assert (tmp_path / "a/Resources/intro/lesson1.bin/lesson1.bin.bv.001").is_file() + + +# --- everything else works per path ------------------------------------------------ + + +def test_update_works_per_path_not_per_name(tmp_path, tree): + pack(tree, tmp_path) + changed = tree / "Resources/notes.txt" + changed.write_bytes(b"edited\n" * 400) + later = time.time() + 30 + os.utime(changed, (later, later)) + + result = pack(tree, tmp_path, update=True) + states = {(e.folder, e.source.name): e.status for e in result.entries} + + assert states[("Resources", "notes.txt")] == bv.Status.SUCCESS + assert states[("Resources/intro", "notes.txt")] == bv.Status.SKIPPED # same name + assert states[("", "top.txt")] == bv.Status.SKIPPED + + +def test_existing_files_in_the_tree_are_skipped_unless_forced(tmp_path, tree): + pack(tree, tmp_path) + bv.unpack(tmp_path / "archives", tmp_path / "out") + + again = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert again.count(bv.Status.SKIPPED) == 4 + + forced = bv.unpack(tmp_path / "archives", tmp_path / "out", force=True) + assert forced.status == bv.Status.SUCCESS + + +def test_verify_against_compares_the_file_in_the_same_folder(tmp_path, tree): + pack(tree, tmp_path) + assert bv.verify(tmp_path / "archives", against=tree).status == bv.Status.SUCCESS + + # Only the intro copy changes; the other notes.txt must not hide that. + (tree / "Resources/intro/notes.txt").write_bytes(b"different\n") + result = bv.verify(tmp_path / "archives", against=tree) + failed = [e for e in result.entries if e.status == bv.Status.FAILED] + assert [e.folder for e in failed] == ["Resources/intro"] + assert failed[0].message.startswith("notes.txt differs") + + +def test_inspect_and_dry_run_show_the_folder(tmp_path, tree): + plan = pack(tree, tmp_path, dry_run=True) + assert {e.folder for e in plan.entries} == {"Resources/intro", "Resources", ""} + assert not (tmp_path / "archives").exists() + + pack(tree, tmp_path) + inspected = bv.inspect(tmp_path / "archives") + found = {(e.folder, e.name) for e in inspected} + assert ("Resources/intro", "notes.txt") in found and ("", "top.txt") in found + assert all("folder" in e.to_dict() for e in inspected) + + +def test_without_a_manifest_files_are_restored_flat(tmp_path, tree): + pack(tree, tmp_path) + (tmp_path / "archives" / "manifest.json").unlink() + + result = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert (tmp_path / "out" / "lesson1.bin").is_file() # the tree is in the manifest + assert result.warnings + + +@pytest.mark.parametrize( + "folder", ["..", "../x", "/abs", "a" + chr(92) + "b", "a//b", "./a", "a/../b", "a/"] +) +def test_unsafe_stored_folders_are_rejected(tmp_path, tree, folder): + pack(tree, tmp_path) + manifest = tmp_path / "archives" / "manifest.json" + data = json.loads(manifest.read_text(encoding="utf-8")) + data["entries"][0]["folder"] = folder + manifest.write_text(json.dumps(data), encoding="utf-8") + + with pytest.raises(ManifestError, match="folder"): + bv.unpack(tmp_path / "archives", tmp_path / "out") + assert not (tmp_path / "escape").exists() + + +def test_deeply_nested_leftover_staging_folders_are_cleaned(tmp_path): + stale = tmp_path / "archives/Resources/intro/file.bin/.batchvault-old" + fresh = tmp_path / "archives/Resources/.batchvault-new" + stale.mkdir(parents=True) + fresh.mkdir(parents=True) + long_ago = time.time() - 3 * 86400 + os.utime(stale, (long_ago, long_ago)) + + assert cleanup_staging(tmp_path / "archives") == 1 + assert not stale.exists() and fresh.exists() + + +def test_the_result_table_shows_the_folder(tmp_path, tree): + buffer = io.StringIO() + console = Console(file=buffer, width=160, force_terminal=False, color_system=None) + ui.print_summary(console, pack(tree, tmp_path), None, "pack") + + assert "Resources/intro/lesson1.bin" in buffer.getvalue() diff --git a/tests/test_hardening.py b/tests/test_hardening.py new file mode 100644 index 0000000..073435f --- /dev/null +++ b/tests/test_hardening.py @@ -0,0 +1,320 @@ +"""Regression tests for the security fixes.""" + +import io +import json +import lzma +import os +import stat +import sys +import tarfile +from pathlib import Path + +import pytest + +import batchvault as bv +from batchvault.cli import main +from batchvault.engines.native import NativeEngine +from batchvault.engines.sevenzip import ( + SevenZipEngine, + build_pack_command, + build_test_command, + build_unpack_command, + folder_size, +) +from batchvault.errors import EngineError +from batchvault.options import MIB, PackOptions +from batchvault.security import redact_arguments + +REPO = Path(__file__).resolve().parents[1] +engine = NativeEngine(block_size=4096) + + +@pytest.fixture +def media(tmp_path): + folder = tmp_path / "media" + folder.mkdir() + (folder / "notes.txt").write_text("hello " * 10_000, encoding="utf-8") + return folder + + +def run(*args): + return main([*args, "--no-log"]) + + +# --- webhook secrets stay out of the log ------------------------------------- + + +def test_redact_arguments_hides_webhook_values(): + argv = ["pack", "a.txt", "--notify", "https://h.example/T0/B0/SECRET", "-q"] + assert redact_arguments(argv) == ["pack", "a.txt", "--notify", "***", "-q"] + + assert redact_arguments(["--notify=https://h.example/SECRET"]) == ["--notify=***"] + assert redact_arguments(["--notify-on", "always"]) == ["--notify-on", "always"] + assert redact_arguments(["https://u:pw@host/x"]) == ["https://***@host/x"] + + +def test_webhook_secret_is_not_written_to_the_log(tmp_path, media): + log = tmp_path / "run.log" + url = "http://127.0.0.1:1/services/SECRETTOKEN123" + + code = main([ + "pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q", "--notify", url, "--notify-on", "always", + "--log-file", str(log), + ]) # fmt: skip + assert code == 0 # the unreachable webhook only warns + + text = log.read_text(encoding="utf-8") + assert "SECRETTOKEN123" not in text + assert "--notify ***" in text + + +def test_plain_http_webhook_gets_a_warning(tmp_path, media, capsys): + code = run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--dry-run", "-q", + "--notify", "http://hooks.example.com/x") # fmt: skip + assert code == 0 + assert "unencrypted" in capsys.readouterr().err + + run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "--dry-run", "-q", + "--notify", "http://127.0.0.1:9/x") # fmt: skip + assert "unencrypted" not in capsys.readouterr().err + + +# --- 7-Zip: wildcards, links, bombs ------------------------------------------ + + +def test_wildcards_in_file_names_are_not_patterns(): + options = PackOptions() + exe, src = Path("7z"), Path("a*b.txt") + assert "-spd" in build_pack_command(exe, Path("out.7z"), src, options) + assert "-spd" in build_unpack_command(exe, Path("x.7z.001"), Path("o")) + assert "-spd" in build_test_command(exe, Path("x.7z.001")) + + +@pytest.fixture +def fake_7zip(tmp_path): + exe = tmp_path / "7z" + exe.write_text("") + volume = tmp_path / "a.7z.001" + volume.write_bytes(b"x") + return SevenZipEngine(exe), volume + + +def test_links_in_7zip_output_are_refused(tmp_path, fake_7zip): + sevenzip, volume = fake_7zip + + def fake_run(command): + target = next(arg[2:] for arg in command if arg.startswith("-o")) + try: + (Path(target) / "link.txt").symlink_to(volume) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available here") + return "" + + sevenzip._run = fake_run + with pytest.raises(EngineError, match="link or special"): + sevenzip.unpack(volume, tmp_path / "out") + assert not (tmp_path / "out" / "link.txt").exists() + + +def test_a_limit_makes_7zip_extract_under_watch(tmp_path, fake_7zip): + sevenzip, volume = fake_7zip + calls = [] + + def fake_limited(command, watch, max_bytes): + calls.append((command[1], watch.is_dir(), max_bytes)) + raise EngineError("would expand beyond the allowed size") + + sevenzip._run_limited = fake_limited + with pytest.raises(EngineError, match="allowed size"): + sevenzip.unpack(volume, tmp_path / "out", max_bytes=1000) + assert calls == [("e", True, 1000)] + assert list((tmp_path / "out").iterdir()) == [] # staging folder cleaned up + + +WRITER = """ +import sys, time +from pathlib import Path +folder = Path(sys.argv[1]) +for number in range(400): # up to about 400 MB + (folder / f"chunk{number}.bin").write_bytes(bytes(1024 * 1024)) + time.sleep(0.02) +""" + + +def test_extraction_is_stopped_when_it_grows_past_the_limit(tmp_path, fake_7zip): + sevenzip, _ = fake_7zip + watched = tmp_path / "watched" + watched.mkdir() + + with pytest.raises(EngineError, match="allowed size"): + sevenzip._run_limited( + [sys.executable, "-c", WRITER, str(watched)], watched, 3 * MIB + ) + assert folder_size(watched) < 100 * MIB # cut off long before it finished + + +def test_a_program_within_the_limit_runs_to_the_end(tmp_path, fake_7zip): + sevenzip, _ = fake_7zip + script = ( + "import sys, pathlib; " + "pathlib.Path(sys.argv[1], 'small.bin').write_bytes(bytes(1000)); " + "print('done')" + ) + output = sevenzip._run_limited( + [sys.executable, "-c", script, str(tmp_path)], tmp_path, MIB + ) + assert "done" in output + + +def test_failures_still_carry_the_program_output(tmp_path, fake_7zip): + sevenzip, _ = fake_7zip + script = "import sys; print('Wrong password?'); sys.exit(2)" + with pytest.raises(EngineError, match=r"(?s)exit code 2.*Wrong password"): + sevenzip._run_limited([sys.executable, "-c", script], tmp_path, MIB) + + +# --- native: decompression bombs --------------------------------------------- + + +def make_volume(path: Path, member_size: int, trailing: int = 0) -> Path: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as tar: + info = tarfile.TarInfo("bomb.txt") + info.size = member_size + tar.addfile(info, io.BytesIO(bytes(member_size))) + path.write_bytes(lzma.compress(buffer.getvalue() + bytes(trailing))) + return path + + +def test_oversized_member_is_refused_before_anything_is_written(tmp_path): + volume = make_volume(tmp_path / "bomb.txt.bv.001", 5 * MIB) + out = tmp_path / "out" + + with pytest.raises(EngineError, match="allowed size"): + engine.unpack(volume, out, max_bytes=MIB) + assert list(out.iterdir()) == [] # not even a staging folder is left + + +def test_a_limit_at_the_real_size_still_restores(tmp_path): + volume = make_volume(tmp_path / "bomb.txt.bv.001", 5 * MIB) + restored = engine.unpack(volume, tmp_path / "out", max_bytes=5 * MIB) + assert restored[0].stat().st_size == 5 * MIB + + +def test_no_limit_by_default(tmp_path): + volume = make_volume(tmp_path / "big.txt.bv.001", 2 * MIB) + assert engine.unpack(volume, tmp_path / "out")[0].stat().st_size == 2 * MIB + + +def test_data_hidden_after_the_end_of_the_archive_is_refused(tmp_path): + volume = make_volume(tmp_path / "x.txt.bv.001", 100, trailing=20 * MIB) + with pytest.raises(EngineError, match="unexpected data"): + engine.test(volume) + + +def test_manifest_size_caps_what_an_archive_may_expand_to(tmp_path, media): + bv.pack([media / "notes.txt"], tmp_path / "a", engine="native") + manifest = tmp_path / "a" / "manifest.json" + data = json.loads(manifest.read_text(encoding="utf-8")) + data["entries"][0]["size"] = 10 # claims a tiny file; the archive holds more + manifest.write_text(json.dumps(data), encoding="utf-8") + + result = bv.unpack(tmp_path / "a", tmp_path / "out") + assert result.status == bv.Status.FAILED + assert "allowed size" in result.entries[0].message + assert not (tmp_path / "out" / "notes.txt").exists() + + +def test_max_output_size_option(tmp_path, media): + bv.pack([media / "notes.txt"], tmp_path / "a", engine="native") + + refused = bv.unpack(tmp_path / "a", tmp_path / "out", max_output_size="1k") + assert refused.status == bv.Status.FAILED + + allowed = bv.unpack(tmp_path / "a", tmp_path / "out", max_output_size="1m") + assert allowed.status == bv.Status.SUCCESS + + +def test_max_output_size_on_the_command_line(tmp_path, media): + assert run("pack", str(media / "notes.txt"), "-o", str(tmp_path / "a"), + "--engine", "native", "-q") == 0 # fmt: skip + + out = str(tmp_path / "out") + assert run("unpack", str(tmp_path / "a"), "-o", out, "-q", + "--max-output-size", "1k") == 1 # fmt: skip + assert not (tmp_path / "out" / "notes.txt").exists() + assert run("unpack", str(tmp_path / "a"), "-o", out, "-q", + "--max-output-size", "10m") == 0 # fmt: skip + + +# --- owner-only files -------------------------------------------------------- + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX permission bits") +def test_archives_and_manifest_are_private_to_the_owner(tmp_path, media): + previous = os.umask(0o022) # a typical umask would have made them 0644 + try: + result = bv.pack([media / "notes.txt"], tmp_path / "a", engine="native", + volume_size="2k") # fmt: skip + finally: + os.umask(previous) + + for volume in result.entries[0].volumes: + assert stat.S_IMODE(volume.stat().st_mode) == 0o600 + assert stat.S_IMODE(result.manifest.stat().st_mode) == 0o600 + assert not list((tmp_path / "a").glob("*.tmp")) + + +# --- the GitHub Action keeps options and paths apart ------------------------- + + +def test_action_passes_paths_after_a_double_dash(): + text = (REPO / "action.yml").read_text(encoding="utf-8") + + assert 'batchvault "$COMMAND" "${args[@]}" -- "${positional[@]}"' in text + assert 'positional+=("$line")' in text + assert 'positional+=("$SOURCE")' in text + assert 'args+=("$line")' not in text + assert 'args+=("$SOURCE")' not in text + assert "::add-mask::" in text + + +def test_a_path_that_looks_like_an_option_is_just_a_path(tmp_path, media): + odd = media / "-q" + odd.write_text("a file whose name looks like an option") + + code = main([ + "pack", "--json", "--engine", "native", "-o", str(tmp_path / "a"), + "--no-log", "--", str(odd), + ]) # fmt: skip + assert code == 0 + assert (tmp_path / "a" / "-q").is_dir() + + +# --- cleanup does not follow links ------------------------------------------- + + +def test_cleanup_ignores_linked_folders(tmp_path): + import time + + from batchvault.volumes import cleanup_staging + + elsewhere = tmp_path / "elsewhere" + victim = elsewhere / ".batchvault-keep" + victim.mkdir(parents=True) + long_ago = time.time() - 3 * 86400 + os.utime(victim, (long_ago, long_ago)) + + root = tmp_path / "archives" + root.mkdir() + try: + (root / "linked").symlink_to(elsewhere, target_is_directory=True) + (root / ".batchvault-link").symlink_to(victim, target_is_directory=True) + except (OSError, NotImplementedError): + pytest.skip("symbolic links are not available here") + + assert cleanup_staging(root) == 0 + assert victim.exists() diff --git a/tests/test_inputs.py b/tests/test_inputs.py new file mode 100644 index 0000000..3a011d3 --- /dev/null +++ b/tests/test_inputs.py @@ -0,0 +1,135 @@ +import json +import os + +import pytest + +from batchvault.errors import InputError +from batchvault.filters import PathFilter +from batchvault.inputs import collect_sources, gather_sources, read_list_entries + + +@pytest.fixture +def files(tmp_path): + paths = [] + for name in ("a one.txt", "b.mp4", "ünï.csv"): + path = tmp_path / name + path.write_text("x", encoding="utf-8") + paths.append(path) + return paths + + +def test_text_list_ignores_blanks_and_comments(tmp_path, files): + listing = tmp_path / "list.txt" + listing.write_text( + f"# comment\n\n{files[0]}\n {files[1]} \n", encoding="utf-8-sig" + ) + assert collect_sources(list_file=listing) == [p.resolve() for p in files[:2]] + + +def test_json_list_accepts_strings_and_objects(tmp_path, files): + listing = tmp_path / "list.json" + listing.write_text( + json.dumps([str(files[0]), {"path": str(files[2]), "size": 1}]), + encoding="utf-8", + ) + assert [e for _, e in read_list_entries(listing)] == [str(files[0]), str(files[2])] + + +def test_duplicates_are_removed(files): + sources = collect_sources([files[0], files[0], files[1]]) + assert len(sources) == 2 + + +def test_missing_file_reports_line(tmp_path): + listing = tmp_path / "list.txt" + listing.write_text("nope.txt\n", encoding="utf-8") + with pytest.raises(InputError, match=r"list\.txt:1"): + collect_sources(list_file=listing) + + +def test_same_name_in_different_folders_is_fine(tmp_path): + for folder in ("x", "y"): + (tmp_path / folder).mkdir() + (tmp_path / folder / "same.txt").write_text("1") + + sourced = gather_sources([tmp_path / "x" / "same.txt", tmp_path / "y" / "same.txt"]) + assert [folder for _, folder in sourced] == ["x", "y"] + + +def test_colliding_places_are_still_refused(tmp_path): + for folder in ("a", "b"): + (tmp_path / folder / "x").mkdir(parents=True) + (tmp_path / folder / "x" / "same.txt").write_text("1") + + # Two lists recorded different roots, so both files would be stored as x/same.txt. + listing = tmp_path / "list.json" + listing.write_text( + json.dumps([ + {"path": str(tmp_path / "a/x/same.txt"), "root": str(tmp_path / "a")}, + {"path": str(tmp_path / "b/x/same.txt"), "root": str(tmp_path / "b")}, + ]), + encoding="utf-8", + ) + with pytest.raises(InputError, match="same archive folder"): + gather_sources(list_file=listing) + + +def test_folders_follow_the_recorded_root_then_the_shared_folder(tmp_path): + for name in ("one/a.txt", "one/two/b.txt", "other.txt"): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x") + one, two = tmp_path / "one" / "a.txt", tmp_path / "one" / "two" / "b.txt" + + shared = dict(gather_sources([one, two])) + assert shared == {one.resolve(): "", two.resolve(): "two"} + + spread = dict(gather_sources([one, tmp_path / "other.txt"])) + assert spread == {one.resolve(): "one", (tmp_path / "other.txt").resolve(): ""} + + explicit = dict(gather_sources([one, two], root=tmp_path)) + assert explicit == {one.resolve(): "one", two.resolve(): "one/two"} + + assert dict(gather_sources([one])) == {one.resolve(): ""} + + +def test_a_recorded_root_in_a_text_list(tmp_path): + (tmp_path / "Resources" / "intro").mkdir(parents=True) + (tmp_path / "Resources" / "intro" / "a.txt").write_text("x") + first = tmp_path / "Resources" / "intro" / "a.txt" + header = f"# batchvault-root: {tmp_path}" + lines = [header, str(first)] + (tmp_path / "list.txt").write_text(os.linesep.join(lines), encoding="utf-8") + + ((_, folder),) = gather_sources(list_file=tmp_path / "list.txt") + assert folder == "Resources/intro" + + +def test_an_edited_list_falls_back_to_the_shared_folder(tmp_path): + (tmp_path / "data").mkdir() + (tmp_path / "data" / "a.txt").write_text("x") + (tmp_path / "elsewhere").mkdir() + + listing = tmp_path / "list.json" + item = {"path": str(tmp_path / "data/a.txt"), "root": str(tmp_path / "elsewhere")} + listing.write_text(json.dumps([item]), encoding="utf-8") + ((_, folder),) = gather_sources(list_file=listing) + assert folder == "" # no error; the recorded root no longer contains the file + + +def test_archives_are_refused_unless_allowed(tmp_path): + volume = tmp_path / "a.txt.bv.001" + volume.write_bytes(b"x") + with pytest.raises(InputError, match="looks like an archive"): + collect_sources([volume]) + assert collect_sources([volume], allow_archives=True) + + +def test_filter_applies(files): + sources = collect_sources(files, path_filter=PathFilter(exclude=["*.mp4"])) + assert [p.name for p in sources] == ["a one.txt", "ünï.csv"] + + +def test_nothing_left_is_an_error(files): + with pytest.raises(InputError, match="No input files"): + collect_sources(files, path_filter=PathFilter(include=["*.zip"])) diff --git a/tests/test_levels.py b/tests/test_levels.py new file mode 100644 index 0000000..4278ff3 --- /dev/null +++ b/tests/test_levels.py @@ -0,0 +1,266 @@ +"""--level auto: the level follows the file's type.""" + +import json +import os + +import pytest + +import batchvault as bv +from batchvault.cli import main +from batchvault.errors import OptionsError +from batchvault.levels import ( + AUTO, + BINARY_LEVEL, + DEFAULT_LEVEL, + MEDIA_LEVEL, + PDF_LEVEL, + TEXT_LEVEL, + choose_level, + level_argument, + parse_level, +) + + +def make(tmp_path, name, content=b"plain bytes\n" * 100): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + return path + + +# --- the table ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name", + ["a.mp4", "a.MKV", "a.jpg", "a.png", "a.zip", "a.7z", "a.gz", "a.docx", "a.mp3", + "a.flac", "a.webp", "a.heic", "a.tar.gz", "a.xlsx", "a.cr2", "a.webm"], +) +def test_already_compressed_files_get_level_zero(tmp_path, name): + choice = choose_level(make(tmp_path, name)) + assert choice.level == 0 and choice.kind == "already compressed" + + +@pytest.mark.parametrize( + "name", ["a.txt", "a.LOG", "a.csv", "a.json", "a.xml", "a.py", "a.sql", "a.md", + "a.html", "a.yaml", "a.svg"], +) +def test_text_gets_a_strong_level(tmp_path, name): + choice = choose_level(make(tmp_path, name)) + assert choice.level == TEXT_LEVEL and choice.kind == "text" + + +@pytest.mark.parametrize("name", ["a.wav", "a.bmp", "a.tiff", "a.TIF"]) +def test_uncompressed_media(tmp_path, name): + assert choose_level(make(tmp_path, name)).level == MEDIA_LEVEL + + +@pytest.mark.parametrize("name", ["a.exe", "a.iso", "a.sqlite", "a.vmdk", "a.tar"]) +def test_binary_data_gets_the_balanced_level(tmp_path, name): + choice = choose_level(make(tmp_path, name)) + assert choice.level == BINARY_LEVEL and choice.kind == "binary data" + + +def test_pdf_and_unknown_types(tmp_path): + assert choose_level(make(tmp_path, "a.pdf")).level == PDF_LEVEL + for name in ("a.bin", "a.dat", "noextension", "a.weirdext"): + choice = choose_level(make(tmp_path, name)) + assert choice.level == DEFAULT_LEVEL and choice.kind == "unknown type" + + +# --- the file's own bytes beat its name --------------------------------------- + + +@pytest.mark.parametrize( + ("label", "head"), + [ + ("zip", b"PK\x03\x04rest"), + ("gzip", b"\x1f\x8b\x08rest"), + ("xz", b"\xfd7zXZ\x00rest"), + ("7z", b"7z\xbc\xaf\x27\x1c rest"), + ("rar", b"Rar!\x1a\x07\x00rest"), + ("jpeg", b"\xff\xd8\xff\xe0rest"), + ("png", b"\x89PNG\r\n\x1a\nrest"), + ("mp4", b"\x00\x00\x00\x18ftypisom"), + ("mkv", b"\x1a\x45\xdf\xa3rest"), + ("ogg", b"OggS\x00rest"), + ("webp", b"RIFF\x10\x00\x00\x00WEBPVP8 "), + ("bzip2", b"BZh91AY&SYrest"), + ("mp3", b"ID3\x03\x00\x00\x00\x00\x00\x10rest"), + ], +) +def test_a_recognised_signature_means_already_compressed(tmp_path, label, head): + # The name says text and the extension says nothing; the bytes decide. + for name in ("notes.txt", "mystery.bin", "noextension"): + assert choose_level(make(tmp_path, name, head + b"x" * 50)).level == 0, label + + +def test_text_that_merely_starts_like_a_signature_is_still_text(tmp_path): + for first_line in (b"BZh is a column\n", b"ID3,name\n", b"PKG,name\n"): + path = make(tmp_path, "table.csv", first_line * 20) + assert choose_level(path).level == TEXT_LEVEL, first_line + + +def test_an_unreadable_or_empty_file_falls_back_to_its_extension(tmp_path): + assert choose_level(make(tmp_path, "empty.txt", b"")).level == TEXT_LEVEL + assert choose_level(tmp_path / "missing.log").level == TEXT_LEVEL + + +# --- parsing ------------------------------------------------------------------ + + +def test_parse_level(): + assert parse_level("auto") == AUTO == parse_level(" AUTO ") + assert parse_level(7) == 7 and parse_level("7") == 7 and parse_level("0") == 0 + + +@pytest.mark.parametrize("bad", [10, -1, "high", "", 3.5, True]) +def test_parse_level_rejects_everything_else(bad): + with pytest.raises((OptionsError, TypeError)): + parse_level(bad) + + +def test_level_argument(): + assert level_argument("Auto") == AUTO + assert level_argument("9") == 9 + with pytest.raises(Exception, match="invalid level"): + level_argument("fast") + + +# --- in a real run ------------------------------------------------------------ + + +@pytest.fixture +def mixed(tmp_path): + folder = tmp_path / "mixed" + make(folder, "notes.txt", b"hello world\n" * 3000) + make(folder, "video.mp4", b"\x00\x00\x00\x18ftypisom" + os.urandom(20_000)) + make(folder, "blob.dat", bytes(range(1, 255)) * 80) # no signature, no extension + make(folder, "report.pdf", b"%PDF-1.4\n" + b"text " * 5000) + return folder + + +def pack_auto(mixed, tmp_path, **kwargs): + files = sorted(mixed.iterdir()) + output = tmp_path / "archives" + return bv.pack(files, output, engine="native", level="auto", **kwargs) + + +def manifest_levels(tmp_path): + text = (tmp_path / "archives" / "manifest.json").read_text(encoding="utf-8") + return {e["name"]: e["level"] for e in json.loads(text)["entries"]} + + +def test_each_file_gets_the_level_for_its_type(tmp_path, mixed): + result = pack_auto(mixed, tmp_path) + assert result.status == bv.Status.SUCCESS + + assert manifest_levels(tmp_path) == { + "notes.txt": TEXT_LEVEL, + "video.mp4": 0, + "blob.dat": DEFAULT_LEVEL, + "report.pdf": PDF_LEVEL, + } + assert {e.name: e.level for e in result.entries} == manifest_levels(tmp_path) + + +def test_the_result_says_which_level_was_chosen(tmp_path, mixed): + by_name = {e.name: e for e in pack_auto(mixed, tmp_path).entries} + + assert "level 6 (text)" in by_name["notes.txt"].message + assert "level 0 (already compressed)" in by_name["video.mp4"].message + assert "level 5 (unknown type)" in by_name["blob.dat"].message + + +def test_a_fixed_level_is_unchanged_and_not_annotated(tmp_path, mixed): + result = bv.pack(sorted(mixed.iterdir()), tmp_path / "archives", + engine="native", level=7) # fmt: skip + assert set(manifest_levels(tmp_path).values()) == {7} + assert all("level" not in e.message for e in result.entries) + + +def test_auto_files_round_trip(tmp_path, mixed): + pack_auto(mixed, tmp_path) + restored = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert restored.status == bv.Status.SUCCESS + for path in mixed.iterdir(): + assert (tmp_path / "out" / path.name).read_bytes() == path.read_bytes() + + +def test_update_recognises_files_packed_with_auto(tmp_path, mixed): + pack_auto(mixed, tmp_path) + again = pack_auto(mixed, tmp_path, update=True) + assert {e.message for e in again.entries} == {"Up to date."} + + +def test_a_change_of_type_based_level_repacks_under_update(tmp_path, mixed): + # Packed with a fixed level first, then auto: the text file's level differs. + bv.pack(sorted(mixed.iterdir()), tmp_path / "archives", engine="native", level=5) + result = pack_auto(mixed, tmp_path, update=True) + states = {e.name: e.status for e in result.entries} + + assert states["notes.txt"] == bv.Status.SUCCESS # 5 -> 6 + assert states["blob.dat"] == bv.Status.SKIPPED # unknown type is level 5 already + + +def test_dry_run_shows_the_plan(tmp_path, mixed): + result = pack_auto(mixed, tmp_path, dry_run=True) + by_name = {e.name: e for e in result.entries} + + expected = "Would archive at level 0 (already compressed)." + assert by_name["video.mp4"].message == expected + assert by_name["notes.txt"].level == TEXT_LEVEL + assert not (tmp_path / "archives").exists() + + +def test_store_ignores_auto(tmp_path, mixed): + result = pack_auto(mixed, tmp_path, format="store") + assert all("level" not in e.message for e in result.entries) + + +def test_invalid_levels_are_refused(tmp_path, mixed): + for bad in ("fast", 12, -1): + with pytest.raises(OptionsError): + bv.pack(sorted(mixed.iterdir()), tmp_path / "a", engine="native", level=bad) + + +# --- command line and configuration ------------------------------------------- + + +def run(*args): + return main([*args, "--no-log"]) + + +def test_command_line_accepts_auto(tmp_path, mixed, capsys): + code = run("pack", str(mixed / "notes.txt"), str(mixed / "video.mp4"), "-o", + str(tmp_path / "a"), "--engine", "native", "--level", "AUTO", + "--json", "-q") # fmt: skip + assert code == 0 + result = json.loads(capsys.readouterr().out) + assert {e["name"]: e["level"] for e in result["entries"]} == { + "notes.txt": TEXT_LEVEL, + "video.mp4": 0, + } + + +@pytest.mark.parametrize("bad", ["fast", "10", "-1"]) +def test_command_line_rejects_other_levels(tmp_path, mixed, bad): + with pytest.raises(SystemExit) as exc: + run("pack", str(mixed / "notes.txt"), "-o", str(tmp_path / "a"), "--level", bad) + assert exc.value.code == 2 + + +def test_config_can_set_auto_and_rejects_nonsense(tmp_path, mixed): + config = tmp_path / "bv.toml" + source = str(mixed / "video.mp4") + + config.write_text('[pack]\nengine = "native"\nlevel = "auto"\n') + assert run("pack", source, "-o", str(tmp_path / "a"), "-q", + "--config", str(config)) == 0 # fmt: skip + entry = json.loads((tmp_path / "a" / "manifest.json").read_text("utf-8")) + assert entry["entries"][0]["level"] == 0 + + for bad in ('"fast"', "12", "-1"): + config.write_text(f'[pack]\nlevel = {bad}\n') + assert run("pack", source, "-o", str(tmp_path / "b"), "-q", + "--config", str(config)) == 2 # fmt: skip diff --git a/tests/test_logs.py b/tests/test_logs.py new file mode 100644 index 0000000..965445c --- /dev/null +++ b/tests/test_logs.py @@ -0,0 +1,30 @@ +import os + +from batchvault.logs import default_log_dir, prune_logs +from batchvault.security import redact + + +def test_redact_hides_url_credentials(): + assert redact("push https://user:secret@host/x") == "push https://***@host/x" + assert redact("no secrets here") == "no secrets here" + + +def test_log_dir_can_be_overridden(tmp_path, monkeypatch): + monkeypatch.setenv("BATCHVAULT_LOG_DIR", str(tmp_path)) + assert default_log_dir() == tmp_path + + +def test_prune_keeps_newest(tmp_path): + for number in range(5): + (tmp_path / f"batchvault-2026010{number}-1.log").write_text("x") + (tmp_path / "other.log").write_text("x") + + prune_logs(tmp_path, keep=2) + + remaining = sorted(p.name for p in tmp_path.iterdir()) + assert remaining == [ + "batchvault-20260103-1.log", + "batchvault-20260104-1.log", + "other.log", + ] + assert os.path.exists(tmp_path / "other.log") diff --git a/tests/test_manifest.py b/tests/test_manifest.py new file mode 100644 index 0000000..5a2fd65 --- /dev/null +++ b/tests/test_manifest.py @@ -0,0 +1,87 @@ +import json + +import pytest + +from batchvault.errors import ManifestError +from batchvault.manifest import ( + Manifest, + ManifestEntry, + VolumeInfo, + hash_file, +) + + +def make_entry(name="a.txt", **overrides): + values = { + "name": name, + "size": 10, + "sha256": "ab" * 32, + "engine": "native", + "format": "xz", + "level": 5, + "volume_size": 1024, + "volumes": [VolumeInfo(f"{name}/{name}.bv.001", 5, "cd" * 32)], + } + values.update(overrides) + return ManifestEntry(**values) + + +def test_save_and_load_roundtrip(tmp_path): + manifest = Manifest( + entries={"a.txt": make_entry(), "b é.txt": make_entry("b é.txt")} + ) + path = tmp_path / "manifest.json" + manifest.save(path) + + loaded = Manifest.load(path) + assert loaded.entries == manifest.entries + assert json.loads(path.read_text(encoding="utf-8"))["schema_version"] == 1 + assert not (tmp_path / "manifest.json.tmp").exists() + + +def test_invalid_json(tmp_path): + path = tmp_path / "manifest.json" + path.write_text("{nope", encoding="utf-8") + with pytest.raises(ManifestError, match="not valid JSON"): + Manifest.load(path) + + +def test_unsupported_schema_version(): + with pytest.raises(ManifestError, match="schema_version"): + Manifest.from_dict({"schema_version": 99, "entries": []}) + + +@pytest.mark.parametrize("name", ["../x", "a/b", "a\\b", "..", ""]) +def test_unsafe_entry_names_are_rejected(name): + data = make_entry("ok").to_dict() | {"name": name} + with pytest.raises(ManifestError, match="unsafe|name"): + Manifest.from_dict({"schema_version": 1, "entries": [data]}) + + +def test_missing_field_is_reported(): + data = make_entry().to_dict() + del data["sha256"] + with pytest.raises(ManifestError, match=r"entries\[0\].*sha256"): + Manifest.from_dict({"schema_version": 1, "entries": [data]}) + + +def test_duplicate_entries_are_rejected(): + data = make_entry().to_dict() + with pytest.raises(ManifestError, match="Duplicate"): + Manifest.from_dict({"schema_version": 1, "entries": [data, data]}) + + +def test_check_volumes(tmp_path): + volume = tmp_path / "v.001" + volume.write_bytes(b"hello") + entry = make_entry(volumes=[VolumeInfo("v.001", 5, hash_file(volume))]) + assert entry.check_volumes(tmp_path) == [] + + volume.write_bytes(b"jello") + assert "checksum" in entry.check_volumes(tmp_path)[0] + + volume.write_bytes(b"hi") + assert "wrong size" in entry.check_volumes(tmp_path)[0] + + volume.unlink() + assert "missing" in entry.check_volumes(tmp_path)[0] diff --git a/tests/test_native.py b/tests/test_native.py new file mode 100644 index 0000000..1941370 --- /dev/null +++ b/tests/test_native.py @@ -0,0 +1,209 @@ +import io +import os +import tarfile + +import pytest + +from batchvault.blocks import sniff_format +from batchvault.engines.native import NativeEngine, _check_member +from batchvault.errors import EngineError, OutputExistsError, VolumeError +from batchvault.options import PackOptions + +# Small blocks and volumes so every test spans several of each. +engine = NativeEngine(block_size=4096) + +COMPRESSED = ["xz", "gzip", "bzip2"] + + +@pytest.fixture +def source(tmp_path): + path = tmp_path / "données 1.bin" + path.write_bytes(os.urandom(20_000)) + return path + + +def pack(source, tmp_path, **kwargs): + kwargs.setdefault("volume_size", 3000) + return engine.pack(source, tmp_path / "archives", PackOptions(**kwargs)) + + +@pytest.mark.parametrize("fmt", ["xz", "gzip", "bzip2", "store"]) +def test_roundtrip(tmp_path, source, fmt): + volumes = pack(source, tmp_path, format=fmt, jobs=3) + + assert len(volumes) > 1 + assert volumes[0].name == "données 1.bin.bv.001" + assert sniff_format(volumes[0]) == fmt + + engine.test(volumes[0]) + + restored = engine.unpack(volumes[-1], tmp_path / "restored") + assert [path.name for path in restored] == [source.name] + assert restored[0].read_bytes() == source.read_bytes() + + +def test_compressible_data_shrinks(tmp_path): + source = tmp_path / "zeros.bin" + source.write_bytes(b"\0" * 100_000) + volumes = pack(source, tmp_path, volume_size=10_000_000) + assert len(volumes) == 1 + assert volumes[0].stat().st_size < 5_000 + + +def test_empty_file(tmp_path): + source = tmp_path / "empty.txt" + source.write_bytes(b"") + volumes = pack(source, tmp_path) + restored = engine.unpack(volumes[0], tmp_path / "restored") + assert restored[0].read_bytes() == b"" + + +@pytest.mark.parametrize("fmt", ["xz", "gzip", "bzip2"]) +def test_joined_volumes_are_a_standard_tar(tmp_path, source, fmt): + volumes = pack(source, tmp_path, format=fmt) + joined = b"".join(volume.read_bytes() for volume in volumes) + + with tarfile.open(fileobj=io.BytesIO(joined), mode="r:*") as tar: + member = tar.getmembers()[0] + assert member.name == source.name + assert tar.extractfile(member).read() == source.read_bytes() + + +def test_existing_archive_is_not_replaced_without_force(tmp_path, source): + pack(source, tmp_path) + + with pytest.raises(OutputExistsError): + pack(source, tmp_path) + + pack(source, tmp_path, force=True) + + +def test_force_removes_stale_volumes(tmp_path, source): + first = pack(source, tmp_path, volume_size=1000) + second = pack(source, tmp_path, volume_size=1_000_000, force=True) + + assert len(second) == 1 + assert not any(volume.exists() for volume in first[1:]) + + +def test_unpack_does_not_overwrite_without_force(tmp_path, source): + volumes = pack(source, tmp_path) + output = tmp_path / "restored" + engine.unpack(volumes[0], output) + + with pytest.raises(OutputExistsError): + engine.unpack(volumes[0], output) + + engine.unpack(volumes[0], output, force=True) + + +def test_no_staging_folders_left_behind(tmp_path, source): + volumes = pack(source, tmp_path) + engine.unpack(volumes[0], tmp_path / "restored") + + for folder in (tmp_path / "archives", tmp_path / "restored"): + assert not [p for p in folder.iterdir() if p.name.startswith(".batchvault-")] + + +@pytest.mark.parametrize("fmt", COMPRESSED) +def test_corrupt_data_is_detected(tmp_path, source, fmt): + volumes = pack(source, tmp_path, format=fmt) + data = bytearray(volumes[1].read_bytes()) + data[len(data) // 2] ^= 0xFF + volumes[1].write_bytes(bytes(data)) + + with pytest.raises(EngineError): + engine.test(volumes[0]) + + +@pytest.mark.parametrize("fmt", COMPRESSED) +def test_truncated_archive_is_detected(tmp_path, source, fmt): + volumes = pack(source, tmp_path, format=fmt) + last = volumes[-1] + last.write_bytes(last.read_bytes()[:-1]) + + with pytest.raises(EngineError): + engine.test(volumes[0]) + + +def test_missing_middle_volume_is_detected(tmp_path, source): + volumes = pack(source, tmp_path) + volumes[1].unlink() + + with pytest.raises(VolumeError): + engine.unpack(volumes[0], tmp_path / "restored") + + +def test_missing_source(tmp_path): + with pytest.raises(FileNotFoundError): + pack(tmp_path / "nope.txt", tmp_path) + + +def test_unsafe_archive_entry_is_rejected(tmp_path): + archive = tmp_path / "evil.bv.001" + with tarfile.open(archive, "w") as tar: + info = tarfile.TarInfo("../escape.txt") + info.size = 1 + tar.addfile(info, io.BytesIO(b"x")) + + with pytest.raises(EngineError, match="unsafe"): + engine.unpack(archive, tmp_path / "restored") + assert not (tmp_path / "escape.txt").exists() + + +# --- names that are unsafe to write ------------------------------------------ + + +def _member(name, kind=tarfile.REGTYPE): + info = tarfile.TarInfo(name) + info.type = kind + return info + + +@pytest.mark.parametrize( + "name", + ["C:foo", "file.txt:stream", "NUL", "con.txt", "Aux", "COM1", "lpt9.log", + "name.", "name ", 'a"b', "a str: + buffer = io.StringIO() + console = Console(file=buffer, width=160, force_terminal=False, color_system=None) + ui.print_summary(console, result, None, kind) + return buffer.getvalue() + + +def pack(media, tmp_path, **kwargs): + return bv.pack( + [media / "notes.txt", media / "data.bin"], + tmp_path / "archives", + engine="native", + volume_size="8k", + **kwargs, + ) + + +def test_pack_messages_say_what_was_made(tmp_path, media): + result = pack(media, tmp_path, verify=True) + by_name = {e.source.name: e for e in result.entries} + + notes = by_name["notes.txt"] + assert notes.archive.name == "notes.txt.bv.001" + assert notes.stored_size == sum(v.stat().st_size for v in notes.volumes) + assert notes.message.startswith(f"1 volume, {format_size(30_000)} -> ") + assert "%)" in notes.message and notes.message.endswith(", verified") + + assert " volumes, " in by_name["data.bin"].message # random data needs several + + +def header_of(text: str) -> str: + return next(line for line in text.splitlines() if "Source" in line) + + +def assert_columns_in_order(header: str, *names: str) -> None: + positions = [header.index(name) for name in names] + assert positions == sorted(positions), header + + +def test_pack_table_has_source_archive_status_and_details(tmp_path, media): + text = render(pack(media, tmp_path), "pack") + + assert_columns_in_order(header_of(text), "Source", "Archive", "Status", "Details") + assert "Pack: source files -> archives" in text + assert "notes.txt.bv.001" in text and "->" in text + + +def test_unpack_table_names_the_archive_and_the_restored_file(tmp_path, media): + pack(media, tmp_path) + result = bv.unpack(tmp_path / "archives", tmp_path / "out") + text = render(result, "unpack") + + assert_columns_in_order(header_of(text), "Source", "Archive", "Status", "Details") + assert "Unpack: archives -> restored files" in text + assert "notes.txt.bv.001" in text # the archive + assert "restored, checksum verified" in text + + # The row names the restored file first and its archive second. + row = next(line for line in text.splitlines() if "notes.txt.bv.001" in line) + assert row.index("notes.txt ") < row.index("notes.txt.bv.001") + + entry = next(e for e in result.entries if e.restored[0].name == "notes.txt") + assert entry.source.name == "notes.txt.bv.001" + assert entry.to_dict()["restored"] == [str(entry.restored[0])] + + +def test_verify_says_what_was_checked(tmp_path, media): + pack(media, tmp_path) + + plain = bv.verify(tmp_path / "archives") + assert all("checksums and integrity test OK" in e.message for e in plain.entries) + + against = bv.verify(tmp_path / "archives", against=media) + assert all(e.message.endswith("matches the original") for e in against.entries) + + (tmp_path / "archives" / "manifest.json").unlink() + bare = bv.verify(tmp_path / "archives") + assert all("no manifest" in e.message for e in bare.entries) + + text = render(against, "verify") + assert "Verify: archives" in text + assert_columns_in_order(header_of(text), "Source", "Archive", "Status", "Details") + + row = next(line for line in text.splitlines() if "notes.txt.bv.001" in line) + assert row.index("notes.txt ") < row.index("notes.txt.bv.001") + assert "OK" in row and "matches the original" in row + + +def test_unpack_without_manifest_says_checksum_not_verified(tmp_path, media): + pack(media, tmp_path) + (tmp_path / "archives" / "manifest.json").unlink() + + result = bv.unpack(tmp_path / "archives", tmp_path / "out") + assert all("checksum not verified" in e.message for e in result.entries) + + +def test_failures_and_skips_keep_their_reason_as_details(): + result = RunResult( + entries=[ + FileResult(Path("a.txt"), Status.SKIPPED, "archive already exists"), + FileResult(Path("b.txt"), Status.FAILED, "disk full"), + ] + ) + text = render(result, "pack") + assert "archive already exists" in text and "disk full" in text + assert "SKIPPED" in text and "FAILED" in text + + +def test_every_result_names_the_original_file(tmp_path, media): + packed = pack(media, tmp_path) + unpacked = bv.unpack(tmp_path / "archives", tmp_path / "out") + verified = bv.verify(tmp_path / "archives") + + expected = {"notes.txt", "data.bin"} + for result in (packed, unpacked, verified): + assert {e.name for e in result.entries} == expected + assert {e.to_dict()["name"] for e in result.entries} == expected + + +def test_the_old_one_word_messages_are_gone(tmp_path, media): + pack(media, tmp_path) + messages = { + e.message + for e in ( + *bv.unpack(tmp_path / "archives", tmp_path / "o").entries, + *bv.verify(tmp_path / "archives").entries, + ) + } + assert not messages & {"ok", "restored"} + + +# --- inspect uses the same panel, colours and layout ------------------------------ + + +def render_inspect(entries, source=None) -> str: + buffer = io.StringIO() + console = Console(file=buffer, width=200, force_terminal=False, color_system=None) + ui.print_inspect(console, entries, source) + return buffer.getvalue() + + +def test_inspect_has_a_panel_a_titled_table_and_a_summary(tmp_path, media): + pack(media, tmp_path) + text = render_inspect(bv.inspect(tmp_path / "archives"), "archives") + + assert "BatchVault inspect" in text # the panel + assert "Archives" in text and "Files" in text and "Size" in text + assert "Inspect: archives" in text # the table title + + header = next(line for line in text.splitlines() if "Source" in line) + for earlier, later in [("Source", "Archive"), ("Archive", "Status"), + ("Status", "Engine"), ("Volumes", "Details")]: # fmt: skip + assert header.index(earlier) < header.index(later) + + assert "notes.txt.bv.001" in text and " OK " in text + assert "Summary: 2 archives" in text + + +def test_inspect_flags_missing_volumes_and_encryption(tmp_path, media): + pack(media, tmp_path) + folder = tmp_path / "archives" / "data.bin" + sorted(folder.glob("*.bv.*"))[-1].unlink() + + text = render_inspect(bv.inspect(tmp_path / "archives")) + assert "MISSING" in text and "some volumes are missing" in text + assert "1 with missing volumes" in text + + +def test_inspect_without_a_manifest_says_what_is_unknown(tmp_path, media): + pack(media, tmp_path) + (tmp_path / "archives" / "manifest.json").unlink() + + text = render_inspect(bv.inspect(tmp_path / "archives")) + assert "no manifest" in text and "MISSING" not in text diff --git a/tests/test_volume_io.py b/tests/test_volume_io.py new file mode 100644 index 0000000..ae60834 --- /dev/null +++ b/tests/test_volume_io.py @@ -0,0 +1,46 @@ +import os + +from batchvault.blocks import BlockWriter, compress_block +from batchvault.volumes import VolumeReader, VolumeWriter + + +def test_writer_rolls_over_and_reader_joins(tmp_path): + data = os.urandom(10_500) + + with VolumeWriter(tmp_path, "x.bv", 4000) as writer: + writer.write(data[:5000]) + writer.write(data[5000:]) + + assert [v.name for v in writer.volumes] == ["x.bv.001", "x.bv.002", "x.bv.003"] + assert [v.stat().st_size for v in writer.volumes] == [4000, 4000, 2500] + + with VolumeReader(writer.volumes) as reader: + assert reader.read() == data + + +def test_writer_with_no_data_creates_no_volumes(tmp_path): + with VolumeWriter(tmp_path, "x.bv", 100) as writer: + pass + assert writer.volumes == [] + + +def test_block_writer_keeps_order_across_threads(tmp_path): + blocks = [bytes([i]) * 1000 for i in range(20)] + + with VolumeWriter(tmp_path, "x.bv", 10_000_000) as sink: + with BlockWriter(sink, "store", 5, 1000, jobs=4) as writer: + for block in blocks: + writer.write(block) + + assert sink.volumes[0].read_bytes() == b"".join(blocks) + + +def test_block_writer_flushes_partial_last_block(tmp_path): + with VolumeWriter(tmp_path, "x.bv", 10_000_000) as sink: + with BlockWriter(sink, "gzip", 5, 1000, jobs=2) as writer: + writer.write(b"a" * 1500) + + expected = compress_block("gzip", 5, b"a" * 1000) + compress_block( + "gzip", 5, b"a" * 500 + ) + assert sink.volumes[0].read_bytes() == expected diff --git a/tests/test_volumes.py b/tests/test_volumes.py new file mode 100644 index 0000000..6c15719 --- /dev/null +++ b/tests/test_volumes.py @@ -0,0 +1,45 @@ +import pytest + +from batchvault.errors import VolumeError +from batchvault.volumes import existing_volumes, find_volumes, split_volume_name + + +def make_volumes(directory, base, numbers): + paths = [] + for number in numbers: + path = directory / f"{base}.{number:03d}" + path.write_bytes(b"x") + paths.append(path) + return paths + + +def test_split_volume_name(): + assert split_volume_name("a.mp4.7z.003") == ("a.mp4.7z", 3) + assert split_volume_name("a.bv.001") == ("a.bv", 1) + assert split_volume_name("a.txt") is None + + +def test_find_volumes_from_any_volume(tmp_path): + paths = make_volumes(tmp_path, "intro.mp4.7z", [1, 2, 3]) + assert find_volumes(paths[1]) == paths + + +def test_find_volumes_ignores_other_sets(tmp_path): + paths = make_volumes(tmp_path, "a.txt.7z", [1, 2]) + make_volumes(tmp_path, "b.txt.7z", [1]) + assert find_volumes(paths[0]) == paths + + +def test_find_volumes_detects_gap(tmp_path): + paths = make_volumes(tmp_path, "a.7z", [1, 3]) + with pytest.raises(VolumeError, match="002"): + find_volumes(paths[0]) + + +def test_find_volumes_rejects_non_volume(tmp_path): + with pytest.raises(VolumeError): + find_volumes(tmp_path / "notes.txt") + + +def test_existing_volumes_missing_directory(tmp_path): + assert existing_volumes(tmp_path / "nope", "a.7z") == [] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..2f726cd --- /dev/null +++ b/uv.lock @@ -0,0 +1,523 @@ +version = 1 +revision = 5 +requires-python = ">=3.11.4" + +[[package]] +name = "batchvault" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pathspec" }, + { name = "rich" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[package.dev-dependencies] +dev = [ + { name = "cryptography" }, + { name = "pre-commit" }, + { name = "pytest" }, + { name = "pytest-xdist" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", marker = "extra == 'crypto'", specifier = ">=42" }, + { name = "pathspec", specifier = ">=0.12" }, + { name = "rich", specifier = ">=13" }, +] +provides-extras = ["crypto"] + +[package.metadata.requires-dev] +dev = [ + { name = "cryptography", specifier = ">=42" }, + { name = "pre-commit", specifier = ">=3" }, + { name = "pytest", specifier = ">=8" }, + { name = "pytest-xdist", specifier = ">=3" }, + { name = "ruff", specifier = ">=0.6" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/d2/16d99a0c4948febc0ebd133a13b2f688ff7f8cb04da971e1128872ce0c03/cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12", size = 183838, upload-time = "2026-08-03T21:19:29.637Z" }, + { url = "https://files.pythonhosted.org/packages/cd/95/31b535a9f0220ae9f357de4a08d57ce89cb417653c2fd9f075f50822a388/cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1", size = 184168, upload-time = "2026-08-03T21:19:30.764Z" }, + { url = "https://files.pythonhosted.org/packages/ad/5a/4707a0dc1f203f5dde5a907b0d4e3c25d71120241048bd5bc6f1bb9d4e71/cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0", size = 211805, upload-time = "2026-08-03T21:19:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/66/c19feabb28485b6e0bbaaafa90837a1ef5d302e90f2178bd33f17a49879b/cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813", size = 218716, upload-time = "2026-08-03T21:19:32.896Z" }, + { url = "https://files.pythonhosted.org/packages/a7/92/500760486c8baab49a7a8a58ba7fc3355ec3974b454b8a09e528efde9e1d/cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990", size = 205569, upload-time = "2026-08-03T21:19:34.142Z" }, + { url = "https://files.pythonhosted.org/packages/a5/a7/a67c733254d6e7373f7822f8082d8d6beade791e0cf12a7611f376fa61c7/cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af", size = 204907, upload-time = "2026-08-03T21:19:35.174Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a4/4399daaf8f7dfee9d7c3327fdb0426ee041cc63edc358b93911ceb2bfc7a/cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632", size = 217807, upload-time = "2026-08-03T21:19:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/28/f7/dabe6da2466ecbd82dc62e7342dc6b1065dad990c06f00f0ede9ebf2a0ed/cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd", size = 221252, upload-time = "2026-08-03T21:19:37.416Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/616202d8e51342c07d2534c510111c4cc37201775ce8f60802c9335d1edd/cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a", size = 214214, upload-time = "2026-08-03T21:19:38.507Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c6/ab025d75d2c26c19b087c0124e75ee31cb65032f4fe345d356d8c507ab97/cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa", size = 219408, upload-time = "2026-08-03T21:19:39.809Z" }, + { url = "https://files.pythonhosted.org/packages/db/e2/7e8109f65445bdc673a7b54f02c677de462db75674220fd1335efc8eb598/cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3", size = 174470, upload-time = "2026-08-03T21:19:41.246Z" }, + { url = "https://files.pythonhosted.org/packages/73/c0/77ba02423c2f7d7091143c45cd49e0e6575c4c1967394bb542bd923a9b74/cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0", size = 185096, upload-time = "2026-08-03T21:19:42.615Z" }, + { url = "https://files.pythonhosted.org/packages/7c/47/9f1f85f9672ceda4984dc6c4f8824e8558992a2972c3d3c81fb8eb28d4ba/cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455", size = 179941, upload-time = "2026-08-03T21:19:43.747Z" }, + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/ce/cb/52eb3770c0d0be2702a98c6e96065ddc0a2877cf0845aa9c23397c142cd4/cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8", size = 3892731, upload-time = "2026-09-30T14:44:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/19/8e/aa1fc533d4546b127b45de8aa024eb5933d23eff9debfe25931e56861095/cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047", size = 4710431, upload-time = "2026-09-30T14:44:15.427Z" }, + { url = "https://files.pythonhosted.org/packages/6a/64/72bc3f75176e7e406b748a3e3830432b8c51297b38368713df04dc04898a/cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539", size = 4694824, upload-time = "2026-09-30T14:44:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c6/62c77550edfa5ca3f14bf44a1e6739b9fa09d6e998a11d97ed8213bccc98/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1", size = 4716967, upload-time = "2026-09-30T14:44:19.661Z" }, + { url = "https://files.pythonhosted.org/packages/f4/37/cce70f150c432914460157a6ecc161752e053aa5ec0ef3b3f7dc6e31039a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7", size = 5328676, upload-time = "2026-09-30T14:44:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/6f2f0304d634ceafdeaf23e84537336664ac419b5d07611675c2ad3f6b7a/cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18", size = 4727698, upload-time = "2026-09-30T14:44:24.178Z" }, + { url = "https://files.pythonhosted.org/packages/1d/de/66bcf9244d118663b2e1aaded8990f4640e3d7b7411870a5765f252074d2/cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37", size = 4354821, upload-time = "2026-09-30T14:44:26.263Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/db28a28c7b6c676addce89136de3d8db49ea825a8c863472e36e42ead4ad/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2", size = 4716748, upload-time = "2026-09-30T14:44:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/30/96/01546c7f69ea0e2ab790a2e4f0934a4052fb9b388147fbf83c2fd72f1e57/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1", size = 5285085, upload-time = "2026-09-30T14:44:30.704Z" }, + { url = "https://files.pythonhosted.org/packages/6c/01/03263395f74d50b071e9e66daace3f8bef80493e5d410726f2ba8554736b/cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05", size = 4727268, upload-time = "2026-09-30T14:44:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/2bfe8f29ec0cc9c0d99359c4161adf32858e4934b72c6d100d2ac0bbe962/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e", size = 4849503, upload-time = "2026-09-30T14:44:34.969Z" }, + { url = "https://files.pythonhosted.org/packages/54/44/e80651ecbf0e42b62e2bb5f5768916e07eea72e1297338956a61df361f88/cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e", size = 5004057, upload-time = "2026-09-30T14:44:37.064Z" }, + { url = "https://files.pythonhosted.org/packages/f8/cc/1d33befb3cd7ea7e77d2d73f43f2066471da1b21f24a6156efcaabf6d2e8/cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45", size = 3795868, upload-time = "2026-09-30T14:44:39.71Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, + { url = "https://files.pythonhosted.org/packages/1d/7a/f08d34ce09d60f89ebd391e2ebc6ba2b995e6dd7552f41820f8085f94e53/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67", size = 4716035, upload-time = "2026-09-30T15:29:48.681Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/e18fb65592451a2acb76e9f2fbe14e0f47a8318b4c5430f1633851d03daa/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a", size = 4726917, upload-time = "2026-09-30T15:29:50.608Z" }, + { url = "https://files.pythonhosted.org/packages/83/28/38fdce17e60f6b825e69fc3b7f75e70a6612759980704697e1de4cbfaf6e/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48", size = 4715341, upload-time = "2026-09-30T15:29:52.522Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b1/d9121a717e0f893c64bd6ca7702614778d7df2a5c309128a002421788516/cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42", size = 4726322, upload-time = "2026-09-30T15:29:54.263Z" }, + { url = "https://files.pythonhosted.org/packages/36/8b/e6d153808bf353e152abd2fd4d8f09670d956ac78379ac46e60d7efbf04c/cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81", size = 3873097, upload-time = "2026-09-30T15:29:56.097Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1d/1271f287ff7170ddafc2aad36260c4eec20ccd2fea70f38455e9d56d427b/cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452", size = 3805376, upload-time = "2026-09-30T15:29:58.729Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/d36966de7304047a1001a7559fbf266c464bb5d8cc85464536ef8c9bcc32/distlib-0.4.4.tar.gz", hash = "sha256:ca4175b4ac7f71223cc30ed7248c45d2b38c2439d8ee6e498ab67ed63cf3508d", size = 621304, upload-time = "2026-10-16T13:56:34.203Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/2b/238f02a4927fa2e27aa995371d240a468b52e42eae44a678434aff736b38/distlib-0.4.4-py2.py3-none-any.whl", hash = "sha256:cf09d55b1e85a89576b036aeb71550bfd68320f3f21761ca329eadadad918638", size = 472099, upload-time = "2026-10-16T13:56:32.053Z" }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "filelock" +version = "4.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/28/3893324bcadf8f89b1616637468d766af3f625008c0735bf704fdfa86617/filelock-4.2.4.tar.gz", hash = "sha256:c6d8663535e1650ee430507b4ffbac7b464dd64b7647ea0112a9f406499d163d", size = 652444, upload-time = "2026-10-19T01:57:51.001Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/1b/5e74289a1878e666bb4b65fe5805a04cafcac2aa5f519ef1ead67ef91c56/filelock-4.2.4-py3-none-any.whl", hash = "sha256:301241438c57936ef93f4243a94f2559022268bfcf9b375a9a72eb0f8db8358e", size = 162475, upload-time = "2026-10-19T01:57:49.419Z" }, +] + +[[package]] +name = "identify" +version = "2.6.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/35/d70c0006c7cee65999ea94a6273e60b2094f600a3d8b71b04318253fc643/identify-2.6.20.tar.gz", hash = "sha256:ad729860a923858d26917c2f4fb0a1d83d27a75b1e090c06440c573f048f3285", size = 99681, upload-time = "2026-09-26T20:29:07.187Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/70/fffc9613501877c0a10a1ef73a165e6ded3e53f6e7502227fef56d973933/identify-2.6.20-py2.py3-none-any.whl", hash = "sha256:6a16b69b93187244e0548cbfd25b3e4a6f9a7a2ad784625c3bec2b8d27b81aaa", size = 99506, upload-time = "2026-09-26T20:29:06.054Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/105de02c1322cfada6d9710d9146ef8026419d433c9d08359a2d35805811/nodeenv-1.11.0.tar.gz", hash = "sha256:3ce8fe5b71d16e8af7039ca65257354100bc772965d6bc549070649e53b1b146", size = 99301, upload-time = "2026-09-26T11:29:21.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/c8/12811c9b48fde162bb72b6f2e78fada9a247a09d7bb5be2050a5d099c77b/nodeenv-1.11.0-py2.py3-none-any.whl", hash = "sha256:edaa16e6c14d7cf395d75d4bbd5a26390f4dc06501a33b4e76282b02cc688a25", size = 34277, upload-time = "2026-09-26T11:29:19.933Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/c3/6965c2690ddce885ec75121572d065a5b94c5a5d8d3e657927ecda9fd715/pre_commit-4.7.0.tar.gz", hash = "sha256:1f3deee12b914c8fa184bd2b5953391c2e4f36763e403d4a23dc9983fa7c6e86", size = 198120, upload-time = "2026-10-12T20:45:40.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/83/ce0a196e4f7b20b11ce3aa1fc72fb973f64b7628d51c1b8aebad37fb7f81/pre_commit-4.7.0-py2.py3-none-any.whl", hash = "sha256:2e229038ad3656081b70c27913157e4b636ef306f643f024a8a8e3f0f7b2f877", size = 226261, upload-time = "2026-10-12T20:45:38.858Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[[package]] +name = "python-discovery" +version = "1.6.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/ee/cf608372c88ce2c136ee13c69ab6f667dbc0d0c1176af188b4ddc8853e66/python_discovery-1.6.4.tar.gz", hash = "sha256:bc70d31f0ec5de11eda13d4d64c16e24b3506a65984684f2b6c98cb83b2ea9df", size = 89883, upload-time = "2026-10-19T01:45:14.855Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/91/1aa70d3dfcab0dbcf88a2cbc3bce7f7c69562287a90216e617eaa4937e25/python_discovery-1.6.4-py3-none-any.whl", hash = "sha256:530179cc51a1816d512b15635b3dc931295ae12efb183783ad31190a5d425d2b", size = 39385, upload-time = "2026-10-19T01:45:13.668Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/58/690cf56da80c8f2c8bf12042b6017beac2c34fe961f65420bcb3fb7fac38/ruff-0.17.2.tar.gz", hash = "sha256:84d632402d14c23b488f941db23712d6736a013225162062d34558c3f3c780eb", size = 5333400, upload-time = "2026-10-22T18:38:00.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/b8/8b6d69c57e8fe1cc6af1aa7493b72a1a5b5a8afe34694e302025441729d4/ruff-0.17.2-py3-none-linux_armv6l.whl", hash = "sha256:96a65e755d8dba094707fb23ab602111d9a31ec2835b3a6992a1914a8fcb211e", size = 10183857, upload-time = "2026-10-22T18:37:14.956Z" }, + { url = "https://files.pythonhosted.org/packages/e8/61/12cc2ff7b2fa90a268ce1e2162c23aaf76e601d22623f6a31b0eeaa764b0/ruff-0.17.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:39c5b55c960515cdb32be16e69efb5734c589c06a0eb38a22694524a336b889d", size = 10210904, upload-time = "2026-10-22T18:37:17.985Z" }, + { url = "https://files.pythonhosted.org/packages/e8/7c/b8850a8e4ad29f64a827dd6c6902db4a9aa538fb2b9eef4e26084001a3bb/ruff-0.17.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:92a80d2f44584dabb0189b3a6c393e37e525e04942838482010575cb81945be7", size = 9408439, upload-time = "2026-10-22T18:37:20.964Z" }, + { url = "https://files.pythonhosted.org/packages/b7/b6/6a64fae86ca5e5e29d5adc311e5481de726cedf9aee607435a16c31c442a/ruff-0.17.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7d0906677d89bbf3cf73cde64b5bbe2ab35df3ae43e49ee2627301e91265bac", size = 10047078, upload-time = "2026-10-22T18:37:23.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/19/23d376547715175aa1bd46df48df6aacc7c6439d9daf19cac15975180aa5/ruff-0.17.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cfd783590edf2b77317882631900f6a13ea2e8eacefe6ebf2370d2715299160", size = 9990478, upload-time = "2026-10-22T18:37:26.584Z" }, + { url = "https://files.pythonhosted.org/packages/93/6c/34cda32cbe13adac5c6b291a307a5570d8ebd37c0c86339f0fe628e55632/ruff-0.17.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122258f6ecde0e0dd1bb60b24263c3edde4338caf723fca711cc729f9bfd414a", size = 10673624, upload-time = "2026-10-22T18:37:29.767Z" }, + { url = "https://files.pythonhosted.org/packages/bf/10/5d762a4dbc851d8bb47863cf3a28a61bf31c804d393b70d891fac899f5d1/ruff-0.17.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f8fba429d6ac4128ab29cfb8fa6240d2fa671d04adf7f84d3b22fc7b6b6bea4", size = 11351797, upload-time = "2026-10-22T18:37:32.535Z" }, + { url = "https://files.pythonhosted.org/packages/3f/04/a0e92a81faf5c81c1c98d4374c8401734ccf2b8cca7d0170cb0fec08c6e2/ruff-0.17.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:583bd67bb9e7061dd09bc46a0a884bae2bbd214dbcd48e509813612619683869", size = 10949414, upload-time = "2026-10-22T18:37:35.501Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d8/59894cf630d2d86882457a2a6a0a1533410c5c805bfbac6f6eb4b5aef69d/ruff-0.17.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c60f93fec7142dba5b8f487f94bf2f6231dae24dc11e4c48648347c8b0720004", size = 10528000, upload-time = "2026-10-22T18:37:37.88Z" }, + { url = "https://files.pythonhosted.org/packages/ab/95/d6e30c51e851d37e10762ad91fa1e76ddaad998589b5667962a3cfbc9edd/ruff-0.17.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c555a877f8f41447e318101cfefea9bf12dfa9f82eedf90c376e57e069f917d0", size = 10571779, upload-time = "2026-10-22T18:37:40.444Z" }, + { url = "https://files.pythonhosted.org/packages/f4/5a/a7e1d74fb464250137d3d4eca645d323536533b672004d04ee0e46e0caf3/ruff-0.17.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ba405cbd5ec7bda1f555d5e52be1fd68d02608d4d95d24591bdac92cfacb579f", size = 10051707, upload-time = "2026-10-22T18:37:42.812Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f2/02fca29ca7999380f0350a7212fe755c11ab9c22d61327356cce8decdbc6/ruff-0.17.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8c21b4ab24485358f17291a4f8dc5399b834e963b8efedd920fe4fdb02081bb4", size = 9998227, upload-time = "2026-10-22T18:37:45.224Z" }, + { url = "https://files.pythonhosted.org/packages/d1/00/55aa604a093dfdc1526e570fae0bb7d37a2221c7b4728d5f080b2add16ba/ruff-0.17.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5875a16b0de70b43cb81eabea5ae5e8f5d1b2fd1b5e6ed07f80cfe5019d0bd5a", size = 10377692, upload-time = "2026-10-22T18:37:47.704Z" }, + { url = "https://files.pythonhosted.org/packages/e4/5a/a7a640f47bb906ec8fa233ea84319d503e6dee02b59e3901bd312ebd1f9b/ruff-0.17.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2d086be452eed686aa51271ecf2cbe5037956517cc6ee39a17b4489ac12f8bbe", size = 10790587, upload-time = "2026-10-22T18:37:50.191Z" }, + { url = "https://files.pythonhosted.org/packages/2c/d4/c3b74e1b4808438ea7f14bb914478025e2d51261f60774a54a0fa222ec46/ruff-0.17.2-py3-none-win32.whl", hash = "sha256:6bce7d1f5165701ae18f5df85c65afe78cddfa6c5fb4b9bdb0014d60388f3edd", size = 9870537, upload-time = "2026-10-22T18:37:52.486Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/c7da96c2c96d20704cf9027f7a5121da272a8c05f1d1531ba949e3b3df04/ruff-0.17.2-py3-none-win_amd64.whl", hash = "sha256:1b59d447f197bef9ef48a22ba1a6016137753c6099e9bac9c36eb566e422424c", size = 10282826, upload-time = "2026-10-22T18:37:55.2Z" }, + { url = "https://files.pythonhosted.org/packages/81/60/a3604c79cb02777b6cb84da894104021986f06c4bc41375102cc95049a32/ruff-0.17.2-py3-none-win_arm64.whl", hash = "sha256:195ad918027f817c119310ca89a42d347887dfc28934aaa869325e9c5dc9d962", size = 9827036, upload-time = "2026-10-22T18:37:58.031Z" }, +] + +[[package]] +name = "virtualenv" +version = "21.14.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/aa/bff50daee0488e16d611258a9b6c5a1d0c9e128f8ca424e9f4e1c0444ef2/virtualenv-21.14.13.tar.gz", hash = "sha256:de5307ba4b9a0001d9c8ea02832d38f63ca432880f416c1bface61c4323fd905", size = 5570301, upload-time = "2026-10-22T20:10:16.952Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a9/c10d8a44a29fb94a18b43fdf11f38caca53ae7c9406f11a72b1cb8f624f1/virtualenv-21.14.13-py3-none-any.whl", hash = "sha256:5fbb04138a4861580b78ba9110d529a26bc48fcc93c8ec36bc0202eeed96a788", size = 5557094, upload-time = "2026-10-22T20:10:15.096Z" }, +] From 93b5e52f4bd34b43a97c82101c1ea7c9d7c2f1a3 Mon Sep 17 00:00:00 2001 From: coldsofttech <161084795+coldsofttech@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:33:00 +0100 Subject: [PATCH 2/4] fix: find 7-Zip volumes whose names differ only in Unicode form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS stores accented file names decomposed, so volumes built for a name such as 'données.bin' were not matched. Also adds the E2E tests badge to the README. --- README.md | 1 + src/batchvault/volumes.py | 11 ++++++++++- tests/test_volumes.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 14ba400..107c9b6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # BatchVault [![Tests](https://github.com/coldsofttech/BatchVault/actions/workflows/tests.yml/badge.svg)](https://github.com/coldsofttech/BatchVault/actions/workflows/tests.yml) +[![E2E Tests](https://github.com/coldsofttech/BatchVault/actions/workflows/e2e.yml/badge.svg)](https://github.com/coldsofttech/BatchVault/actions/workflows/e2e.yml) [![Lint](https://github.com/coldsofttech/BatchVault/actions/workflows/lint.yml/badge.svg)](https://github.com/coldsofttech/BatchVault/actions/workflows/lint.yml) [![Release](https://img.shields.io/github/v/release/coldsofttech/BatchVault?logo=github)](https://github.com/coldsofttech/BatchVault/releases) [![PyPI](https://img.shields.io/pypi/v/batchvault?logo=pypi&logoColor=white)](https://pypi.org/project/batchvault/) diff --git a/src/batchvault/volumes.py b/src/batchvault/volumes.py index 5161dc0..c45b23a 100644 --- a/src/batchvault/volumes.py +++ b/src/batchvault/volumes.py @@ -5,6 +5,7 @@ import re import shutil import time +import unicodedata from pathlib import Path from batchvault.errors import VolumeError @@ -21,6 +22,14 @@ def split_volume_name(name: str) -> tuple[str, int] | None: return match.group("base"), int(match.group("number")) +def _same_name(a: str, b: str) -> bool: + """Compare names ignoring case and Unicode form (macOS stores accents decomposed).""" + return ( + unicodedata.normalize("NFC", a).casefold() + == unicodedata.normalize("NFC", b).casefold() + ) + + def existing_volumes(directory: Path, base: str) -> list[Path]: """Volumes of ``base`` already present in ``directory``, in volume order.""" if not directory.is_dir(): @@ -29,7 +38,7 @@ def existing_volumes(directory: Path, base: str) -> list[Path]: found = [] for path in directory.iterdir(): parsed = split_volume_name(path.name) - if path.is_file() and parsed and parsed[0].casefold() == base.casefold(): + if path.is_file() and parsed and _same_name(parsed[0], base): found.append((parsed[1], path)) return [path for _, path in sorted(found)] diff --git a/tests/test_volumes.py b/tests/test_volumes.py index 6c15719..61d51d4 100644 --- a/tests/test_volumes.py +++ b/tests/test_volumes.py @@ -43,3 +43,14 @@ def test_find_volumes_rejects_non_volume(tmp_path): def test_existing_volumes_missing_directory(tmp_path): assert existing_volumes(tmp_path / "nope", "a.7z") == [] + + +def test_existing_volumes_match_names_in_any_unicode_form(tmp_path): + composed = "données.bin.7z" + decomposed = "données.bin.7z" + (tmp_path / f"{decomposed}.001").write_bytes(b"x") + (tmp_path / f"{decomposed}.002").write_bytes(b"x") + + found = existing_volumes(tmp_path, composed) + + assert [path.name for path in found] == [f"{decomposed}.001", f"{decomposed}.002"] From 4febc54e4de9943536fe8021ca825ad62ebb5f8f Mon Sep 17 00:00:00 2001 From: coldsofttech <161084795+coldsofttech@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:33:40 +0100 Subject: [PATCH 3/4] style: shorten docstring to satisfy line length --- src/batchvault/volumes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/batchvault/volumes.py b/src/batchvault/volumes.py index c45b23a..9a80c8a 100644 --- a/src/batchvault/volumes.py +++ b/src/batchvault/volumes.py @@ -23,7 +23,7 @@ def split_volume_name(name: str) -> tuple[str, int] | None: def _same_name(a: str, b: str) -> bool: - """Compare names ignoring case and Unicode form (macOS stores accents decomposed).""" + """Compare names ignoring case and Unicode form (macOS decomposes accents).""" return ( unicodedata.normalize("NFC", a).casefold() == unicodedata.normalize("NFC", b).casefold() From 7291ff629f2a393dfc12a597ab140e7103a7be20 Mon Sep 17 00:00:00 2001 From: coldsofttech <161084795+coldsofttech@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:37:40 +0100 Subject: [PATCH 4/4] fix: restore 7-Zip archives whose names differ only in Unicode form The restore check now ignores Unicode form and restored files use the name recorded in the manifest. Integration tests compare names the same way. --- src/batchvault/api.py | 13 ++++++++++--- tests/integration/test_sevenzip_roundtrip.py | 8 ++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/batchvault/api.py b/src/batchvault/api.py index 75ddcda..694da22 100644 --- a/src/batchvault/api.py +++ b/src/batchvault/api.py @@ -7,6 +7,7 @@ import os import tempfile import time +import unicodedata from collections.abc import Callable, Iterable from concurrent.futures import ThreadPoolExecutor from pathlib import Path @@ -671,7 +672,8 @@ def _restore_one( if entry: names = [path.name for path in restored] - if names != [entry.name]: + # macOS may hand back the same name in another Unicode form. + if [_nfc(n) for n in names] != [_nfc(entry.name)]: return FileResult( volume, Status.FAILED, @@ -687,8 +689,9 @@ def _restore_one( destination_dir.mkdir(parents=True, exist_ok=True) final = [] for path in restored: - os.replace(path, destination_dir / path.name) - final.append(destination_dir / path.name) + name = entry.name if entry else path.name + os.replace(path, destination_dir / name) + final.append(destination_dir / name) size = sum(path.stat().st_size for path in final) checked = "checksum verified" if entry else "checksum not verified" @@ -698,6 +701,10 @@ def _restore_one( return FileResult(volume, Status.FAILED, str(exc)) +def _nfc(name: str) -> str: + return unicodedata.normalize("NFC", name) + + def _matches(path: Path, entry: ManifestEntry) -> bool: return path.is_file() and path.stat().st_size == entry.size and ( hash_file(path) == entry.sha256 diff --git a/tests/integration/test_sevenzip_roundtrip.py b/tests/integration/test_sevenzip_roundtrip.py index c02f0c7..bcdd3d6 100644 --- a/tests/integration/test_sevenzip_roundtrip.py +++ b/tests/integration/test_sevenzip_roundtrip.py @@ -1,4 +1,5 @@ import os +import unicodedata import pytest @@ -26,12 +27,15 @@ def test_roundtrip_with_split_volumes(tmp_path, source): volumes = engine.pack(source, archives, PackOptions(volume_size=4096, jobs=2)) assert len(volumes) > 1 - assert volumes[0].name == "données 1.bin.7z.001" + # macOS may report accents in decomposed form; the name is the same. + assert unicodedata.normalize("NFC", volumes[0].name) == "données 1.bin.7z.001" engine.test(volumes[0]) restored = engine.unpack(volumes[2], tmp_path / "restored") - assert [p.name for p in restored] == [source.name] + assert [unicodedata.normalize("NFC", p.name) for p in restored] == [ + unicodedata.normalize("NFC", source.name) + ] assert restored[0].read_bytes() == source.read_bytes()