diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdf7022..9dbc24d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: cache-dependency-path: web/package-lock.json - name: Install Python package - run: python -m pip install -e . + run: python -m pip install -e ".[dev]" - name: Run Python contracts run: make test @@ -56,7 +56,7 @@ jobs: run: make web-typecheck web-build - name: Run native Go tests - run: make native-go-test native-vuln-scan-test native-ci-contract-test + run: make native-go-test native-vuln-scan-test native-ci-contract-test native-live-supply-chain-test - name: Run Rust gateway tests run: make native-rust-test @@ -83,7 +83,7 @@ jobs: run: make native-incident-workflow-sample - name: Validate CI contract - run: make native-ci-contract-sample + run: make native-ci-contract-live - name: Refresh evaluation index run: make evaluation-index-sample @@ -124,6 +124,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Normalize image prefix + run: | + LOWER_REPOSITORY="$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + echo "IMAGE_PREFIX=${REGISTRY}/${LOWER_REPOSITORY}" >> "${GITHUB_ENV}" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -145,8 +150,8 @@ jobs: ${{ env.IMAGE_PREFIX }}/tryops-${{ matrix.image.role }}:ci load: ${{ github.event_name == 'pull_request' }} push: ${{ github.event_name != 'pull_request' }} - sbom: true - provenance: true + sbom: ${{ github.event_name != 'pull_request' }} + provenance: ${{ github.event_name != 'pull_request' }} - name: Prepare image evidence directories run: mkdir -p artifacts/eval/ci/sbom artifacts/eval/ci/trivy artifacts/eval/ci/signatures @@ -159,12 +164,13 @@ jobs: output-file: artifacts/eval/ci/sbom/${{ matrix.image.role }}.spdx.json - name: Run Trivy image scan - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@v0.36.0 with: image-ref: ${{ env.IMAGE_PREFIX }}/tryops-${{ matrix.image.role }}:${{ github.sha }} scanners: vuln,secret,misconfig vuln-type: os,library severity: HIGH,CRITICAL + limit-severities-for-sarif: true ignore-unfixed: true format: sarif output: artifacts/eval/ci/trivy/${{ matrix.image.role }}.sarif diff --git a/Dockerfile.api b/Dockerfile.api index db9e5d0..79a7b59 100644 --- a/Dockerfile.api +++ b/Dockerfile.api @@ -7,4 +7,6 @@ COPY configs ./configs RUN pip install --no-cache-dir fastapi uvicorn prometheus-client ENV PYTHONPATH=/app/src EXPOSE 8080 +RUN addgroup --system tryops && adduser --system --ingroup tryops --home /nonexistent --no-create-home tryops +USER tryops CMD ["uvicorn", "tryops.api:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Dockerfile.benchmark b/Dockerfile.benchmark index e88bbe4..44a7a5c 100644 --- a/Dockerfile.benchmark +++ b/Dockerfile.benchmark @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS go-build +FROM golang:1.26.4-alpine AS go-build WORKDIR /src COPY native/go/tryops-benchmark/go.mod ./ @@ -22,5 +22,9 @@ COPY --from=go-build /out/tryops_benchmark /usr/local/bin/tryops_benchmark COPY --from=gateway-build /build/target/release/tryops-gateway /usr/local/bin/tryops-gateway RUN pip install --no-cache-dir fastapi uvicorn prometheus-client ENV PYTHONPATH=/app/src +RUN addgroup --system tryops && adduser --system --ingroup tryops --home /nonexistent --no-create-home tryops \ + && mkdir -p /tmp/tryops-benchmark-output \ + && chown -R tryops:tryops /tmp/tryops-benchmark-output +USER tryops ENTRYPOINT ["tryops_benchmark"] CMD ["--gateway-bin", "/usr/local/bin/tryops-gateway", "--python-bin", "python", "--output", "/tmp/tryops-benchmark-output/native_gateway_benchmark.json"] diff --git a/Dockerfile.controller b/Dockerfile.controller index 17e5480..15ce913 100644 --- a/Dockerfile.controller +++ b/Dockerfile.controller @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS build +FROM golang:1.26.4-alpine AS build WORKDIR /src COPY native/go/tryops-controller/go.mod ./ diff --git a/Dockerfile.cpp-tools b/Dockerfile.cpp-tools index 718070f..0470db5 100644 --- a/Dockerfile.cpp-tools +++ b/Dockerfile.cpp-tools @@ -14,7 +14,10 @@ FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates libstdc++6 \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system tryops \ + && useradd --system --gid tryops --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin tryops COPY --from=build /out/tryops_* /usr/local/bin/ +USER tryops ENTRYPOINT ["/bin/sh", "-lc"] CMD ["ls -1 /usr/local/bin/tryops_*"] diff --git a/Dockerfile.gateway b/Dockerfile.gateway index e61659a..187b1ff 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -18,7 +18,9 @@ FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system tryops \ + && useradd --system --gid tryops --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin tryops COPY --from=builder /build/target/release/tryops-gateway /usr/local/bin/tryops-gateway COPY --from=web-builder /web/dist /opt/tryops/web @@ -26,5 +28,6 @@ COPY configs/api_keys.json /opt/tryops/configs/api_keys.json ENV TRYOPS_GATEWAY_STATIC_DIR=/opt/tryops/web ENV TRYOPS_GATEWAY_API_KEYS_PATH=/opt/tryops/configs/api_keys.json EXPOSE 8081 +USER tryops CMD ["tryops-gateway"] diff --git a/Dockerfile.guardrail b/Dockerfile.guardrail index 75c4050..d675ea6 100644 --- a/Dockerfile.guardrail +++ b/Dockerfile.guardrail @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS build +FROM golang:1.26.4-alpine AS build WORKDIR /src COPY native/go/tryops-guardrail/go.mod ./ @@ -9,4 +9,5 @@ FROM scratch COPY --from=build /out/tryops_guardrail /tryops_guardrail EXPOSE 18083 +USER 65532:65532 ENTRYPOINT ["/tryops_guardrail", "serve"] diff --git a/Dockerfile.mlflow b/Dockerfile.mlflow index c2f68a1..5bc740e 100644 --- a/Dockerfile.mlflow +++ b/Dockerfile.mlflow @@ -1,3 +1,4 @@ FROM ghcr.io/mlflow/mlflow:v2.14.3 RUN pip install --no-cache-dir boto3 psycopg2-binary +USER 1000 diff --git a/Dockerfile.web-assets b/Dockerfile.web-assets index 4a1e0b8..dbcaf16 100644 --- a/Dockerfile.web-assets +++ b/Dockerfile.web-assets @@ -7,8 +7,11 @@ COPY web/index.html web/tsconfig.json web/tsconfig.node.json web/vite.config.ts COPY web/src ./src RUN npm run build -FROM nginx:alpine +FROM nginxinc/nginx-unprivileged:alpine +USER root +RUN apk upgrade --no-cache libcrypto3 libssl3 COPY --from=build /web/dist /usr/share/nginx/html -EXPOSE 80 -HEALTHCHECK --interval=10s --timeout=3s --retries=5 CMD wget -qO- http://127.0.0.1/ >/dev/null || exit 1 +EXPOSE 8080 +USER nginx +HEALTHCHECK --interval=10s --timeout=3s --retries=5 CMD wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1 diff --git a/MLOPS_VTON_LLM_ENTERPRISE_ROADMAP.md b/MLOPS_VTON_LLM_ENTERPRISE_ROADMAP.md index de6ecff..d7e1497 100644 --- a/MLOPS_VTON_LLM_ENTERPRISE_ROADMAP.md +++ b/MLOPS_VTON_LLM_ENTERPRISE_ROADMAP.md @@ -51,7 +51,7 @@ This is the honesty centerpiece and itself a maturity signal — a platform that | Production boundary | **DONE:** Native boundary is no longer Python-only: Go controller/guardrail sidecar build and smoke (`make native-go-smoke`, `make native-guardrail-smoke`), the Rust Axum gateway builds/smokes, proxies `/api/*`, enforces signed-artifact, quota preflight, and Go-sidecar LLM guardrails at the edge, and seventeen C++ engines cover policy, image metrics, VTON preprocessing/evaluation, perf/SLO, burn-rate, energy, eval stats, online experiment routing/statistics, batch scheduling, model scan, model provenance, OpenLineage validation, GitOps manifest validation, semantic cache, and chaos (`make smoke`). | KServe deploy profile; full-stack production acceptance | | Eval statistics | **DONE (N):** bootstrap-CI hot path in native C++ `tryops_eval_stats`, used by the leaderboard with a Python fallback | judge-vs-rubric κ at scale with a live Claude judge | | Serving throughput | **DONE (measured with native driver):** Go stdlib load generator removes the Python/GIL driver limit. Rust gateway vs FastAPI: `/health` **24,841 vs 1,539 req/s (16.14x)**; direct validated promotion POST **22,261 vs 755 req/s (29.49x)**; full edge proxy POST **699 vs 759 req/s (0.92x)**, explicitly measuring the gateway preflight/proxy hop (`make gateway-benchmark-native`). Legacy Python-driven benchmark remains as historical lower-bound evidence (`make gateway-benchmark`). | external wrk/k6 confirmation; full KServe/vLLM validated path | -| Supply chain | Policy *expects* SBOM + signature fields | Real Syft SBOM, Trivy scan gate, Cosign signing in CI | +| Supply chain | **DONE:** policy, fallback SBOM, GitHub Actions image SBOM/signing gates, and local live Syft/Trivy/Cosign execution are verified (`make native-ci-contract-live`) | Keyless model transparency/Rekor proof for released model artifacts | | Monitoring | Local Prometheus rules + Grafana JSON + offline drift | Live exporters + Alertmanager + sanitized live-traffic drift windows | ## Executive Intent @@ -1003,7 +1003,7 @@ Items use the [Status Legend](#status-legend): `[x]` done & real, `[~]` contract - [x] J007 Add image payload abuse tests based on oversized or malformed files. - [x] J008 Add dependency lockfile. - [x] J009 Generate SBOM if tooling is available. -- [~] J010 Run vulnerability scanning if tooling is available. (`make vulnerability-scan-sample` builds `native/go/tryops-vuln-scan`, runs available `npm audit` for `web/` with 0 vulnerabilities, and emits `tryops.vulnerability_scan.v1`; Trivy/Syft/Grype/pip-audit/gitleaks/osv-scanner/Cosign are recorded as missing production coverage.) +- [x] J010 Run vulnerability scanning if tooling is available. (`make vulnerability-scan-sample` builds `native/go/tryops-vuln-scan`, runs available `npm audit` for `web/` with 0 vulnerabilities, and emits `tryops.vulnerability_scan.v1`; `make native-live-supply-chain-sample` now executes pinned Syft/Trivy/Cosign containers and emits `tryops.live_supply_chain.v1` with 613 Syft packages, 0 HIGH/CRITICAL Trivy findings, and verified Cosign SBOM signature evidence.) - [x] J011 Pin model sources and record licenses. - [x] J012 Record dataset licenses and usage restrictions. - [x] J013 Add model card template. @@ -1218,7 +1218,7 @@ Canonical detailed plan: `docs/production_app_plan.md`. Research refresh: 2026-0 Goal: turn the platform into a **real, runnable enterprise product** an end user actually uses — browser console, backend, database, services, monitoring, dashboard, quota, audit, incident flow — on top of the existing real LLM (R1/R2), real diffusion VTON, native Rust/Go/C++ boundary, and governance spine. -Current readiness: backend/data/native edge are real; the browser Console still needs screenshot/export/accessibility hardening, while production Kubernetes External Secrets sync, live OTLP exporters/full traces, scanner/signing CI, and the production runbook are not done yet. The product must remain `make`-runnable locally with deterministic degraded-mode fallbacks when GPU/network dependencies are unavailable. +Current readiness: backend/data/native edge are real; the browser Console still needs screenshot/export/accessibility hardening, while production Kubernetes External Secrets sync, live OTLP exporters/full traces, and the production runbook are not done yet. The product must remain `make`-runnable locally with deterministic degraded-mode fallbacks when GPU/network dependencies are unavailable. ### Research-backed direction - **Frontend/static serving:** Vite production build -> `web/dist`; local serving through FastAPI `StaticFiles`; production static serving through nginx or Rust edge profile. @@ -1229,16 +1229,16 @@ Current readiness: backend/data/native edge are real; the browser Console still ### Build phases (each real, tested, `make`-runnable) - [x] **P1 — Data layer.** DONE: `src/tryops/db.py` (SQLite, Postgres-compatible SQL) — requests/feedback/jobs/models/audit_log + dashboard rollup, 4 tests, `make db-init`. -- [x] **P2 — Product backend.** FastAPI BFF exposes LLM, VTON, history, request detail, feedback, models, promotion, lineage, and dashboard rollup routes; real paths keep deterministic fallbacks; requests/feedback/models persist to DB; admin/read and promotion scopes are enforced. -- [~] **P3 — Frontend.** PARTIAL: React/Vite/TypeScript Console shell, typed API client, LLM Playground, VTON Studio contract form with persisted side-by-side comparison outputs, Dashboard, Request History, Pipeline Runs, Model Registry, Evaluation evidence plus optimization/Pareto/sustainability panel, Governance, Incident posture plus live bad-candidate and rollback drills, native incident workflow/postmortem evidence loading, API-key session field, RBAC role-aware navigation, degraded banner, and responsive styling are implemented under `web/`; browser upload/download controls, full audit-log UI, live alert workflow controls, screenshots, and accessibility/export hardening remain. +- [x] **P2 — Product backend.** FastAPI BFF exposes LLM, VTON, history, request detail, feedback, models, promotion, lineage, dashboard rollup, native quota, and online-experiment route/analysis routes; real paths keep deterministic fallbacks; requests/feedback/models persist to DB; admin/read and promotion scopes are enforced. +- [~] **P3 — Frontend.** PARTIAL: React/Vite/TypeScript Console shell, typed API client, LLM Playground with direct/canary/A-B/bandit routing, VTON Studio contract form with persisted side-by-side comparison outputs, Dashboard, Request History, Pipeline Runs, Model Registry, Evaluation evidence plus optimization/Pareto/sustainability panel, Experiments board, Governance, Incident posture plus live bad-candidate and rollback drills, native incident workflow/postmortem evidence loading, API-key session field, RBAC role-aware navigation, degraded banner, and responsive styling are implemented under `web/`; browser upload/download controls, full audit-log UI, live alert workflow controls, screenshots, and accessibility/export hardening remain. - [x] **P4 — Services & edge wiring.** Rust gateway reverse-proxies `/api/*` to FastAPI `/v1/*`, adds request IDs, propagates trace context, enforces native rate/payload/signed-artifact/quota controls plus artifact-path preflight, calls the Go guardrail sidecar before LLM generation, and `docker-compose` ties gateway+api+Postgres+Prometheus+Grafana+guardrail+MinIO+MLflow with healthchecks/restart policies. `make app-smoke` now validates the stack through a disposable Compose project so stale local volumes do not taint readiness. - [~] **P5 — Monitoring & dashboard.** API metrics, native Rust gateway metrics, native Go runtime telemetry, OpenTelemetry Collector wiring, native trace/log correlation evidence, and Alertmanager page/ticket routing are in place; remaining work: in-app timeseries, audit-log UI, external pager/chat credentials, and live OTLP exporters for full gateway->API->model trace stitching. -- [~] **P6 — Enterprise hardening.** PARTIAL: Rust edge API-key/JWT auth preflight, viewer/operator/admin RBAC session and nav enforcement, native edge limits, quota admission, optional file quota persistence, opt-in Postgres-backed distributed quota admission with fail-closed behavior, Postgres quota usage upsert mirroring, native Postgres migration/pooling evidence, native Postgres/MinIO backup-restore drill evidence, Valkey-compatible quota counter mirroring, `.env.example` plus Compose secret loading, native Go secret-rotation/workload-identity contract evidence, live Vault KV v2 secret retrieval/rotation evidence, Kubernetes Vault `SecretStore`/`ExternalSecret` manifests, native dependency-lock contract evidence across Python/Node/Rust/Go, native rustls TLS termination, payload limits, Rust->Go guardrail enforcement, native Go full-stack load/SLO evidence, native Go incident workflow/postmortem evidence, and GitHub Actions plus native Go CI/supply-chain contract evidence are in place. Remaining work: production Kubernetes External Secrets sync, live Syft/Trivy/Cosign execution, external k6/locust confirmation, and external error-tracker DSN exercise. +- [~] **P6 — Enterprise hardening.** PARTIAL: Rust edge API-key/JWT auth preflight, viewer/operator/admin RBAC session and nav enforcement, native edge limits, quota admission, optional file quota persistence, opt-in Postgres-backed distributed quota admission with fail-closed behavior, Postgres quota usage upsert mirroring, native Postgres migration/pooling evidence, native Postgres/MinIO backup-restore drill evidence, Valkey-compatible quota counter mirroring, `.env.example` plus Compose secret loading, native Go secret-rotation/workload-identity contract evidence, live Vault KV v2 secret retrieval/rotation evidence, Kubernetes Vault `SecretStore`/`ExternalSecret` manifests, native dependency-lock contract evidence across Python/Node/Rust/Go, native rustls TLS termination, payload limits, Rust->Go guardrail enforcement, native Go full-stack load/SLO evidence, native Go incident workflow/postmortem evidence, GitHub Actions plus native Go CI/supply-chain contract evidence, and local live Syft/Trivy/Cosign execution are in place. Remaining work: production Kubernetes External Secrets sync, external k6/locust confirmation, and external error-tracker DSN exercise. - [~] **P7 — Demo & docs.** PARTIAL: `docs/presentation_outline.md` contains the professor demo path and `make professor-demo-acceptance` now verifies the live bad-candidate gate plus seeded Pareto, energy, full-stack, VTON, lineage, promotion, rollback, and governance artifacts. Remaining work: seed data loader, product runbook, screenshots/walkthrough, and full end-user/admin guide. -- [~] **P8 — Native-first product expansion.** PARTIAL: local P8 rows PA072-PA086 are closed by verified native evidence. Rust gateway now serves `web/dist` with SPA fallback, performs API-key/JWT auth preflight, terminates optional rustls HTTPS, makes native semantic-cache admission decisions and optional C++ vector-cache lookups before FastAPI for LLM generation, mirrors accepted quota usage into Postgres plus Valkey-compatible counters with hashed-tenant usage snapshots, can use Postgres row locks as the shared quota admission ledger when `TRYOPS_GATEWAY_QUOTA_POSTGRES_ADMISSION=true`, falls back to the local quota ledger when a non-admission durable mirror is unavailable, emits the shared native trace/log envelope as JSONL, and has a validated split-image contract with Rust builder/runtime ABI checks for gateway/controller/guardrail/benchmark/C++ tools/API/web assets; native Go modules now cover VTON/LLM job execution, signed audit/webhook event dispatch, full-stack gateway/BFF load generation with SLO reporting, native CI/supply-chain contract validation, native dependency-lock contract validation, native secret-rotation/workload-identity contract validation plus live Vault KV rotation, native incident workflow/postmortem evidence, distributed quota admission validation, config-contract drift checks, TLS termination contract validation, native Postgres migration/pool execution, native Postgres/MinIO backup-restore drills, container-contract validation, quota read-model/showback generation for the BFF dashboard, trace-envelope validation/reporting, observability Collector/correlation validation, Alertmanager routing validation, runtime telemetry export, SLO regression gating over native benchmark output, and a CI-grade Rust/Go/C++ performance-budget report; native C++ VTON preprocessing, image-quality metrics, semantic-cache lookup, and trace-envelope validation now flow through edge/API/evidence paths. Production Kubernetes External Secrets sync, live Syft/Trivy/Cosign execution, external k6/locust confirmation, external error-tracker DSN exercise, and live OTLP exporters remain. +- [~] **P8 — Native-first product expansion.** PARTIAL: local P8 rows PA072-PA086 are closed by verified native evidence. Rust gateway now serves `web/dist` with SPA fallback, performs API-key/JWT auth preflight, terminates optional rustls HTTPS, makes native semantic-cache admission decisions and optional C++ vector-cache lookups before FastAPI for LLM generation, mirrors accepted quota usage into Postgres plus Valkey-compatible counters with hashed-tenant usage snapshots, can use Postgres row locks as the shared quota admission ledger when `TRYOPS_GATEWAY_QUOTA_POSTGRES_ADMISSION=true`, falls back to the local quota ledger when a non-admission durable mirror is unavailable, emits the shared native trace/log envelope as JSONL, and has a validated split-image contract with Rust builder/runtime ABI checks for gateway/controller/guardrail/benchmark/C++ tools/API/web assets; native Go modules now cover VTON/LLM job execution, signed audit/webhook event dispatch, full-stack gateway/BFF load generation with SLO reporting, native CI/supply-chain contract validation plus live Syft/Trivy/Cosign evidence, native dependency-lock contract validation, native secret-rotation/workload-identity contract validation plus live Vault KV rotation, native incident workflow/postmortem evidence, distributed quota admission validation, config-contract drift checks, TLS termination contract validation, native Postgres migration/pool execution, native Postgres/MinIO backup-restore drills, container-contract validation, quota read-model/showback generation for the BFF dashboard, trace-envelope validation/reporting, observability Collector/correlation validation, Alertmanager routing validation, runtime telemetry export, SLO regression gating over native benchmark output, and a CI-grade Rust/Go/C++ performance-budget report; native C++ VTON preprocessing, image-quality metrics, production A/B/bandit routing, semantic-cache lookup, and trace-envelope validation now flow through edge/API/evidence paths. Production Kubernetes External Secrets sync, external k6/locust confirmation, external error-tracker DSN exercise, and live OTLP exporters remain. ### Execution order P3 Console shell -> thin LLM slice -> VTON product slice -> operator control room -> native enterprise boundary -> production profile. Ship the thin vertical slice first: UI -> Rust gateway -> API -> real/fallback LLM -> DB -> feedback -> dashboard -> Grafana. ### Acceptance (Gate: Product Ready) -A user opens the Console in a browser and can, with no terminal, run LLM and VTON requests, see metrics/quota/trace/lineage, submit feedback, review history/dashboard/registry/audit, and as an admin promote or block a model through policy-gated controls. The full stack starts with `make app-up` and is smoke-tested with `make app-smoke`; native edge/job/SLO/config/container/CI/load/trace/observability/alerts/postgres/backup/TLS/performance/quota-read-model/distributed-quota/runtime-telemetry/live-secret evidence comes from `make native-rust-smoke`, `make native-edge-cache-smoke`, `make native-edge-guardrail-smoke`, `make native-quota-ledger-smoke`, `make native-distributed-quota-smoke`, `make native-secret-rotation-live`, `make native-quota-read-model-sample`, `make native-runtime-telemetry-sample`, `make native-observability-contract-sample`, `make native-alertmanager-contract-sample`, `make native-db-migrator-sample`, `make native-backup-restore-sample`, `make native-tls-contract-sample`, `make native-tls-smoke`, `make native-static-smoke`, `make native-job-runner-sample`, `make vton-native-api-sample`, `make gateway-benchmark-native`, `make native-fullstack-load-sample`, `make native-ci-contract-sample`, `make native-slo-gate-sample`, `make native-config-contract-sample`, `make native-container-contract-sample`, `make native-trace-envelope-sample`, and `make native-performance-budget-sample`; backend contracts remain covered by the Python test suite. The professor-demo evidence bundle is gated by `make professor-demo-acceptance`. +A user opens the Console in a browser and can, with no terminal, run LLM and VTON requests, see metrics/quota/trace/lineage, submit feedback, review history/dashboard/registry/audit, and as an admin promote or block a model through policy-gated controls. The full stack starts with `make app-up` and is smoke-tested with `make app-smoke`; native edge/job/SLO/config/container/CI/load/trace/observability/alerts/postgres/backup/TLS/performance/quota-read-model/distributed-quota/runtime-telemetry/live-secret evidence comes from `make native-rust-smoke`, `make native-edge-cache-smoke`, `make native-edge-guardrail-smoke`, `make native-quota-ledger-smoke`, `make native-distributed-quota-smoke`, `make native-secret-rotation-live`, `make native-quota-read-model-sample`, `make native-runtime-telemetry-sample`, `make native-observability-contract-sample`, `make native-alertmanager-contract-sample`, `make native-db-migrator-sample`, `make native-backup-restore-sample`, `make native-tls-contract-sample`, `make native-tls-smoke`, `make native-static-smoke`, `make native-job-runner-sample`, `make vton-native-api-sample`, `make gateway-benchmark-native`, `make native-fullstack-load-sample`, `make native-ci-contract-live`, `make native-slo-gate-sample`, `make native-config-contract-sample`, `make native-container-contract-sample`, `make native-trace-envelope-sample`, and `make native-performance-budget-sample`; backend contracts remain covered by the Python test suite. The professor-demo evidence bundle is gated by `make professor-demo-acceptance`. diff --git a/Makefile b/Makefile index 9e92575..c855f89 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ PYTHONPATH := src CXX ?= g++ CXXFLAGS ?= -std=c++17 -O2 -Wall -Wextra -pedantic CARGO ?= $(shell if command -v cargo >/dev/null 2>&1; then command -v cargo; elif [ -x "$$HOME/.cargo/bin/cargo" ]; then echo "$$HOME/.cargo/bin/cargo"; else echo cargo; fi) +GO_VERSION ?= go1.25.5 +GO_WRAPPER ?= $(CURDIR)/artifacts/tools/go-toolchain-bin/$(GO_VERSION) +GO ?= $(shell if [ -x "$(GO_WRAPPER)" ]; then echo "$(GO_WRAPPER)"; elif command -v go >/dev/null 2>&1; then command -v go; else echo go; fi) DVC ?= artifacts/tools/dvc-venv/bin/dvc GGUF_MODEL ?= artifacts/models/gguf/SmolLM2-135M-Instruct-Q2_K.gguf GGUF_MODEL_URL ?= https://huggingface.co/bartowski/SmolLM2-135M-Instruct-GGUF/resolve/main/SmolLM2-135M-Instruct-Q2_K.gguf @@ -13,6 +16,9 @@ VLLM_MODEL ?= HuggingFaceTB/SmolLM2-135M-Instruct TRYOPS_VAULT_IMAGE ?= hashicorp/vault:1.19 TRYOPS_VAULT_PORT ?= 18200 TRYOPS_VAULT_DEV_TOKEN ?= tryops-dev-root-token +TRYOPS_SYFT_IMAGE ?= anchore/syft:v1.45.1 +TRYOPS_TRIVY_IMAGE ?= aquasec/trivy:0.71.0 +TRYOPS_COSIGN_IMAGE ?= ghcr.io/sigstore/cosign/cosign:v2.4.1 .PHONY: native-gguf-preflight-build native-gguf-preflight-test llm-gguf-preflight-sample .PHONY: native-trace-envelope-cpp-build native-trace-envelope-cpp-test native-trace-envelope-build native-trace-envelope-test native-trace-envelope-sample @@ -28,7 +34,22 @@ TRYOPS_VAULT_DEV_TOKEN ?= tryops-dev-root-token .PHONY: native-db-migrator-build native-db-migrator-test native-db-migrator-sample native-db-migrator-apply .PHONY: native-backup-restore-build native-backup-restore-test native-backup-restore-sample native-backup-restore-live .PHONY: native-tls-cert-sample native-tls-contract-build native-tls-contract-test native-tls-contract-sample native-tls-smoke -.PHONY: ci native-ci-contract-build native-ci-contract-test native-ci-contract-sample +.PHONY: ci native-go-toolchain prepare-container-artifacts native-live-supply-chain-build native-live-supply-chain-test native-live-supply-chain-sample native-ci-contract-build native-ci-contract-test native-ci-contract-sample native-ci-contract-live + +native-go-toolchain: + @if [ ! -x "$(GO_WRAPPER)" ]; then \ + mkdir -p "$(CURDIR)/artifacts/tools/go-toolchain-bin"; \ + GOBIN="$(CURDIR)/artifacts/tools/go-toolchain-bin" go install "golang.org/dl/$(GO_VERSION)@latest"; \ + fi + @$(GO_WRAPPER) version >/dev/null 2>&1 || $(GO_WRAPPER) download + +prepare-container-artifacts: + @set -eu; \ + uid=$$(id -u); gid=$$(id -g); \ + mkdir -p artifacts/app artifacts/cache artifacts/logs artifacts/traces artifacts/runtime artifacts/runtime/vton artifacts/eval/full_stack artifacts/eval/jobs; \ + if ! chown -R "$$uid:$$gid" artifacts/app artifacts/cache artifacts/logs artifacts/traces artifacts/runtime artifacts/eval/full_stack artifacts/eval/jobs 2>/dev/null; then \ + docker run --rm -v "$(CURDIR)/artifacts:/work" alpine:3.20 sh -lc "chown -R $$uid:$$gid /work/app /work/cache /work/logs /work/traces /work/runtime /work/eval/full_stack /work/eval/jobs"; \ + fi web-typecheck: cd web && npm ci && npm run typecheck @@ -36,7 +57,7 @@ web-typecheck: web-build: cd web && npm ci && npm run build -ci: test web-typecheck native-go-test native-rust-test native-cpp-test supply-chain-sample vulnerability-scan-sample native-container-contract-sample native-dependency-lock-contract-sample native-secret-rotation-contract-sample native-incident-workflow-sample native-ci-contract-sample evaluation-index-sample +ci: test web-typecheck native-go-test native-rust-test native-cpp-test supply-chain-sample vulnerability-scan-sample native-container-contract-sample native-dependency-lock-contract-sample native-secret-rotation-contract-sample native-incident-workflow-sample native-ci-contract-live evaluation-index-sample docker compose config --quiet docker compose --profile tls config --quiet @@ -375,17 +396,17 @@ native-trace-envelope-cpp-test: native-guardrail-build: mkdir -p artifacts/native - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-guardrail && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_guardrail_cli .; \ + cd native/go/tryops-guardrail && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_guardrail_cli .; \ else \ echo "go not installed; guardrail sample will use the Python deterministic fallback"; \ fi native-guardrail-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-guardrail && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-guardrail && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native guardrail tests"; \ fi @@ -401,29 +422,30 @@ native-guardrail-smoke: native-guardrail-build echo "metrics:"; curl -fsS http://127.0.0.1:18083/metrics | head -n 8 native-go-build: - cd native/go/tryops-controller && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops-controller . + cd native/go/tryops-controller && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops-controller . -native-go-test: - @if command -v go >/dev/null 2>&1; then \ +native-go-test: native-go-toolchain + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-controller && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-config-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-db-migrator && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-backup-restore && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-tls-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-container-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-distributed-quota && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-quota-read-model && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-runtime-telemetry && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-observability-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-alertmanager-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-incident-workflow && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-secret-rotation-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-dependency-lock-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-performance-budget && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-fullstack-load && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-ci-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ - cd $(CURDIR)/native/go/tryops-trace-envelope && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-controller && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-config-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-db-migrator && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-backup-restore && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-tls-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-container-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-distributed-quota && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-quota-read-model && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-runtime-telemetry && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-observability-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-alertmanager-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-incident-workflow && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-secret-rotation-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-dependency-lock-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-performance-budget && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-fullstack-load && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-ci-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-live-supply-chain && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + cd $(CURDIR)/native/go/tryops-trace-envelope && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native controller tests"; \ fi @@ -442,36 +464,36 @@ native-go-smoke: native-go-build native-data-versioning-build: mkdir -p artifacts/native - cd native/go/tryops-data-versioning && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_data_versioning . + cd native/go/tryops-data-versioning && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_data_versioning . native-data-versioning-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-data-versioning && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-data-versioning && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native data versioning tests"; \ fi native-benchmark-build: mkdir -p artifacts/native - cd native/go/tryops-benchmark && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_benchmark . + cd native/go/tryops-benchmark && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_benchmark . native-benchmark-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-benchmark && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-benchmark && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native benchmark tests"; \ fi native-fullstack-load-build: mkdir -p artifacts/native - cd native/go/tryops-fullstack-load && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_fullstack_load . + cd native/go/tryops-fullstack-load && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_fullstack_load . native-fullstack-load-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-fullstack-load && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-fullstack-load && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native full-stack load tests"; \ fi @@ -481,48 +503,48 @@ native-fullstack-load-sample: native-rust-build native-fullstack-load-build native-vllm-probe-build: mkdir -p artifacts/native - cd native/go/tryops-vllm-probe && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_vllm_probe . + cd native/go/tryops-vllm-probe && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_vllm_probe . native-vllm-probe-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-vllm-probe && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-vllm-probe && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native vLLM probe tests"; \ fi native-quantized-preflight-build: mkdir -p artifacts/native - cd native/go/tryops-quantized-preflight && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_quantized_preflight . + cd native/go/tryops-quantized-preflight && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_quantized_preflight . native-quantized-preflight-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-quantized-preflight && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-quantized-preflight && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native quantized preflight tests"; \ fi native-stack-smoke-build: mkdir -p artifacts/native - cd native/go/tryops-stack-smoke && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_stack_smoke . + cd native/go/tryops-stack-smoke && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_stack_smoke . native-stack-smoke-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-stack-smoke && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-stack-smoke && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native stack smoke tests"; \ fi native-job-runner-build: mkdir -p artifacts/native - cd native/go/tryops-job-runner && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_job_runner . + cd native/go/tryops-job-runner && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_job_runner . native-job-runner-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-job-runner && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-job-runner && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native job runner tests"; \ fi @@ -534,12 +556,12 @@ native-job-runner-sample: native-job-runner-build native-slo-gate-build: mkdir -p artifacts/native - cd native/go/tryops-slo-gate && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_slo_gate . + cd native/go/tryops-slo-gate && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_slo_gate . native-slo-gate-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-slo-gate && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-slo-gate && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native SLO gate tests"; \ fi @@ -549,12 +571,12 @@ native-slo-gate-sample: native-slo-gate-build native-event-dispatcher-build: mkdir -p artifacts/native - cd native/go/tryops-event-dispatcher && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_event_dispatcher . + cd native/go/tryops-event-dispatcher && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_event_dispatcher . native-event-dispatcher-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-event-dispatcher && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-event-dispatcher && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native event dispatcher tests"; \ fi @@ -564,24 +586,24 @@ native-event-dispatcher-sample: native-event-dispatcher-build native-demo-acceptance-build: mkdir -p artifacts/native - cd native/go/tryops-demo-acceptance && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_demo_acceptance . + cd native/go/tryops-demo-acceptance && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_demo_acceptance . native-demo-acceptance-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-demo-acceptance && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-demo-acceptance && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native demo acceptance tests"; \ fi native-demo-recorder-build: mkdir -p artifacts/native - cd native/go/tryops-demo-recorder && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_demo_recorder . + cd native/go/tryops-demo-recorder && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_demo_recorder . native-demo-recorder-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-demo-recorder && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-demo-recorder && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native demo recorder tests"; \ fi @@ -597,24 +619,24 @@ professor-demo-video: native-demo-recorder-build native-vuln-scan-build: mkdir -p artifacts/native - cd native/go/tryops-vuln-scan && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_vuln_scan . + cd native/go/tryops-vuln-scan && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_vuln_scan . native-vuln-scan-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-vuln-scan && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-vuln-scan && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native vulnerability scan tests"; \ fi native-config-contract-build: mkdir -p artifacts/native - cd native/go/tryops-config-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_config_contract . + cd native/go/tryops-config-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_config_contract . native-config-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-config-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-config-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native config contract tests"; \ fi @@ -622,14 +644,43 @@ native-config-contract-test: native-config-contract-sample: native-config-contract-build artifacts/native/tryops_config_contract --root . --output artifacts/eval/config/native_config_contract_report.json +native-live-supply-chain-build: native-go-toolchain + mkdir -p artifacts/native + cd native/go/tryops-live-supply-chain && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_live_supply_chain . + +native-live-supply-chain-test: native-go-toolchain + @if command -v "$(GO)" >/dev/null 2>&1; then \ + mkdir -p artifacts/.gocache; \ + cd native/go/tryops-live-supply-chain && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ + else \ + echo "go not installed; skipping native live supply-chain tests"; \ + fi + +native-live-supply-chain-sample: native-live-supply-chain-build + @set -eu; \ + mkdir -p artifacts/eval/ci/syft artifacts/eval/ci/trivy artifacts/eval/ci/cosign artifacts/.trivycache artifacts/tmp/cosign-live; \ + rm -f artifacts/eval/ci/syft/filesystem.spdx.json artifacts/eval/ci/trivy/filesystem.json artifacts/eval/ci/cosign/sbom.spdx.json.sig artifacts/eval/ci/cosign/tryops-local.pub artifacts/eval/ci/cosign/verify-blob.txt artifacts/eval/ci/cosign/sign-blob.txt artifacts/eval/ci/cosign/generate-key.txt; \ + docker run --rm -v "$(CURDIR)":/work -w /work "$(TRYOPS_SYFT_IMAGE)" version > artifacts/eval/ci/syft/version.txt; \ + docker run --rm -v "$(CURDIR)":/work -w /work "$(TRYOPS_SYFT_IMAGE)" dir:/work --exclude './artifacts/**' --exclude './.git/**' --exclude './.venv/**' -o spdx-json=/work/artifacts/eval/ci/syft/filesystem.spdx.json; \ + docker run --rm -v "$(CURDIR)":/work -w /work -v "$(CURDIR)/artifacts/.trivycache":/root/.cache "$(TRYOPS_TRIVY_IMAGE)" version > artifacts/eval/ci/trivy/version.txt; \ + docker run --rm -v "$(CURDIR)":/work -w /work -v "$(CURDIR)/artifacts/.trivycache":/root/.cache "$(TRYOPS_TRIVY_IMAGE)" fs --scanners vuln,secret,misconfig --severity HIGH,CRITICAL --format json --output /work/artifacts/eval/ci/trivy/filesystem.json --skip-dirs /work/artifacts --skip-dirs /work/.git --skip-dirs /work/.venv --exit-code 0 /work; \ + uidgid="$$(id -u):$$(id -g)"; \ + docker run --rm --user "$$uidgid" -e COSIGN_PASSWORD= -v "$(CURDIR)":/work -w /work "$(TRYOPS_COSIGN_IMAGE)" version > artifacts/eval/ci/cosign/version.txt; \ + docker run --rm --user "$$uidgid" -e COSIGN_PASSWORD= -v "$(CURDIR)":/work -w /work "$(TRYOPS_COSIGN_IMAGE)" generate-key-pair --output-key-prefix artifacts/tmp/cosign-live/tryops-local > artifacts/eval/ci/cosign/generate-key.txt 2>&1; \ + cp artifacts/tmp/cosign-live/tryops-local.pub artifacts/eval/ci/cosign/tryops-local.pub; \ + docker run --rm --user "$$uidgid" -e COSIGN_PASSWORD= -v "$(CURDIR)":/work -w /work "$(TRYOPS_COSIGN_IMAGE)" sign-blob --tlog-upload=false --key artifacts/tmp/cosign-live/tryops-local.key --output-signature artifacts/eval/ci/cosign/sbom.spdx.json.sig artifacts/eval/ci/syft/filesystem.spdx.json > artifacts/eval/ci/cosign/sign-blob.txt 2>&1; \ + docker run --rm --user "$$uidgid" -e COSIGN_PASSWORD= -v "$(CURDIR)":/work -w /work "$(TRYOPS_COSIGN_IMAGE)" verify-blob --insecure-ignore-tlog --key artifacts/eval/ci/cosign/tryops-local.pub --signature artifacts/eval/ci/cosign/sbom.spdx.json.sig artifacts/eval/ci/syft/filesystem.spdx.json > artifacts/eval/ci/cosign/verify-blob.txt 2>&1; \ + rm -rf artifacts/tmp/cosign-live; \ + artifacts/native/tryops_live_supply_chain --root . --output artifacts/eval/ci/live_supply_chain_report.json --syft-image "$(TRYOPS_SYFT_IMAGE)" --trivy-image "$(TRYOPS_TRIVY_IMAGE)" --cosign-image "$(TRYOPS_COSIGN_IMAGE)" + native-ci-contract-build: mkdir -p artifacts/native - cd native/go/tryops-ci-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_ci_contract . + cd native/go/tryops-ci-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_ci_contract . native-ci-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-ci-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-ci-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native CI contract tests"; \ fi @@ -637,14 +688,17 @@ native-ci-contract-test: native-ci-contract-sample: native-ci-contract-build supply-chain-sample vulnerability-scan-sample native-container-contract-sample native-dependency-lock-contract-sample artifacts/native/tryops_ci_contract --root . --output artifacts/eval/ci/native_ci_contract.json +native-ci-contract-live: native-ci-contract-build native-live-supply-chain-sample supply-chain-sample vulnerability-scan-sample native-container-contract-sample native-dependency-lock-contract-sample + artifacts/native/tryops_ci_contract --root . --output artifacts/eval/ci/native_ci_contract.json + native-container-contract-build: mkdir -p artifacts/native - cd native/go/tryops-container-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_container_contract . + cd native/go/tryops-container-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_container_contract . native-container-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-container-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-container-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native container contract tests"; \ fi @@ -654,12 +708,12 @@ native-container-contract-sample: native-container-contract-build native-distributed-quota-build: mkdir -p artifacts/native - cd native/go/tryops-distributed-quota && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_distributed_quota . + cd native/go/tryops-distributed-quota && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_distributed_quota . native-distributed-quota-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-distributed-quota && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-distributed-quota && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native distributed quota tests"; \ fi @@ -703,12 +757,12 @@ native-distributed-quota-smoke: native-rust-build native-distributed-quota-build native-quota-read-model-build: mkdir -p artifacts/native - cd native/go/tryops-quota-read-model && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_quota_read_model . + cd native/go/tryops-quota-read-model && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_quota_read_model . native-quota-read-model-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-quota-read-model && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-quota-read-model && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native quota read model tests"; \ fi @@ -718,12 +772,12 @@ native-quota-read-model-sample: native-quota-read-model-build quota-sample native-runtime-telemetry-build: mkdir -p artifacts/native - cd native/go/tryops-runtime-telemetry && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_runtime_telemetry . + cd native/go/tryops-runtime-telemetry && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_runtime_telemetry . native-runtime-telemetry-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-runtime-telemetry && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-runtime-telemetry && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native runtime telemetry tests"; \ fi @@ -735,12 +789,12 @@ native-runtime-telemetry-sample: native-runtime-telemetry-build native-observability-contract-build: mkdir -p artifacts/native - cd native/go/tryops-observability-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_observability_contract . + cd native/go/tryops-observability-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_observability_contract . native-observability-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-observability-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-observability-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native observability contract tests"; \ fi @@ -750,12 +804,12 @@ native-observability-contract-sample: native-observability-contract-build native native-alertmanager-contract-build: mkdir -p artifacts/native - cd native/go/tryops-alertmanager-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_alertmanager_contract . + cd native/go/tryops-alertmanager-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_alertmanager_contract . native-alertmanager-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-alertmanager-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-alertmanager-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native Alertmanager contract tests"; \ fi @@ -765,12 +819,12 @@ native-alertmanager-contract-sample: native-alertmanager-contract-build native-incident-workflow-build: mkdir -p artifacts/native - cd native/go/tryops-incident-workflow && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_incident_workflow . + cd native/go/tryops-incident-workflow && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_incident_workflow . native-incident-workflow-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-incident-workflow && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-incident-workflow && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native incident workflow tests"; \ fi @@ -780,12 +834,12 @@ native-incident-workflow-sample: native-incident-workflow-build rollback-sample native-secret-rotation-contract-build: mkdir -p artifacts/native - cd native/go/tryops-secret-rotation-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_secret_rotation_contract . + cd native/go/tryops-secret-rotation-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_secret_rotation_contract . native-secret-rotation-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-secret-rotation-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-secret-rotation-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native secret rotation contract tests"; \ fi @@ -823,12 +877,12 @@ native-secret-rotation-live: native-secret-rotation-contract-build native-dependency-lock-contract-build: mkdir -p artifacts/native - cd native/go/tryops-dependency-lock-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_dependency_lock_contract . + cd native/go/tryops-dependency-lock-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_dependency_lock_contract . native-dependency-lock-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-dependency-lock-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-dependency-lock-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native dependency lock contract tests"; \ fi @@ -836,14 +890,14 @@ native-dependency-lock-contract-test: native-dependency-lock-contract-sample: native-dependency-lock-contract-build artifacts/native/tryops_dependency_lock_contract --root . --output artifacts/eval/dependencies/native_dependency_lock_contract.json -native-db-migrator-build: +native-db-migrator-build: native-go-toolchain mkdir -p artifacts/native - cd native/go/tryops-db-migrator && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_db_migrator . + cd native/go/tryops-db-migrator && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_db_migrator . -native-db-migrator-test: - @if command -v go >/dev/null 2>&1; then \ +native-db-migrator-test: native-go-toolchain + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-db-migrator && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-db-migrator && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native Postgres migrator tests"; \ fi @@ -858,12 +912,12 @@ native-db-migrator-apply: native-db-migrator-build native-backup-restore-build: mkdir -p artifacts/native - cd native/go/tryops-backup-restore && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_backup_restore . + cd native/go/tryops-backup-restore && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_backup_restore . native-backup-restore-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-backup-restore && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-backup-restore && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native backup/restore tests"; \ fi @@ -887,12 +941,12 @@ native-tls-cert-sample: native-tls-contract-build: mkdir -p artifacts/native - cd native/go/tryops-tls-contract && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_tls_contract . + cd native/go/tryops-tls-contract && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_tls_contract . native-tls-contract-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-tls-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-tls-contract && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native TLS contract tests"; \ fi @@ -902,12 +956,12 @@ native-tls-contract-sample: native-tls-contract-build native-tls-cert-sample native-performance-budget-build: mkdir -p artifacts/native - cd native/go/tryops-performance-budget && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_performance_budget . + cd native/go/tryops-performance-budget && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_performance_budget . native-performance-budget-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-performance-budget && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-performance-budget && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native performance budget tests"; \ fi @@ -917,12 +971,12 @@ native-performance-budget-sample: native-performance-budget-build native-rust-bu native-trace-envelope-build: mkdir -p artifacts/native - cd native/go/tryops-trace-envelope && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_trace_envelope . + cd native/go/tryops-trace-envelope && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_trace_envelope . native-trace-envelope-test: native-trace-envelope-cpp-test - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-trace-envelope && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-trace-envelope && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native trace envelope Go tests"; \ fi @@ -937,12 +991,12 @@ native-trace-envelope-sample: native-trace-envelope-build native-trace-envelope- native-evaluation-index-build: mkdir -p artifacts/native - cd native/go/tryops-evaluation-index && GOFLAGS=-mod=mod go build -buildvcs=false -o ../../../artifacts/native/tryops_evaluation_index . + cd native/go/tryops-evaluation-index && GOFLAGS=-mod=mod $(GO) build -buildvcs=false -o ../../../artifacts/native/tryops_evaluation_index . native-evaluation-index-test: - @if command -v go >/dev/null 2>&1; then \ + @if command -v "$(GO)" >/dev/null 2>&1; then \ mkdir -p artifacts/.gocache; \ - cd native/go/tryops-evaluation-index && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod go test ./...; \ + cd native/go/tryops-evaluation-index && GOCACHE=$(CURDIR)/artifacts/.gocache GOFLAGS=-mod=mod $(GO) test ./...; \ else \ echo "go not installed; skipping native evaluation index tests"; \ fi @@ -1071,10 +1125,15 @@ gateway-benchmark-native: native-rust-build native-benchmark-build native-tooling: @command -v $(CARGO) >/dev/null 2>&1 && $(CARGO) --version || echo "cargo not installed; build the Rust gateway with 'make native-rust-build' after installing rustup" - @command -v go >/dev/null 2>&1 && go version || echo "go not installed; build the Go controller with 'make native-go-build' after installing go>=1.22" + @command -v "$(GO)" >/dev/null 2>&1 && $(GO) version || echo "go not installed; build the Go controller with 'make native-go-build' after installing go>=1.22" @$(CXX) --version | head -n 1 app-up: evaluation-index-sample + @set -eu; \ + $(MAKE) prepare-container-artifacts; \ + uid=$$(id -u); gid=$$(id -g); \ + TRYOPS_CONTAINER_UID=$$uid \ + TRYOPS_CONTAINER_GID=$$gid \ TRYOPS_POSTGRES_PORT=15432 \ TRYOPS_POSTGRES_USER=$${TRYOPS_POSTGRES_USER:-tryops} \ TRYOPS_POSTGRES_DB=$${TRYOPS_POSTGRES_DB:-tryops} \ @@ -1097,9 +1156,13 @@ app-up: evaluation-index-sample app-smoke: native-stack-smoke-build native-job-runner-build evaluation-index-sample @set -eu; \ project=tryops_app_smoke; \ + $(MAKE) prepare-container-artifacts; \ + uid=$$(id -u); gid=$$(id -g); \ COMPOSE_PROJECT_NAME=$$project docker compose down --volumes --remove-orphans >/dev/null 2>&1 || true; \ trap "COMPOSE_PROJECT_NAME=$$project docker compose down --volumes --remove-orphans >/dev/null 2>&1 || true" EXIT; \ COMPOSE_PROJECT_NAME=$$project \ + TRYOPS_CONTAINER_UID=$$uid \ + TRYOPS_CONTAINER_GID=$$gid \ TRYOPS_POSTGRES_PORT=15432 \ TRYOPS_POSTGRES_USER=$${TRYOPS_POSTGRES_USER:-tryops} \ TRYOPS_POSTGRES_DB=$${TRYOPS_POSTGRES_DB:-tryops} \ diff --git a/configs/container_images.json b/configs/container_images.json index 6660665..84cad19 100644 --- a/configs/container_images.json +++ b/configs/container_images.json @@ -77,7 +77,7 @@ "compose_service": "web-assets", "required_stage": "node", "source_paths": ["web/src", "web/package.json", "web/vite.config.ts"], - "ports": [80] + "ports": [8080] } ] } diff --git a/docker-compose.yml b/docker-compose.yml index fc05009..f299060 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,6 +172,7 @@ services: build: context: . dockerfile: Dockerfile.benchmark + user: "${TRYOPS_CONTAINER_UID:-1000}:${TRYOPS_CONTAINER_GID:-1000}" environment: TRYOPS_BENCHMARK_PROFILE: container volumes: @@ -189,12 +190,13 @@ services: context: . dockerfile: Dockerfile.web-assets ports: - - "${TRYOPS_WEB_ASSETS_PORT:-8088}:80" + - "${TRYOPS_WEB_ASSETS_PORT:-8088}:8080" api: build: context: . dockerfile: Dockerfile.api + user: "${TRYOPS_CONTAINER_UID:-1000}:${TRYOPS_CONTAINER_GID:-1000}" restart: unless-stopped environment: TRYOPS_ENV: local @@ -205,6 +207,8 @@ services: ports: - "${TRYOPS_API_PORT:-8080}:8080" volumes: + - ./artifacts/app:/app/artifacts/app + - ./artifacts/cache:/app/artifacts/cache - ./artifacts/eval:/app/artifacts/eval:ro - ./artifacts/demo:/app/artifacts/demo:ro - ./artifacts/deployments:/app/artifacts/deployments:ro @@ -225,6 +229,7 @@ services: build: context: . dockerfile: Dockerfile.gateway + user: "${TRYOPS_CONTAINER_UID:-1000}:${TRYOPS_CONTAINER_GID:-1000}" restart: unless-stopped environment: TRYOPS_GATEWAY_ADDR: 0.0.0.0:8081 @@ -265,6 +270,7 @@ services: build: context: . dockerfile: Dockerfile.gateway + user: "${TRYOPS_CONTAINER_UID:-1000}:${TRYOPS_CONTAINER_GID:-1000}" restart: unless-stopped environment: TRYOPS_GATEWAY_ADDR: 0.0.0.0:8443 diff --git a/docs/experiment_log.md b/docs/experiment_log.md index 186479f..ff68630 100644 --- a/docs/experiment_log.md +++ b/docs/experiment_log.md @@ -1605,9 +1605,8 @@ Date: 2026-06-11 generates Syft SPDX SBOMs, gates HIGH/CRITICAL Trivy findings, and signs pushed images with Cosign keyless OIDC on non-PR runs. The native Go contract passed 16/16 checks and is surfaced as `ci_contract` in the evaluation index. -- Notes: this partially completes PA062. Local `production_ready=false` because `syft`, `trivy`, and - `cosign` are not installed in this workspace; the GitHub workflow is ready to execute those tools - in CI. +- Notes: this partially completed PA062 at the CI-contract layer. It is superseded by EXP-093, which + adds local live Syft/Trivy/Cosign execution and production-ready native CI evidence. ## EXP-088: Native Secret Rotation And Workload Identity Contract @@ -1650,8 +1649,8 @@ Date: 2026-06-11 - `artifacts/eval/evaluation_index/evaluation_index.json` - Outcome: `uv.lock` pins the Python project resolution, including `accelerate=1.14.0`, `bitsandbytes=0.49.2`, `torch=2.11.0`, `transformers=5.11.0`, and `vllm=0.22.1`. The split Go - verifier checks Python, Console, Rust gateway, and native Go module locks and passed 87/87 checks - with 326 Python packages, 59 Node packages, 228 Rust crates, and 30 Go modules. The evaluation + verifier checks Python, Console, Rust gateway, and native Go module locks and now passes 89/89 + checks with 326 Python packages, 59 Node packages, 228 Rust crates, and 32 Go modules. The evaluation index highlights the report as `dependency_lock`. - Notes: this closes PA063 for local/native reproducibility evidence. The older generated `requirements.lock` remains as a supply-chain fallback artifact, while `uv.lock` is the canonical @@ -1684,10 +1683,10 @@ Date: 2026-06-11 exit. - Notes: this closes PA072-PA086 for local/native P8 evidence. The final smoke passed 18/18 full-stack checks and then passed both native job-runner jobs through the Rust gateway. Live - Syft/Trivy/Cosign execution, external k6/locust confirmation, external error-tracker DSN exercise, - and live OTLP exporters remain production-profile work rather than local P8 gaps; distributed - quota validation is closed separately in EXP-091 and live Vault KV rotation is closed separately - in EXP-092. + External k6/locust confirmation, external error-tracker DSN exercise, and live OTLP exporters + remain production-profile work rather than local P8 gaps; distributed quota validation is closed + separately in EXP-091, live Vault KV rotation is closed separately in EXP-092, and live + Syft/Trivy/Cosign execution is closed separately in EXP-093. ## EXP-091: Distributed Native Quota Admission @@ -1709,8 +1708,8 @@ Date: 2026-06-11 started disposable Postgres plus two Rust gateway processes and passed with exactly 20/32 allowed, 12 rejected, zero request errors, and `no_cluster_quota_oversell=true`. - Notes: this closes the local distributed multi-gateway quota validation gap for P6/P8. Production - still needs live Syft/Trivy/Cosign, external k6/locust, external error-tracker DSN, production - Kubernetes External Secrets sync, and live OTLP exporter exercises. + still needs external k6/locust, external error-tracker DSN, production Kubernetes External Secrets + sync, and live OTLP exporter exercises. Live Syft/Trivy/Cosign execution is closed in EXP-093. ## EXP-092: Live Native Vault KV Rotation @@ -1731,3 +1730,32 @@ Date: 2026-06-11 8 rotated secret properties, token-file auth, Vault 1.19.5 health, and version range 1..2. - Notes: this closes the local live Vault secret fetch/rotation gap. A production Kubernetes cluster should still exercise the External Secrets controller and Kubernetes auth TokenReview flow. + +## EXP-093: Live Syft/Trivy/Cosign Supply-Chain Gate + +- Command: `make native-live-supply-chain-test`; `make native-live-supply-chain-sample`; + `make native-ci-contract-test`; `make native-ci-contract-live`; `make native-evaluation-index-test` +- Workload: PA062 local live SBOM, vulnerability/misconfiguration/secret scan, SBOM blob signing, + and native CI production-readiness contract +- Evidence files: + - `.github/workflows/ci.yml` + - `Makefile` + - `native/go/tryops-live-supply-chain/` + - `native/go/tryops-ci-contract/` + - `native/go/tryops-evaluation-index/` + - `artifacts/eval/ci/syft/filesystem.spdx.json` + - `artifacts/eval/ci/trivy/filesystem.json` + - `artifacts/eval/ci/cosign/sbom.spdx.json.sig` + - `artifacts/eval/ci/cosign/tryops-local.pub` + - `artifacts/eval/ci/cosign/verify-blob.txt` + - `artifacts/eval/ci/live_supply_chain_report.json` + - `artifacts/eval/ci/native_ci_contract.json` +- Outcome: `make native-live-supply-chain-sample` executes pinned open-source container images + `anchore/syft:v1.45.1`, `aquasec/trivy:0.71.0`, and + `ghcr.io/sigstore/cosign/cosign:v2.4.1`, so host PATH installs are not required. The latest live + report passed with 613 Syft packages, 0 HIGH/CRITICAL Trivy vulnerability/misconfiguration/secret + findings, and a Cosign-verified signature over the SPDX SBOM blob. `make native-ci-contract-live` + then passed 17/17 CI checks with `production_ready=true` and `missing_tools=[]`. +- Notes: this closes the local live Syft/Trivy/Cosign execution gap for PA062/J010. The older + host-installed-tool vulnerability runner remains partial by design and records optional missing + Grype, pip-audit, gitleaks, and osv-scanner coverage. diff --git a/docs/known_limitations.md b/docs/known_limitations.md index fa57f21..3d03435 100644 --- a/docs/known_limitations.md +++ b/docs/known_limitations.md @@ -37,7 +37,11 @@ has native evidence for hash-only API-key rotation, Vault/External Secrets manifests, Kubernetes workload identity, and live local Vault KV rotation, but production OIDC/JWKS and External Secrets controller sync still need to be exercised. -- A local SPDX SBOM fallback is generated, `make vulnerability-scan-sample` runs the available `npm audit` check for `web/`, and `.github/workflows/ci.yml` plus `make native-ci-contract-sample` define the Syft/Trivy/Cosign production path, but Syft, Trivy, Grype, pip-audit, gitleaks, osv-scanner, and Cosign are not installed in this workspace. +- A local SPDX SBOM fallback is generated, `make vulnerability-scan-sample` runs the available + `npm audit` check for `web/`, and `make native-ci-contract-live` now executes pinned + Syft/Trivy/Cosign containers with production-ready evidence. Optional secondary scanner coverage + from Grype, pip-audit, gitleaks, and osv-scanner is still not present. - Model provenance uses a local DSSE-shaped digest bundle verified by native C++; real Sigstore keyless OIDC identity and Rekor transparency-log proof are not generated locally yet. -- Local vulnerability evidence is partial: npm audit found 0 web vulnerabilities, but Python, container, OS-package, secret, and misconfiguration scans still need production scanners. +- The legacy host-installed-tool vulnerability report is still partial by design; the production + scanner gate is now the live Trivy filesystem scan inside `tryops.live_supply_chain.v1`. diff --git a/docs/production_app_plan.md b/docs/production_app_plan.md index bd789f5..57e5e89 100644 --- a/docs/production_app_plan.md +++ b/docs/production_app_plan.md @@ -4,7 +4,7 @@ Date added: 2026-06-11. Research refresh: 2026-06-12. Goal: turn the platform into a **real, runnable enterprise product** an end user actually uses — browser console, backend, database, services, monitoring, dashboard, quota, audit, and incident flow — on top of the existing real LLM (R1/R2), real diffusion VTON, native Rust/Go/C++ boundary, and governance spine. -Current readiness: backend/data/native edge are real; the browser Console is functional but still needs screenshot/export/accessibility hardening, while production Kubernetes External Secrets sync, live OTLP exporters/full traces, scanner/signing CI, and the production runbook are not done yet. The product must stay runnable locally through `make`, with deterministic degraded-mode fallbacks when GPU/network dependencies are unavailable. +Current readiness: backend/data/native edge are real; the browser Console is functional but still needs screenshot/export/accessibility hardening, while production Kubernetes External Secrets sync, live OTLP exporters/full traces, and the production runbook are not done yet. The product must stay runnable locally through `make`, with deterministic degraded-mode fallbacks when GPU/network dependencies are unavailable. ### Target architecture @@ -86,13 +86,13 @@ Python stays as BFF/control-plane glue and model adapter code where the current ### Build phases (each real, tested, `make`-runnable) - [x] **P1 — Data layer.** DONE: `src/tryops/db.py` (SQLite, Postgres-compatible SQL) — requests/feedback/jobs/models/audit_log + dashboard rollup, 4 tests, `make db-init`. -- [x] **P2 — Product backend.** FastAPI exposes the product BFF routes for LLM, VTON, history, request detail, feedback, models, model promotion, lineage, dashboard rollups, and native quota summaries; real LLM/VTON paths keep deterministic fallbacks, product events persist to DB, and admin/read plus promotion scopes are tested. -- [~] **P3 — Frontend.** PARTIAL: React/Vite/TypeScript Console shell, left nav, typed API client, LLM Playground, VTON Studio contract form plus persisted side-by-side comparison outputs, Dashboard with native quota/showback read model, Request History, Pipeline Runs, Model Registry, Evaluation evidence plus optimization/Pareto/sustainability panel, Governance, Incident posture plus live bad-candidate and rollback drills, native incident workflow/postmortem evidence loading, Professor Demo mode, API-key session field, RBAC role-aware navigation, degraded banner, and responsive styling are implemented under `web/`; browser upload/download controls, full audit-log UI, live alert workflow controls, screenshots, and accessibility/export hardening remain. +- [x] **P2 — Product backend.** FastAPI exposes the product BFF routes for LLM, VTON, history, request detail, feedback, models, model promotion, lineage, dashboard rollups, native quota summaries, and online-experiment routing/analysis; real LLM/VTON paths keep deterministic fallbacks, product events persist to DB, and admin/read plus promotion scopes are tested. +- [~] **P3 — Frontend.** PARTIAL: React/Vite/TypeScript Console shell, left nav, typed API client, LLM Playground with direct/canary/A-B/bandit routing modes, VTON Studio contract form plus persisted side-by-side comparison outputs, Dashboard with native quota/showback read model, Request History, Pipeline Runs, Model Registry, Evaluation evidence plus optimization/Pareto/sustainability panel, Experiments board, Governance, Incident posture plus live bad-candidate and rollback drills, native incident workflow/postmortem evidence loading, Professor Demo mode, API-key session field, RBAC role-aware navigation, degraded banner, and responsive styling are implemented under `web/`; browser upload/download controls, full audit-log UI, live alert workflow controls, screenshots, and accessibility/export hardening remain. - [x] **P4 — Services & edge wiring.** DONE: modular Rust gateway reverse-proxies `/api/*` to FastAPI `/v1/*`, adds request IDs, propagates trace context, enforces edge limits/preflight plus artifact-path preflight, calls the modular Go guardrail sidecar before LLM generation, and `docker-compose` ties gateway+api+db+prometheus+grafana+guardrail+minio+mlflow with healthchecks plus `make app-up`, `make app-smoke`, and `make app-down`. - [~] **P5 — Monitoring & dashboard.** PARTIAL: API metrics, native Rust gateway metrics, native Go runtime telemetry for LLM tokens/sec plus live NVIDIA GPU memory/utilization/power, OpenTelemetry Collector wiring, native trace/log correlation, and Alertmanager page/ticket routing evidence now emit Prometheus/JSONL evidence; in-app dashboard timeseries, audit-log UI, external pager/chat credentials, and live OTLP exporters for full gateway->API->model trace stitching remain. -- [~] **P6 — Enterprise hardening.** PARTIAL: Rust edge API-key/JWT auth preflight, viewer/operator/admin RBAC session and nav enforcement, edge rate limits, native quota admission, optional file quota persistence, opt-in Postgres-backed distributed quota admission with fail-closed startup behavior, Postgres durable quota upsert mirroring, native Postgres migration/pooling evidence, native Postgres/MinIO backup-restore drill evidence, Valkey-compatible quota counter mirroring, `.env.example` plus Compose secret loading, native Go secret-rotation/workload-identity contract evidence, live Vault KV v2 secret retrieval/rotation evidence, Kubernetes Vault `SecretStore`/`ExternalSecret` manifests, native dependency-lock contract evidence across Python/Node/Rust/Go, native Rust TLS termination, payload limits, Rust->Go LLM guardrail enforcement, native Go full-stack load/SLO evidence, native Go incident workflow/postmortem evidence, and a GitHub Actions plus native Go CI/supply-chain contract are in place; production Kubernetes External Secrets sync, live Syft/Trivy/Cosign execution, external k6/locust confirmation, and external error-tracker DSN exercise remain. +- [~] **P6 — Enterprise hardening.** PARTIAL: Rust edge API-key/JWT auth preflight, viewer/operator/admin RBAC session and nav enforcement, edge rate limits, native quota admission, optional file quota persistence, opt-in Postgres-backed distributed quota admission with fail-closed startup behavior, Postgres durable quota upsert mirroring, native Postgres migration/pooling evidence, native Postgres/MinIO backup-restore drill evidence, Valkey-compatible quota counter mirroring, `.env.example` plus Compose secret loading, native Go secret-rotation/workload-identity contract evidence, live Vault KV v2 secret retrieval/rotation evidence, Kubernetes Vault `SecretStore`/`ExternalSecret` manifests, native dependency-lock contract evidence across Python/Node/Rust/Go, native Rust TLS termination, payload limits, Rust->Go LLM guardrail enforcement, native Go full-stack load/SLO evidence, native Go incident workflow/postmortem evidence, GitHub Actions plus native Go CI/supply-chain contract, and local live Syft/Trivy/Cosign execution are in place; production Kubernetes External Secrets sync, external k6/locust confirmation, and external error-tracker DSN exercise remain. - [~] **P7 — Demo & docs.** PARTIAL: `docs/presentation_outline.md` defines the professor demo path, the Console has a seeded Professor Demo view, `make professor-demo-acceptance` verifies the platform evidence bundle, and `make professor-demo-video` records a deterministic offline MP4 backup. Seed loader, product runbook, and full user/admin guides remain. -- [~] **P8 — Native-first product expansion.** PARTIAL: local P8 rows PA072-PA086 are now closed by verified native evidence. Rust gateway serves `web/dist` with SPA fallback, performs API-key/JWT auth preflight, terminates optional rustls HTTPS, makes native semantic-cache admission decisions plus optional C++ vector-cache lookups before FastAPI for LLM generation, mirrors accepted quota usage into Postgres plus Valkey-compatible counters with tenant usage snapshots, can use Postgres row locks as the shared quota admission ledger when `TRYOPS_GATEWAY_QUOTA_POSTGRES_ADMISSION=true`, degrades to the local quota ledger when a non-admission durable mirror is unavailable, emits a shared native trace/log envelope as JSONL, and has split container image definitions with Rust builder/runtime ABI compatibility checks for gateway/controller/guardrail/benchmark/C++ tools/API/web assets; native Go modules now cover VTON/LLM job execution, signed audit/webhook event dispatch, registry-webhook C++ policy enforcement, full-stack gateway/BFF load generation with SLO reporting, native CI/supply-chain contract validation, native secret-rotation/workload-identity contract validation plus live Vault KV rotation, native incident workflow/postmortem evidence, distributed quota admission validation, SLO regression gating over native benchmark output, config-contract drift checks, TLS termination contract validation, Postgres migration/pool execution, Postgres/MinIO backup-restore drills, container-contract validation, trace-envelope validation/reporting, observability Collector/correlation validation, runtime telemetry export for LLM throughput/GPU memory, quota read-model/showback generation for the BFF dashboard, a CI-grade Rust/Go/C++ performance-budget report, GPTQ/AWQ model/runtime preflight, and vLLM OpenAI-compatible endpoint probing; native C++ now also validates real GGUF artifacts for CPU-first LLM deployment preflight, validates trace envelopes, and feeds semantic-cache lookup plus VTON preprocessing/image-quality metrics through edge/API/job paths. Live Syft/Trivy/Cosign execution, external k6/locust confirmation, external error-tracker DSN exercise, live GPTQ/AWQ loading, live llama.cpp generation, live vLLM serving, production Kubernetes External Secrets sync, and live OTLP exporters remain. +- [~] **P8 — Native-first product expansion.** PARTIAL: local P8 rows PA072-PA086 are now closed by verified native evidence. Rust gateway serves `web/dist` with SPA fallback, performs API-key/JWT auth preflight, terminates optional rustls HTTPS, makes native semantic-cache admission decisions plus optional C++ vector-cache lookups before FastAPI for LLM generation, mirrors accepted quota usage into Postgres plus Valkey-compatible counters with tenant usage snapshots, can use Postgres row locks as the shared quota admission ledger when `TRYOPS_GATEWAY_QUOTA_POSTGRES_ADMISSION=true`, degrades to the local quota ledger when a non-admission durable mirror is unavailable, emits a shared native trace/log envelope as JSONL, and has split container image definitions with Rust builder/runtime ABI compatibility checks for gateway/controller/guardrail/benchmark/C++ tools/API/web assets; native Go modules now cover VTON/LLM job execution, signed audit/webhook event dispatch, registry-webhook C++ policy enforcement, full-stack gateway/BFF load generation with SLO reporting, native CI/supply-chain contract validation plus live Syft/Trivy/Cosign evidence, native secret-rotation/workload-identity contract validation plus live Vault KV rotation, native incident workflow/postmortem evidence, distributed quota admission validation, SLO regression gating over native benchmark output, config-contract drift checks, TLS termination contract validation, Postgres migration/pool execution, Postgres/MinIO backup-restore drills, container-contract validation, trace-envelope validation/reporting, observability Collector/correlation validation, runtime telemetry export for LLM throughput/GPU memory, quota read-model/showback generation for the BFF dashboard, a CI-grade Rust/Go/C++ performance-budget report, GPTQ/AWQ model/runtime preflight, and vLLM OpenAI-compatible endpoint probing; native C++ now also validates real GGUF artifacts for CPU-first LLM deployment preflight, validates trace envelopes, powers production A/B and guarded bandit routing through FastAPI/React, and feeds semantic-cache lookup plus VTON preprocessing/image-quality metrics through edge/API/job paths. External k6/locust confirmation, external error-tracker DSN exercise, live GPTQ/AWQ loading, live llama.cpp generation, live vLLM serving, production Kubernetes External Secrets sync, and live OTLP exporters remain. ### Acceptance (Gate: Product Ready) @@ -102,7 +102,7 @@ A user opens the Console in a browser and can, with no terminal: - Upload person+garment assets, run VTON, compare inputs/output, download through a persisted local/MinIO object URL, and inspect lineage. - Review request history, request detail, feedback, dashboard rollups, model registry, governance/lineage, and audit log. - As an admin/operator, promote or block a model through signed/policy-gated controls and see the action in metrics, audit, and incident views. -- Start the full local stack with `make app-up`; prove startup with `make app-smoke`; prove the native edge/job/SLO/config/container/CI/dependencies/performance/load/trace/observability/alerts/secrets/live-secrets/postgres/backup/TLS/quota-read-model/distributed-quota/runtime-telemetry/incident layer with `make native-rust-smoke`, `make native-edge-cache-smoke`, `make native-edge-guardrail-smoke`, `make native-quota-ledger-smoke`, `make native-distributed-quota-smoke`, `make native-quota-read-model-sample`, `make native-runtime-telemetry-sample`, `make native-observability-contract-sample`, `make native-alertmanager-contract-sample`, `make native-dependency-lock-contract-sample`, `make native-secret-rotation-contract-sample`, `make native-secret-rotation-live`, `make native-incident-workflow-sample`, `make native-db-migrator-sample`, `make native-backup-restore-sample`, `make native-tls-contract-sample`, `make native-tls-smoke`, `make native-static-smoke`, `make native-job-runner-sample`, `make vton-native-api-sample`, `make gateway-benchmark-native`, `make native-fullstack-load-sample`, `make native-ci-contract-sample`, `make native-slo-gate-sample`, `make native-config-contract-sample`, `make native-container-contract-sample`, `make native-trace-envelope-sample`, and `make native-performance-budget-sample`; prove the native CPU-first LLM artifact path with `make llm-gguf-preflight-sample`; prove GPTQ/AWQ candidate/runtime readiness with `make llm-quantized-preflight-sample`; prove vLLM serving readiness or explicit skip status with `make llm-vllm-probe-sample`; prove the professor demo evidence with `make professor-demo-acceptance`; prove backend contracts with `PYTHONPATH=src python -m unittest discover -s tests`. +- Start the full local stack with `make app-up`; prove startup with `make app-smoke`; prove the native edge/job/SLO/config/container/CI/dependencies/performance/load/trace/observability/alerts/secrets/live-secrets/postgres/backup/TLS/quota-read-model/distributed-quota/runtime-telemetry/incident layer with `make native-rust-smoke`, `make native-edge-cache-smoke`, `make native-edge-guardrail-smoke`, `make native-quota-ledger-smoke`, `make native-distributed-quota-smoke`, `make native-quota-read-model-sample`, `make native-runtime-telemetry-sample`, `make native-observability-contract-sample`, `make native-alertmanager-contract-sample`, `make native-dependency-lock-contract-sample`, `make native-secret-rotation-contract-sample`, `make native-secret-rotation-live`, `make native-incident-workflow-sample`, `make native-db-migrator-sample`, `make native-backup-restore-sample`, `make native-tls-contract-sample`, `make native-tls-smoke`, `make native-static-smoke`, `make native-job-runner-sample`, `make vton-native-api-sample`, `make gateway-benchmark-native`, `make native-fullstack-load-sample`, `make native-ci-contract-live`, `make native-slo-gate-sample`, `make native-config-contract-sample`, `make native-container-contract-sample`, `make native-trace-envelope-sample`, and `make native-performance-budget-sample`; prove the native CPU-first LLM artifact path with `make llm-gguf-preflight-sample`; prove GPTQ/AWQ candidate/runtime readiness with `make llm-quantized-preflight-sample`; prove vLLM serving readiness or explicit skip status with `make llm-vllm-probe-sample`; prove the professor demo evidence with `make professor-demo-acceptance`; prove backend contracts with `PYTHONPATH=src python -m unittest discover -s tests`. - Open the Console Professor Demo view and walk the seeded no-network/no-GPU path across quota, optimization, VTON quality, lineage, rollback, and governance without relying on live model service availability. --- @@ -184,8 +184,8 @@ Evidence note: `tests/test_api_surface.py` verifies `/api/history`, `/api/reques - [x] PA059 DB backup/restore scripts + scheduled restore drill. (`native/go/tryops-backup-restore/` validates Compose storage, `infra/backup/restore_drill.cron`, restore isolation, and required tooling; `make native-backup-restore-sample` emits 20/20 plan checks; live drill with `TRYOPS_POSTGRES_BACKUP_DSN=... make native-backup-restore-live` wrote `artifacts/eval/backup/native_backup_restore_live.json` with 50/50 checks, a 42,609-byte Postgres custom dump restored into `tryops_restore_drill`, seven table row-count matches, MinIO `mc mirror` backup/restore of one object into `tryops-restore-drill`, and cleanup of temporary restore targets.) - [~] PA060 Secrets management (vault/workload identity), key rotation. (`configs/secret_rotation_policy.json` declares Vault KV paths, owners, rotation windows, hash-only API-key rotation, and SPIFFE-ready workload identity; `infra/kubernetes/secret-management/` adds a Vault `SecretStore`, `ExternalSecret`, non-automounted `tryops-runtime` ServiceAccount, and projected service-account token for the gateway; `native/go/tryops-secret-rotation-contract/` validates the plan with parsed JSON/YAML and can exercise live Vault KV v2 write/read/rotation through `make native-secret-rotation-live`. Latest local live evidence passes 65/65 checks with 8 managed secrets, 5 live KV paths, 8 rotated secret properties, Vault 1.19.5 health, token-file auth, versioning 1->2, no raw secret emission, and `production_ready=true`; production Kubernetes External Secrets controller sync remains.) - [x] PA061 TLS/HTTPS termination config for the production profile. (The Rust gateway reads `TRYOPS_GATEWAY_TLS_CERT_PATH`/`TRYOPS_GATEWAY_TLS_KEY_PATH` and serves HTTPS through axum-server/rustls; `docker-compose.yml` adds optional `gateway-tls` profile with `TRYOPS_TLS_CERT_PEM`/`TRYOPS_TLS_KEY_PEM` Compose secrets; `native/go/tryops-tls-contract/` validates the profile. `make native-tls-contract-sample` emits 24/24 plan checks, and `make native-tls-smoke` emits 30/30 live checks with TLS1.3, HTTPS `/health` 200, and plaintext HTTP rejection.) -- [~] PA062 `make ci`: lint + tests + build images + SBOM + Trivy scan + Cosign sign. (`.github/workflows/ci.yml` now defines Python/Node/Go/Rust/C++ tests, Compose validation, native contract evidence, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL image scan gate, artifact upload, and Cosign keyless signing on non-PR pushes. `make ci` mirrors the local test/evidence path, and `native/go/tryops-ci-contract/` emits `artifacts/eval/ci/native_ci_contract.json` with 16/16 checks passing. Local `production_ready=false` because Syft, Trivy, and Cosign are not installed in this workspace.) -- [x] PA063 Dependency lockfile (fix the accelerate/bitsandbytes drift caught in the benchmark run). (`uv.lock` now pins the Python project resolution including `accelerate=1.14.0`, `bitsandbytes=0.49.2`, `torch=2.11.0`, `transformers=5.11.0`, and `vllm=0.22.1`; `web/package-lock.json`, Rust `Cargo.lock`, and Go `go.sum` files cover the other runtimes. `native/go/tryops-dependency-lock-contract/` validates all four ecosystems and emits `artifacts/eval/dependencies/native_dependency_lock_contract.json`; latest local evidence passes 87/87 checks with 326 Python packages, 59 Node packages, 228 Rust crates, and 30 Go modules.) +- [x] PA062 `make ci`: lint + tests + build images + SBOM + Trivy scan + Cosign sign. (`.github/workflows/ci.yml` defines Python/Node/Go/Rust/C++ tests, Compose validation, native contract evidence, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL image scan gate, artifact upload, and Cosign keyless signing on non-PR pushes. `make ci` now mirrors the local live evidence path through `make native-ci-contract-live`; `native/go/tryops-live-supply-chain/` executes pinned Syft/Trivy/Cosign containers and emits `artifacts/eval/ci/live_supply_chain_report.json` with 613 Syft packages, 0 HIGH/CRITICAL Trivy findings, and verified Cosign SBOM signature evidence. `native/go/tryops-ci-contract/` now emits `artifacts/eval/ci/native_ci_contract.json` with 17/17 checks, `missing_tools=[]`, and `production_ready=true`.) +- [x] PA063 Dependency lockfile (fix the accelerate/bitsandbytes drift caught in the benchmark run). (`uv.lock` now pins the Python project resolution including `accelerate=1.14.0`, `bitsandbytes=0.49.2`, `torch=2.11.0`, `transformers=5.11.0`, and `vllm=0.22.1`; `web/package-lock.json`, Rust `Cargo.lock`, and Go `go.sum` files cover the other runtimes. `native/go/tryops-dependency-lock-contract/` validates all four ecosystems and emits `artifacts/eval/dependencies/native_dependency_lock_contract.json`; latest local evidence passes 89/89 checks with 326 Python packages, 59 Node packages, 228 Rust crates, and 32 Go modules.) - [~] PA064 Full-stack load test (native Go driver plus k6/locust confirmation) with SLOs + report. (`native/go/tryops-fullstack-load/` starts FastAPI plus the Rust gateway, drives six weighted product scenarios through `/api/*`, applies per-scenario SLOs, records k6/locust availability, and emits `artifacts/eval/load/native_fullstack_load.json`; latest local run passed 6/6 scenarios with 504 requests, zero errors, worst p95 39.965 ms, and `external_ready=false` because neither `k6` nor `locust` is installed.) - [~] PA065 Error tracking + incident workflow + postmortem template. (`native/go/tryops-incident-workflow/` emits `artifacts/eval/incidents/native_incident_workflow.json` with OTel-shaped local error event fields, Alertmanager payload validation, controller/event-dispatcher checks, rollback linkage, a 5-step detected->triaged->mitigated->postmortem_drafted->resolved timeline, and `docs/incident_postmortem_template.md` rendering to `artifacts/eval/incidents/postmortem_bad_candidate.md`. Latest local evidence passes 8/8 checks; `production_ready=false` until `GLITCHTIP_DSN`, `TRYOPS_ERROR_TRACKING_DSN`, or `SENTRY_DSN` is configured and exercised.) @@ -221,7 +221,7 @@ Evidence note: `tests/test_api_surface.py` verifies `/api/history`, `/api/reques 3. VTON product slice: PA026-PA027, PA045, PA079, PA081. 4. Operator views: PA030-PA034, PA054, PA066-PA070. 5. Native-first enterprise boundary: PA072-PA086 is closed locally; keep live/distributed followups under the production-profile items below and the long-term backlog. -6. Production profile remaining: PA060 production Kubernetes External Secrets controller sync, PA062 live Syft/Trivy/Cosign execution, PA064 external confirmation, and PA065 external error-tracker DSN exercise. +6. Production profile remaining: PA060 production Kubernetes External Secrets controller sync, PA064 external confirmation, and PA065 external error-tracker DSN exercise. Ship the thin vertical slice first: UI -> Rust gateway -> API -> real/fallback LLM -> DB -> feedback -> dashboard -> Grafana. Breadth is secondary until this loop is smooth. @@ -257,7 +257,7 @@ Advanced enterprise dimensions beyond the MVP product. Not blocking the first ve - [ ] PA090 Live MLflow tracking wired to the promotion pipeline - [ ] PA091 DVC dataset versioning + MinIO push, surfaced in UI - [ ] PA092 Trigger evaluation/leaderboard runs from the UI -- [ ] PA093 A/B test + multi-armed-bandit routing in production (Theme T) +- [x] PA093 A/B test + multi-armed-bandit routing in production (Theme T). (`/api/llm/generate` now accepts `experiment_ab` and `experiment_bandit` routing modes that call the native C++ `tryops_experiment_router` when built; `/api/experiments/route`, `/api/experiments/analyze`, and `/api/experiments/summary` expose guarded route decisions plus sequential-test evidence; the React Console adds an Experiments board and the LLM Playground can exercise the production experiment path.) - [ ] PA094 Canary / shadow deploy controllable from the Console - [ ] PA095 Auto model-card publish to the registry viewer - [ ] PA096 Dataset upload + validation (Great Expectations) in UI diff --git a/docs/progress_log.md b/docs/progress_log.md index 8368933..d8a6fe7 100644 --- a/docs/progress_log.md +++ b/docs/progress_log.md @@ -105,13 +105,14 @@ - Closed PA061 native TLS termination evidence: the Rust gateway now serves HTTPS through axum-server/rustls when cert/key env vars are set, Compose has an optional `gateway-tls` profile with `tryops_tls_cert` and `tryops_tls_key` secrets, `native/go/tryops-tls-contract/` validates plan/live coverage, and `make native-tls-smoke` passed with TLS1.3, HTTPS health 200, and plaintext HTTP rejection. - Closed PA055 RBAC enforcement: `configs/api_keys.json` now defines viewer/operator/admin principals, FastAPI exposes a `tryops.rbac_session.v1` session route, the Rust gateway preflights `/v1/auth/session` with `session:read`, and the React Console filters navigation from the active session permissions while using the active key for promotion drills. - Added PA064 native Go full-stack load/SLO evidence: `native/go/tryops-fullstack-load/` starts the FastAPI BFF and Rust gateway, drives six product scenarios through `/api/*`, emits `tryops.native_fullstack_load.v1`, and is indexed by the evaluation registry; the latest run passed 504 requests with zero errors, while k6/locust external confirmation remains pending because neither binary is installed locally. -- Added PA062 native CI/supply-chain contract evidence: `.github/workflows/ci.yml` now defines the Python/Node/Go/Rust/C++ test path, Compose validation, seven-image Docker Buildx matrix, Syft SBOM generation, Trivy HIGH/CRITICAL scan gate, artifact uploads, and Cosign keyless signing; `native/go/tryops-ci-contract/` emits `tryops.native_ci_contract.v1` with 16/16 checks passing and explicit missing local Syft/Trivy/Cosign tools. +- Added PA062 native CI/supply-chain contract evidence: `.github/workflows/ci.yml` now defines the Python/Node/Go/Rust/C++ test path, Compose validation, seven-image Docker Buildx matrix, Syft SBOM generation, Trivy HIGH/CRITICAL scan gate, artifact uploads, and Cosign keyless signing; this initial contract passed 16/16 checks and was later upgraded by the live Syft/Trivy/Cosign gate. - Added PA060 native secret-rotation/workload-identity plan evidence: `configs/secret_rotation_policy.json` declares Vault KV secret ownership and rotation windows, `infra/kubernetes/secret-management/` adds Vault `SecretStore`/`ExternalSecret` manifests with a projected service-account token, and `native/go/tryops-secret-rotation-contract/` emits `tryops.native_secret_rotation_contract.v1` with 50/50 plan checks passing and `production_ready=false` until live Vault identity is configured. - Added PA060 live Vault KV rotation evidence: `make native-secret-rotation-live` runs a disposable `hashicorp/vault:1.19` dev server, authenticates the native Go verifier from a temporary workload token file, bootstraps the policy KV v2 mount, rotates all 8 managed secret properties across 5 live KV paths, emits 65/65 checks with `production_ready=true`, and removes the Vault container/token file on exit. -- Closed PA063 dependency-lock evidence: `uv.lock` now pins the Python project resolution including accelerate/bitsandbytes, and `native/go/tryops-dependency-lock-contract/` validates `uv.lock`, `web/package-lock.json`, Rust `Cargo.lock`, and Go `go.sum` coverage into `tryops.native_dependency_lock_contract.v1` with 87/87 checks passing. +- Closed PA063 dependency-lock evidence: `uv.lock` now pins the Python project resolution including accelerate/bitsandbytes, and `native/go/tryops-dependency-lock-contract/` validates `uv.lock`, `web/package-lock.json`, Rust `Cargo.lock`, and Go `go.sum` coverage into `tryops.native_dependency_lock_contract.v1`; latest local evidence passes 89/89 checks. - Hardened the gateway and benchmark Docker builds against Rust runtime ABI drift by pinning Rust builder/runtime images to Bookworm and extending the native container contract with explicit Rust builder/runtime suite checks. - Hardened `make app-smoke` so it runs in a disposable `tryops_app_smoke` Compose project with fresh volumes and automatic teardown, avoiding stale local Postgres/MLflow state while preserving the normal developer stack. - Hardened native quota startup so the Rust gateway logs durable mirror failures and falls back to the local quota ledger instead of crash-looping when an optional Postgres mirror is unavailable. - Closed local P8 evidence rows PA072-PA086 after rerunning native Rust/static/cache, Go job/event/SLO/config/container/quota/trace/performance, C++ policy/VTON, and full-stack app-smoke evidence; the latest full-stack smoke passed 18/18 checks and the job runner passed 2/2 gateway jobs. - Added opt-in distributed quota admission in the Rust gateway: `TRYOPS_GATEWAY_QUOTA_POSTGRES_ADMISSION=true` makes Postgres the authoritative shared ledger using transactional `SELECT ... FOR UPDATE` row locks, fails closed when the Postgres adapter is unavailable, and still mirrors admitted decisions to Valkey when configured. - Added `native/go/tryops-distributed-quota/` plus `make native-distributed-quota-smoke`; the live smoke starts disposable Postgres and two Rust gateway instances, sends 32 concurrent free-plan LLM quota requests, and emits `tryops.distributed_quota_admission.v1` with exactly 20 allowed, 12 rejected, and zero oversell. +- Closed PA062 live supply-chain execution: `native/go/tryops-live-supply-chain/` verifies pinned Syft/Trivy/Cosign container outputs, `make native-ci-contract-live` emits `tryops.live_supply_chain.v1` plus production-ready `tryops.native_ci_contract.v1`, and the latest run recorded 613 Syft packages, 0 HIGH/CRITICAL Trivy findings, a verified Cosign SBOM signature, 17/17 CI checks, and `missing_tools=[]`. diff --git a/docs/reproducibility_checklist.md b/docs/reproducibility_checklist.md index a34812b..c547da5 100644 --- a/docs/reproducibility_checklist.md +++ b/docs/reproducibility_checklist.md @@ -12,7 +12,8 @@ Use this checklist before any final demo or report claim. - `make professor-demo-acceptance` - `make vulnerability-scan-sample` - `make native-ci-contract-test` -- `make native-ci-contract-sample` +- `make native-live-supply-chain-test` +- `make native-ci-contract-live` - `make evaluation-index-sample` - `make native-job-runner-test` - `make native-job-runner-sample` @@ -119,8 +120,9 @@ Use this checklist before any final demo or report claim. - Professor demo acceptance: `artifacts/eval/demo_acceptance/professor_demo_acceptance.json` must contain `tryops.professor_demo_acceptance.v1`, `passed=true`, a live blocked bad-candidate gate, and evidence checks for LLM Pareto, energy, full-stack smoke, native quota ledger, VTON comparison, lineage, promotion, rollback, governance, the Console Professor Demo view, and the seeded demo data contract. - Professor demo backup video: `artifacts/eval/demo_video/professor_demo_video.json` must contain `tryops.professor_demo_video.v1`, `passed=true`, `frame_count >= 9`, `step_count >= 7`, `duration_seconds > 0`, and `video_path=artifacts/demo/professor_demo_video/professor_demo_backup.mp4`; FFprobe must report H.264 video at 1280x720 and 30 FPS. - Vulnerability scan: `artifacts/eval/security/vulnerability_scan_report.json` must contain `tryops.vulnerability_scan.v1`, `passed=true`, the installed-tool scan results, and explicit `missing_required_tools` when enterprise scanners are absent. -- Native CI supply-chain contract: `artifacts/eval/ci/native_ci_contract.json` must contain `tryops.native_ci_contract.v1`, `passed=true`, workflow path `.github/workflows/ci.yml`, `make ci` coverage, GitHub Actions OIDC permissions, upload-artifact evidence, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL scan gate, Cosign keyless signing on non-PR pushes, and schema references to vulnerability, supply-chain, and container-contract reports. If `production_ready=false`, the report must list missing required local tools and cannot be used as live Syft/Trivy/Cosign execution evidence. -- Evaluation index: `artifacts/eval/evaluation_index/evaluation_index.json` must contain `tryops.evaluation_index.v1`, report counts, highlighted Pareto/energy/VTON/native-VTON-API/drift/full-stack/demo/security/CI/dependency-lock/config/secrets/postgres/backup/tls/container/performance/fullstack-load/trace-envelope/observability/alertmanager/quota-read-model/distributed-quota/runtime-telemetry evidence, `pipeline_runs` from run-context/OpenLineage/lineage artifacts, `optimization_panel` with `recommended_variant="4bit"`, `carbon_gate_verdict="pass"`, per-variant energy and SCI metrics, native SLO gate evidence, native event-dispatcher evidence, native CI contract evidence, native dependency-lock evidence, native config-contract evidence, native secret-rotation evidence, native Postgres migration evidence, native backup/restore drill evidence, native TLS termination evidence, native container-contract evidence, native trace-envelope evidence, native full-stack load SLO evidence, native observability evidence, native Alertmanager evidence, native quota-read-model evidence, native distributed-quota evidence, native runtime-telemetry evidence, native performance-budget evidence, and the Console endpoint `/api/evaluations/summary` must pass through `make app-smoke`. +- Live supply-chain scan/signature evidence: `artifacts/eval/ci/live_supply_chain_report.json` must contain `tryops.live_supply_chain.v1`, `passed=true`, `production_ready=true`, pinned Syft/Trivy/Cosign container image names, a non-empty SPDX SBOM package count, zero HIGH/CRITICAL Trivy vulnerability/misconfiguration/secret findings, a Cosign public key, a signature over the SBOM blob, and `verify-blob` output containing `Verified OK`. +- Native CI supply-chain contract: `artifacts/eval/ci/native_ci_contract.json` must contain `tryops.native_ci_contract.v1`, `passed=true`, workflow path `.github/workflows/ci.yml`, `make ci` coverage, GitHub Actions OIDC permissions, upload-artifact evidence, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL scan gate, Cosign keyless signing on non-PR pushes, and schema references to vulnerability, supply-chain, live supply-chain, and container-contract reports. If `production_ready=false`, the report must list missing required local tools and cannot be used as live Syft/Trivy/Cosign execution evidence. +- Evaluation index: `artifacts/eval/evaluation_index/evaluation_index.json` must contain `tryops.evaluation_index.v1`, report counts, highlighted Pareto/energy/VTON/native-VTON-API/drift/full-stack/demo/security/live-supply-chain/CI/dependency-lock/config/secrets/postgres/backup/tls/container/performance/fullstack-load/trace-envelope/observability/alertmanager/quota-read-model/distributed-quota/runtime-telemetry evidence, `pipeline_runs` from run-context/OpenLineage/lineage artifacts, `optimization_panel` with `recommended_variant="4bit"`, `carbon_gate_verdict="pass"`, per-variant energy and SCI metrics, live supply-chain evidence, native SLO gate evidence, native event-dispatcher evidence, native CI contract evidence, native dependency-lock evidence, native config-contract evidence, native secret-rotation evidence, native Postgres migration evidence, native backup/restore drill evidence, native TLS termination evidence, native container-contract evidence, native trace-envelope evidence, native full-stack load SLO evidence, native observability evidence, native Alertmanager evidence, native quota-read-model evidence, native distributed-quota evidence, native runtime-telemetry evidence, native performance-budget evidence, and the Console endpoint `/api/evaluations/summary` must pass through `make app-smoke`. - Native Go job runner: `artifacts/eval/jobs/native_job_runner_report.json` must contain `tryops.native_job_runner.v1`, `passed=true`, one passed LLM direct-generation job, one passed async VTON job with a non-empty `job_id`, context-bound HTTP attempts, retry attempt counts, and at least one VTON poll. - Console pipeline run history: the Runs view must render the `pipeline_runs` ledger from `/api/evaluations/summary`; `make app-smoke` must require `pipeline_runs`, `run-vton-001`, and `COMPLETE` in the gateway response. - Console optimization panel: the Evaluation view must render the `optimization_panel` from `/api/evaluations/summary`; `make app-smoke` must require `optimization_panel`, `recommended_variant":"4bit"`, and `carbon_gate_verdict":"pass"` in the gateway response. diff --git a/docs/roadmap_audit.md b/docs/roadmap_audit.md index 794a10b..4fc5cbd 100644 --- a/docs/roadmap_audit.md +++ b/docs/roadmap_audit.md @@ -428,16 +428,18 @@ This verifies: health, RBAC session, evaluation summary, quota summary, LLM generation, and operator promotion gate traffic through the Rust gateway. The latest local run passed 6/6 scenarios with 504 total requests, zero errors, worst p95 39.965 ms, and explicit `k6`/`locust` unavailability records. -- Native Go CI supply-chain contract evidence: `make native-ci-contract-sample` writes - `artifacts/eval/ci/native_ci_contract.json` as `tryops.native_ci_contract.v1`, proving 16/16 +- Native Go CI supply-chain contract evidence: `make native-ci-contract-live` writes + `artifacts/eval/ci/live_supply_chain_report.json` as `tryops.live_supply_chain.v1` and + `artifacts/eval/ci/native_ci_contract.json` as `tryops.native_ci_contract.v1`, proving 17/17 checks across GitHub Actions OIDC/artifact permissions, language tests, Compose validation, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL scan gating, - Cosign keyless signing, `make ci`, and referenced supply-chain/vulnerability/container reports. + Cosign keyless signing, local live Syft/Trivy/Cosign execution, `make ci`, and referenced + supply-chain/vulnerability/container reports. - Native Go dependency-lock contract evidence: `make native-dependency-lock-contract-sample` writes `artifacts/eval/dependencies/native_dependency_lock_contract.json` as - `tryops.native_dependency_lock_contract.v1`, proving 87/87 checks across `uv.lock`, + `tryops.native_dependency_lock_contract.v1`, proving 89/89 checks across `uv.lock`, `web/package-lock.json`, Rust `Cargo.lock`, and Go module checksum coverage. Current evidence - records 326 locked Python packages, 59 Node packages, 228 Rust crates, and 30 Go modules. + records 326 locked Python packages, 59 Node packages, 228 Rust crates, and 32 Go modules. - Native Go secret-rotation/workload-identity contract evidence: `make native-secret-rotation-contract-sample` writes `artifacts/eval/secrets/native_secret_rotation_contract.json` as `tryops.native_secret_rotation_contract.v1`, proving 50/50 plan checks across Vault KV policy, @@ -451,6 +453,12 @@ This verifies: contains `optimization_panel` with the recommended LLM variant, leaderboard rank, Pareto-frontier state, per-variant VRAM/energy/SCI, and carbon-gate verdict; the React Evaluation view renders the interactive panel; and `make app-smoke` verifies the payload through the Rust gateway. +- Production online experimentation surface: FastAPI accepts `experiment_ab` and `experiment_bandit` + on `/api/llm/generate`, exposes `/api/experiments/route`, `/api/experiments/analyze`, and + `/api/experiments/summary`, and delegates guarded A/B plus UCB bandit routing to the native C++ + `tryops_experiment_router` when present. The React Console now includes an Experiments board and + the LLM Playground can exercise the production experiment path; focused API tests verify RBAC, + guarded candidate blocking, and LLM generation through `experiment_bandit`. - Rust gateway auth preflight: `native/rust/tryops-gateway/src/auth.rs` validates scoped API keys and optional HS256 JWTs before proxying protected routes to FastAPI; Docker builds the gateway image with `configs/api_keys.json`; and `artifacts/eval/full_stack/full_stack_smoke.json` proves @@ -486,10 +494,11 @@ These remain intentionally unchecked because evidence is not strong enough yet: evidence with token-file auth, but a real cluster should still exercise the External Secrets controller and Vault Kubernetes auth TokenReview flow. - KServe/Kubeflow live deployment. The local Kubeflow-target orchestration skeleton exists, but there is no cluster execution or pipeline upload yet. -- Live Syft/Trivy/Cosign execution in this workspace. The GitHub Actions workflow and native CI - contract are wired, but local `production_ready=false` because Syft, Trivy, and Cosign are not - installed here. The vulnerability scan also records missing Grype, pip-audit, gitleaks, and - osv-scanner coverage. +- Live Syft/Trivy/Cosign execution in this workspace is now covered by `make native-ci-contract-live`. + The latest local evidence emits `tryops.live_supply_chain.v1` with 613 Syft packages, 0 + HIGH/CRITICAL Trivy findings, a verified Cosign SBOM signature, and + `tryops.native_ci_contract.v1` with `production_ready=true`. Optional secondary scanners + (Grype, pip-audit, gitleaks, osv-scanner) are still recorded as fallback coverage gaps. - Real Sigstore keyless OIDC and Rekor transparency-log verification. Local model provenance is a DSSE-shaped digest bundle verified by native C++, not public Sigstore transparency evidence. - External k6/locust load confirmation. The native Go full-stack load/SLO gate passes, but neither @@ -502,7 +511,8 @@ These remain intentionally unchecked because evidence is not strong enough yet: 1. ~~Add Transformers inference for SmolLM2-135M-Instruct.~~ **Done (R1):** real GPU inference behind unchanged contracts via `make llm-real-sample`. 2. Add live AWQ/GPTQ loading, live llama.cpp GGUF generation, and live vLLM variants to extend the current fp16/8-bit/4-bit Pareto report; keep `make llm-quantized-preflight-sample` and `make llm-gguf-preflight-sample` as artifact/runtime sanity gates. 3. Install or document Rust toolchain bootstrap so the gateway rebuilds in CI. -4. Install Trivy or Grype and run a CVE-backed vulnerability scan against the repo or image. +4. Add optional secondary scanner coverage (Grype, pip-audit, gitleaks, osv-scanner) around the now + passing Trivy-backed live scan. 5. Add live MLflow logging around the promotion pipeline. 6. Run the verified CLIP garment-image/text path across a fixed VTON benchmark and report confidence intervals. 7. Real VTON (CatVTON) execution on the L4 GPU behind the existing VTON contracts. diff --git a/docs/supply_chain.md b/docs/supply_chain.md index ddf5aca..154ea4d 100644 --- a/docs/supply_chain.md +++ b/docs/supply_chain.md @@ -12,7 +12,7 @@ Run: make supply-chain-sample make model-supply-chain-sample make native-dependency-lock-contract-sample -make native-ci-contract-sample +make native-ci-contract-live ``` Artifacts: @@ -22,6 +22,8 @@ uv.lock requirements.lock artifacts/eval/supply_chain/dependency_lock.json artifacts/eval/supply_chain/sbom.spdx.json +artifacts/eval/ci/live_supply_chain_report.json +artifacts/eval/ci/native_ci_contract.json artifacts/eval/supply_chain/supply_chain_report.json artifacts/eval/dependencies/native_dependency_lock_contract.json artifacts/eval/model_supply_chain/model_supply_chain_report.json @@ -54,6 +56,8 @@ artifacts/eval/ci/native_ci_contract.json - Python, C++, and Rego promotion gates requiring a passing model artifact scan and model provenance. - GitHub Actions CI contract for language tests, seven container image roles, Syft SBOM generation, Trivy HIGH/CRITICAL scan gating, uploaded evidence artifacts, and Cosign keyless signing. +- Native live supply-chain evidence from pinned Syft, Trivy, and Cosign containers, with a signed + SPDX SBOM blob verified locally before the CI contract can report production readiness. ## Current Tooling Status @@ -64,23 +68,28 @@ artifacts/eval/ci/native_ci_contract.json - `artifacts/eval/security/vulnerability_scan_report.json` - `artifacts/eval/security/npm_audit_web.json` -The latest local run found 0 npm vulnerabilities, but coverage is still `partial` and -`production_ready=false`. Syft, Trivy, Grype, pip-audit, gitleaks, osv-scanner, and Cosign are not -installed in this workspace, so the local fallback SBOM plus npm audit is not a replacement for a -production Syft/Trivy or Trivy-only pipeline. +The latest local `vulnerability-scan-sample` run found 0 npm vulnerabilities, but that specific +fallback report remains `coverage=partial` and `production_ready=false` because it only runs tools +installed on the host. It remains useful as a fast local smoke check and explicit missing-tool +inventory. PA062 CI wiring now exists in `.github/workflows/ci.yml`. The workflow builds the seven declared image roles, requests GitHub OIDC permissions, uploads evidence artifacts, generates Syft SPDX SBOMs, fails on HIGH/CRITICAL Trivy image findings, and signs pushed images with Cosign keyless identity on -non-PR runs. `make native-ci-contract-sample` validates that contract locally and emits -`tryops.native_ci_contract.v1`; its current local status is `production_ready=false` because Syft, -Trivy, and Cosign are not installed here. +non-PR runs. `make native-live-supply-chain-sample` executes `anchore/syft:v1.45.1`, +`aquasec/trivy:0.71.0`, and `ghcr.io/sigstore/cosign/cosign:v2.4.1` as pinned containers, so host +PATH installation is not required. The latest live report is `tryops.live_supply_chain.v1` with 613 +Syft packages, 0 HIGH/CRITICAL Trivy vulnerability/misconfiguration/secret findings, and a verified +Cosign signature over `artifacts/eval/ci/syft/filesystem.spdx.json`. `make native-ci-contract-live` +validates the full contract locally and emits `tryops.native_ci_contract.v1` with +`production_ready=true`. Preferred production commands: ```bash syft . -o spdx-json=artifacts/eval/supply_chain/syft.spdx.json trivy fs --scanners vuln,secret,config,license --format json . +make native-ci-contract-live ``` Syft is the preferred open-source SBOM generator for filesystems and supports SPDX/CycloneDX output. @@ -127,10 +136,9 @@ See `docs/model_supply_chain.md` for the exact schema and policy contract. ## Residual Risk -The local report proves that the repo can generate lock, SBOM, model-source, dataset-license, -model-artifact serialization/provenance evidence, and a partial installed-tool vulnerability scan. -It does not prove that no CVEs exist across Python, containers, OS packages, or infrastructure in -this workspace. PA062/J010 remain partial until the GitHub Actions workflow or local environment -executes Syft, Trivy, and Cosign and archives those artifacts. The current model signature is local -offline evidence; production Sigstore keyless OIDC and Rekor transparency-log inclusion still require -OpenSSF Model Signing / Sigstore tooling. +The local reports now prove that the repo can generate lock, SBOM, model-source, dataset-license, +model-artifact serialization/provenance evidence, a partial installed-tool vulnerability scan, and +live Syft/Trivy/Cosign execution through pinned containers. Residual risk remains for optional +secondary scanners (Grype, pip-audit, gitleaks, osv-scanner) and for public keyless model +transparency: the current model signature is local offline evidence; production Sigstore keyless OIDC +and Rekor transparency-log inclusion still require OpenSSF Model Signing / Sigstore tooling. diff --git a/infra/kubernetes/secret-management/tryops-external-secrets.yaml b/infra/kubernetes/secret-management/tryops-external-secrets.yaml index 3a22e9b..2e5944b 100644 --- a/infra/kubernetes/secret-management/tryops-external-secrets.yaml +++ b/infra/kubernetes/secret-management/tryops-external-secrets.yaml @@ -61,6 +61,13 @@ spec: spec: serviceAccountName: tryops-runtime automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault volumes: - name: vault-identity-token projected: @@ -72,6 +79,14 @@ spec: containers: - name: gateway image: ghcr.io/tryops/gateway:latest + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + capabilities: + drop: ["ALL"] envFrom: - secretRef: name: tryops-runtime-secrets diff --git a/native/README.md b/native/README.md index f607fc0..800766b 100644 --- a/native/README.md +++ b/native/README.md @@ -448,6 +448,34 @@ Current binary: artifacts/native/tryops_container_contract ``` +## Go Live Supply-Chain Evidence Gate + +Path: `native/go/tryops-live-supply-chain` + +Purpose: + +- Dependency-free Go verifier for local live Syft/Trivy/Cosign execution evidence. +- Reads the Syft SPDX SBOM, Trivy filesystem JSON report, Cosign public key, signature, and + `verify-blob` output generated by `make native-live-supply-chain-sample`. +- Fails production readiness on missing SBOM packages, any HIGH/CRITICAL Trivy vulnerability, + misconfiguration, or secret finding, or failed Cosign blob verification. +- Emits `tryops.live_supply_chain.v1` for the Console evidence registry and native CI contract. + +Source layout: + +- `main.go`: thin CLI entrypoint. +- `config.go`: evidence path, output path, and pinned container image flags. +- `evaluate.go`: Syft, Trivy, and Cosign readiness checks. +- `jsonutil.go`, `files.go`, `types.go`: report parsing and JSON emission. +- `live_supply_chain_test.go`: pass/fail regression coverage. + +Verified commands: + +```bash +make native-live-supply-chain-test +make native-live-supply-chain-sample +``` + ## Go CI Supply-Chain Contract Gate Path: `native/go/tryops-ci-contract` @@ -459,9 +487,10 @@ Purpose: artifact upload, seven-image Docker Buildx matrix, Syft SPDX SBOM generation, Trivy HIGH/CRITICAL scan gating, and Cosign keyless signing on non-PR pushes. - Checks `Makefile` for the local `ci` mirror and `native-ci-contract-*` targets. -- Cross-checks vulnerability, supply-chain, and container-contract report schemas. -- Records local required-tool availability for Docker, Syft, Trivy, and Cosign without treating - missing external tools as successful production evidence. +- Cross-checks vulnerability, supply-chain, live supply-chain, and container-contract report schemas. +- Records local required-tool availability for Docker, Syft, Trivy, and Cosign; a passing + `tryops.live_supply_chain.v1` report can satisfy Syft/Trivy/Cosign readiness without host PATH + installs because the tools ran from pinned containers. - Emits `tryops.native_ci_contract.v1` for the Console evidence registry. Source layout: @@ -479,7 +508,7 @@ Verified commands: ```bash make native-ci-contract-test -make native-ci-contract-sample +make native-ci-contract-live ``` Current binary: diff --git a/native/go/tryops-ci-contract/ci_contract_test.go b/native/go/tryops-ci-contract/ci_contract_test.go index 3fb450f..6be35fd 100644 --- a/native/go/tryops-ci-contract/ci_contract_test.go +++ b/native/go/tryops-ci-contract/ci_contract_test.go @@ -18,16 +18,18 @@ security-events: write actions/setup-python@v5 actions/setup-go@v5 actions/setup-node@v4 +python -m pip install -e ".[dev]" make test make web-typecheck web-build make native-go-test +make native-live-supply-chain-test make native-rust-test make native-cpp-test docker compose config --quiet docker compose --profile tls config --quiet make native-container-contract-sample make native-dependency-lock-contract-sample -make native-ci-contract-sample +make native-ci-contract-live make evaluation-index-sample actions/upload-artifact@v4 artifacts/eval @@ -39,16 +41,22 @@ role: benchmark role: cpp-tools role: api role: web-assets +Normalize image prefix +tr '[:upper:]' '[:lower:]' +IMAGE_PREFIX=${REGISTRY}/${LOWER_REPOSITORY} +GITHUB_ENV docker/setup-buildx-action@v3 docker/build-push-action@v6 -sbom: true -provenance: true +load: ${{ github.event_name == 'pull_request' }} +sbom: ${{ github.event_name != 'pull_request' }} +provenance: ${{ github.event_name != 'pull_request' }} anchore/sbom-action@v0 format: spdx-json output-file: artifacts/eval/ci/sbom/ aquasecurity/trivy-action@ scanners: vuln,secret,misconfig severity: HIGH,CRITICAL +limit-severities-for-sarif: true exit-code: "1" sigstore/cosign-installer@ cosign sign --yes @@ -75,16 +83,18 @@ security-events: write actions/setup-python@v5 actions/setup-go@v5 actions/setup-node@v4 +python -m pip install -e ".[dev]" make test make web-typecheck web-build make native-go-test +make native-live-supply-chain-test make native-rust-test make native-cpp-test docker compose config --quiet docker compose --profile tls config --quiet make native-container-contract-sample make native-dependency-lock-contract-sample -make native-ci-contract-sample +make native-ci-contract-live make evaluation-index-sample actions/upload-artifact@v4 artifacts/eval @@ -96,26 +106,36 @@ role: benchmark role: cpp-tools role: api role: web-assets +Normalize image prefix +tr '[:upper:]' '[:lower:]' +IMAGE_PREFIX=${REGISTRY}/${LOWER_REPOSITORY} +GITHUB_ENV docker/setup-buildx-action@v3 docker/build-push-action@v6 -sbom: true -provenance: true +load: ${{ github.event_name == 'pull_request' }} +sbom: ${{ github.event_name != 'pull_request' }} +provenance: ${{ github.event_name != 'pull_request' }} anchore/sbom-action@v0 format: spdx-json output-file: artifacts/eval/ci/sbom/ aquasecurity/trivy-action@ scanners: vuln,secret,misconfig severity: HIGH,CRITICAL +limit-severities-for-sarif: true exit-code: "1" sigstore/cosign-installer@ cosign sign --yes github.event_name != 'pull_request' `) writeFile(t, root, "Makefile", ` -ci: test web-typecheck native-go-test native-rust-test native-cpp-test supply-chain-sample vulnerability-scan-sample native-container-contract-sample native-dependency-lock-contract-sample native-ci-contract-sample evaluation-index-sample +ci: test web-typecheck native-go-test native-rust-test native-cpp-test supply-chain-sample vulnerability-scan-sample native-live-supply-chain-sample native-container-contract-sample native-dependency-lock-contract-sample native-ci-contract-live evaluation-index-sample +native-live-supply-chain-build: +native-live-supply-chain-test: +native-live-supply-chain-sample: native-ci-contract-build: native-ci-contract-test: native-ci-contract-sample: +native-ci-contract-live: `) writeFile(t, root, "artifacts/eval/security/vulnerability_scan_report.json", `{"schema_version":"tryops.vulnerability_scan.v1","passed":true,"production_ready":false}`) writeFile(t, root, "artifacts/eval/supply_chain/supply_chain_report.json", `{"schema_version":"tryops.supply_chain_report.v1","passed":true}`) @@ -126,6 +146,7 @@ native-ci-contract-sample: MakefilePath: "Makefile", VulnerabilityPath: "artifacts/eval/security/vulnerability_scan_report.json", SupplyChainPath: "artifacts/eval/supply_chain/supply_chain_report.json", + LiveSupplyChainPath: "artifacts/eval/ci/live_supply_chain_report.json", ContainerReportPath: "artifacts/eval/containers/native_container_contract_report.json", }) if err != nil { @@ -142,6 +163,105 @@ native-ci-contract-sample: } } +func TestEvaluateProductionReadyWithLiveSupplyChainEvidence(t *testing.T) { + root := t.TempDir() + writeFile(t, root, ".github/workflows/ci.yml", ` +pull_request: +push: +workflow_dispatch: +contents: read +id-token: write +packages: write +security-events: write +actions/setup-python@v5 +actions/setup-go@v5 +actions/setup-node@v4 +python -m pip install -e ".[dev]" +make test +make web-typecheck web-build +make native-go-test +make native-live-supply-chain-test +make native-rust-test +make native-cpp-test +docker compose config --quiet +docker compose --profile tls config --quiet +make native-container-contract-sample +make native-dependency-lock-contract-sample +make native-ci-contract-live +make evaluation-index-sample +actions/upload-artifact@v4 +artifacts/eval +retention-days: +role: gateway +role: controller +role: guardrail +role: benchmark +role: cpp-tools +role: api +role: web-assets +Normalize image prefix +tr '[:upper:]' '[:lower:]' +IMAGE_PREFIX=${REGISTRY}/${LOWER_REPOSITORY} +GITHUB_ENV +docker/setup-buildx-action@v3 +docker/build-push-action@v6 +load: ${{ github.event_name == 'pull_request' }} +sbom: ${{ github.event_name != 'pull_request' }} +provenance: ${{ github.event_name != 'pull_request' }} +anchore/sbom-action@v0 +format: spdx-json +output-file: artifacts/eval/ci/sbom/ +aquasecurity/trivy-action@ +scanners: vuln,secret,misconfig +severity: HIGH,CRITICAL +limit-severities-for-sarif: true +exit-code: "1" +sigstore/cosign-installer@ +cosign sign --yes +github.event_name != 'pull_request' +`) + writeFile(t, root, "Makefile", ` +ci: test web-typecheck native-go-test native-rust-test native-cpp-test supply-chain-sample vulnerability-scan-sample native-live-supply-chain-sample native-container-contract-sample native-dependency-lock-contract-sample native-ci-contract-live evaluation-index-sample +native-live-supply-chain-build: +native-live-supply-chain-test: +native-live-supply-chain-sample: +native-ci-contract-build: +native-ci-contract-test: +native-ci-contract-sample: +native-ci-contract-live: +`) + writeFile(t, root, "artifacts/eval/security/vulnerability_scan_report.json", `{"schema_version":"tryops.vulnerability_scan.v1","passed":true,"production_ready":false}`) + writeFile(t, root, "artifacts/eval/supply_chain/supply_chain_report.json", `{"schema_version":"tryops.supply_chain_report.v1","passed":true}`) + writeFile(t, root, "artifacts/eval/ci/live_supply_chain_report.json", `{"schema_version":"tryops.live_supply_chain.v1","passed":true,"production_ready":true}`) + writeFile(t, root, "artifacts/eval/containers/native_container_contract_report.json", `{"schema_version":"tryops.native_container_contract.v1","passed":true}`) + report, err := evaluateWithTools(Config{ + RootPath: root, + WorkflowPath: ".github/workflows/ci.yml", + MakefilePath: "Makefile", + VulnerabilityPath: "artifacts/eval/security/vulnerability_scan_report.json", + SupplyChainPath: "artifacts/eval/supply_chain/supply_chain_report.json", + LiveSupplyChainPath: "artifacts/eval/ci/live_supply_chain_report.json", + ContainerReportPath: "artifacts/eval/containers/native_container_contract_report.json", + }, []ToolStatus{ + {Name: "docker", Required: true, Available: true, Path: "/usr/bin/docker"}, + {Name: "syft", Required: true}, + {Name: "trivy", Required: true}, + {Name: "cosign", Required: true}, + }) + if err != nil { + t.Fatal(err) + } + if !report.Passed || !report.ProductionReady { + t.Fatalf("expected production-ready live contract: %#v", report) + } + if len(report.MissingRequiredTools) != 0 { + t.Fatalf("expected live evidence to cover host scanner tools, got %#v", report.MissingRequiredTools) + } + if report.CoverageLevel != "native_ci_supply_chain_contract" { + t.Fatalf("unexpected coverage: %s", report.CoverageLevel) + } +} + func writeFile(t *testing.T, root, rel, content string) { t.Helper() path := filepath.Join(root, rel) diff --git a/native/go/tryops-ci-contract/config.go b/native/go/tryops-ci-contract/config.go index 86127b6..40b7d03 100644 --- a/native/go/tryops-ci-contract/config.go +++ b/native/go/tryops-ci-contract/config.go @@ -13,6 +13,7 @@ func parseConfig() Config { flag.StringVar(&cfg.MakefilePath, "makefile", getenv("TRYOPS_CI_CONTRACT_MAKEFILE", "Makefile"), "Makefile path") flag.StringVar(&cfg.VulnerabilityPath, "vulnerability", getenv("TRYOPS_CI_CONTRACT_VULN", "artifacts/eval/security/vulnerability_scan_report.json"), "vulnerability report path") flag.StringVar(&cfg.SupplyChainPath, "supply-chain", getenv("TRYOPS_CI_CONTRACT_SUPPLY", "artifacts/eval/supply_chain/supply_chain_report.json"), "supply-chain report path") + flag.StringVar(&cfg.LiveSupplyChainPath, "live-supply-chain", getenv("TRYOPS_CI_CONTRACT_LIVE_SUPPLY", "artifacts/eval/ci/live_supply_chain_report.json"), "live Syft/Trivy/Cosign evidence path") flag.StringVar(&cfg.ContainerReportPath, "container", getenv("TRYOPS_CI_CONTRACT_CONTAINER", "artifacts/eval/containers/native_container_contract_report.json"), "container contract report path") flag.StringVar(&cfg.OutputPath, "output", getenv("TRYOPS_CI_CONTRACT_OUTPUT", "artifacts/eval/ci/native_ci_contract.json"), "JSON report output path") flag.Parse() diff --git a/native/go/tryops-ci-contract/evaluate.go b/native/go/tryops-ci-contract/evaluate.go index cf71de8..07c7374 100644 --- a/native/go/tryops-ci-contract/evaluate.go +++ b/native/go/tryops-ci-contract/evaluate.go @@ -5,6 +5,10 @@ import ( ) func evaluate(cfg Config) (Report, error) { + return evaluateWithTools(cfg, discoverTools()) +} + +func evaluateWithTools(cfg Config, tools []ToolStatus) (Report, error) { workflow, err := readText(cfg.RootPath, cfg.WorkflowPath) if err != nil { return Report{}, err @@ -13,8 +17,11 @@ func evaluate(cfg Config) (Report, error) { if err != nil { return Report{}, err } - tools := discoverTools() missing := missingRequiredTools(tools) + liveReady := liveSupplyChainReady(cfg.RootPath, cfg) + if liveReady { + missing = filterMissingForLiveSupplyChain(missing) + } checks := append(validateWorkflow(workflow), validateMakefile(makefile)...) evidence, evidenceChecks := inspectEvidence(cfg.RootPath, cfg) checks = append(checks, evidenceChecks...) @@ -52,7 +59,7 @@ func evaluate(cfg Config) (Report, error) { }, Notes: []string{ "The workflow has production CI wiring for tests, image build, SBOM, Trivy scan, artifact upload, and Cosign keyless signing.", - "Local production readiness remains false until required external tools and CVE/signing evidence are available in this workspace.", + "Local production readiness accepts pinned container executions of Syft, Trivy, and Cosign when the live report passes, so host PATH installation is not required.", }, }, nil } diff --git a/native/go/tryops-ci-contract/evidence.go b/native/go/tryops-ci-contract/evidence.go index 071da8d..b70733b 100644 --- a/native/go/tryops-ci-contract/evidence.go +++ b/native/go/tryops-ci-contract/evidence.go @@ -34,6 +34,7 @@ func inspectEvidence(root string, cfg Config) ([]EvidenceRef, []Check) { } addJSON("vulnerability_scan", cfg.VulnerabilityPath, "tryops.vulnerability_scan.v1", true) addJSON("supply_chain", cfg.SupplyChainPath, "tryops.supply_chain_report.v1", true) + addJSON("live_supply_chain", cfg.LiveSupplyChainPath, "tryops.live_supply_chain.v1", false) addJSON("container_contract", cfg.ContainerReportPath, "tryops.native_container_contract.v1", true) return evidence, checks } @@ -43,7 +44,10 @@ func evidenceProductionReady(root string, cfg Config) bool { if err != nil { return false } - if prod, ok := boolField(vuln, "production_ready"); ok && !prod { + if passed, ok := boolField(vuln, "passed"); !ok || !passed { + return false + } + if !liveSupplyChainReady(root, cfg) { return false } supply, err := readJSON(root, cfg.SupplyChainPath) @@ -62,3 +66,20 @@ func evidenceProductionReady(root string, cfg Config) bool { } return true } + +func liveSupplyChainReady(root string, cfg Config) bool { + live, err := readJSON(root, cfg.LiveSupplyChainPath) + if err != nil { + return false + } + if schema := stringField(live, "schema_version"); schema != "tryops.live_supply_chain.v1" { + return false + } + if passed, ok := boolField(live, "passed"); !ok || !passed { + return false + } + if prod, ok := boolField(live, "production_ready"); !ok || !prod { + return false + } + return true +} diff --git a/native/go/tryops-ci-contract/makefile.go b/native/go/tryops-ci-contract/makefile.go index ef02287..3df88c2 100644 --- a/native/go/tryops-ci-contract/makefile.go +++ b/native/go/tryops-ci-contract/makefile.go @@ -7,7 +7,7 @@ func validateMakefile(makefile string) []Check { checks = append(checks, Check{Name: name, Passed: passed, Detail: detail}) } addContains("make_ci_target_exists", []string{"ci:", "test", "web-typecheck", "native-go-test", "native-rust-test", "native-cpp-test"}) - addContains("make_ci_supply_chain_evidence", []string{"supply-chain-sample", "vulnerability-scan-sample", "native-container-contract-sample", "native-dependency-lock-contract-sample", "native-ci-contract-sample", "evaluation-index-sample"}) - addContains("make_native_ci_contract_targets", []string{"native-ci-contract-build:", "native-ci-contract-test:", "native-ci-contract-sample:"}) + addContains("make_ci_supply_chain_evidence", []string{"supply-chain-sample", "vulnerability-scan-sample", "native-live-supply-chain-sample", "native-container-contract-sample", "native-dependency-lock-contract-sample", "native-ci-contract-live", "evaluation-index-sample"}) + addContains("make_native_ci_contract_targets", []string{"native-live-supply-chain-build:", "native-live-supply-chain-test:", "native-live-supply-chain-sample:", "native-ci-contract-build:", "native-ci-contract-test:", "native-ci-contract-sample:", "native-ci-contract-live:"}) return checks } diff --git a/native/go/tryops-ci-contract/tools.go b/native/go/tryops-ci-contract/tools.go index d68c72f..d3d29da 100644 --- a/native/go/tryops-ci-contract/tools.go +++ b/native/go/tryops-ci-contract/tools.go @@ -32,3 +32,19 @@ func missingRequiredTools(tools []ToolStatus) []string { } return missing } + +func filterMissingForLiveSupplyChain(missing []string) []string { + covered := map[string]bool{ + "syft": true, + "trivy": true, + "cosign": true, + } + filtered := make([]string, 0, len(missing)) + for _, name := range missing { + if covered[name] { + continue + } + filtered = append(filtered, name) + } + return filtered +} diff --git a/native/go/tryops-ci-contract/types.go b/native/go/tryops-ci-contract/types.go index ceb5030..708204d 100644 --- a/native/go/tryops-ci-contract/types.go +++ b/native/go/tryops-ci-contract/types.go @@ -6,6 +6,7 @@ type Config struct { MakefilePath string VulnerabilityPath string SupplyChainPath string + LiveSupplyChainPath string ContainerReportPath string OutputPath string } diff --git a/native/go/tryops-ci-contract/workflow.go b/native/go/tryops-ci-contract/workflow.go index cb8d25c..b1a9446 100644 --- a/native/go/tryops-ci-contract/workflow.go +++ b/native/go/tryops-ci-contract/workflow.go @@ -8,13 +8,14 @@ func validateWorkflow(workflow string) []Check { } addContains("workflow_triggers_push_pr_dispatch", []string{"pull_request:", "push:", "workflow_dispatch:"}) addContains("workflow_oidc_and_package_permissions", []string{"contents: read", "id-token: write", "packages: write", "security-events: write"}) - addContains("workflow_language_test_setup", []string{"actions/setup-python@v5", "actions/setup-go@v5", "actions/setup-node@v4", "make test", "make web-typecheck web-build", "make native-go-test", "make native-rust-test", "make native-cpp-test"}) - addContains("workflow_compose_and_native_contracts", []string{"docker compose config --quiet", "docker compose --profile tls config --quiet", "make native-container-contract-sample", "make native-dependency-lock-contract-sample", "make native-ci-contract-sample", "make evaluation-index-sample"}) + addContains("workflow_language_test_setup", []string{"actions/setup-python@v5", "actions/setup-go@v5", "actions/setup-node@v4", "python -m pip install -e \".[dev]\"", "make test", "make web-typecheck web-build", "make native-go-test", "make native-rust-test", "make native-cpp-test"}) + addContains("workflow_compose_and_native_contracts", []string{"docker compose config --quiet", "docker compose --profile tls config --quiet", "make native-container-contract-sample", "make native-dependency-lock-contract-sample", "make native-ci-contract-live", "make evaluation-index-sample"}) addContains("workflow_artifact_uploads", []string{"actions/upload-artifact@v4", "artifacts/eval", "retention-days:"}) addContains("workflow_image_matrix_roles", []string{"role: gateway", "role: controller", "role: guardrail", "role: benchmark", "role: cpp-tools", "role: api", "role: web-assets"}) - addContains("workflow_docker_build_metadata", []string{"docker/setup-buildx-action@v3", "docker/build-push-action@v6", "sbom: true", "provenance: true"}) + addContains("workflow_lowercase_ghcr_image_prefix", []string{"Normalize image prefix", "tr '[:upper:]' '[:lower:]'", "IMAGE_PREFIX=${REGISTRY}/${LOWER_REPOSITORY}", "GITHUB_ENV"}) + addContains("workflow_docker_build_metadata", []string{"docker/setup-buildx-action@v3", "docker/build-push-action@v6", "load: ${{ github.event_name == 'pull_request' }}", "sbom: ${{ github.event_name != 'pull_request' }}", "provenance: ${{ github.event_name != 'pull_request' }}"}) addContains("workflow_syft_sbom_generation", []string{"anchore/sbom-action@v0", "format: spdx-json", "output-file: artifacts/eval/ci/sbom/"}) - addContains("workflow_trivy_high_critical_gate", []string{"aquasecurity/trivy-action@", "scanners: vuln,secret,misconfig", "severity: HIGH,CRITICAL", "exit-code: \"1\""}) + addContains("workflow_trivy_high_critical_gate", []string{"aquasecurity/trivy-action@", "scanners: vuln,secret,misconfig", "severity: HIGH,CRITICAL", "limit-severities-for-sarif: true", "exit-code: \"1\""}) addContains("workflow_cosign_keyless_signing", []string{"sigstore/cosign-installer@", "cosign sign --yes", "github.event_name != 'pull_request'"}) return checks } diff --git a/native/go/tryops-db-migrator/go.mod b/native/go/tryops-db-migrator/go.mod index e78f8b2..faebba2 100644 --- a/native/go/tryops-db-migrator/go.mod +++ b/native/go/tryops-db-migrator/go.mod @@ -1,14 +1,13 @@ module tryops.dev/tryops-db-migrator -go 1.19 +go 1.25.0 -require github.com/jackc/pgx/v5 v5.5.5 +require github.com/jackc/pgx/v5 v5.9.0 require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/text v0.29.0 // indirect ) diff --git a/native/go/tryops-db-migrator/go.sum b/native/go/tryops-db-migrator/go.sum index a7b97cb..1059fe8 100644 --- a/native/go/tryops-db-migrator/go.sum +++ b/native/go/tryops-db-migrator/go.sum @@ -1,11 +1,12 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= +github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -13,13 +14,13 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/native/go/tryops-evaluation-index/classify.go b/native/go/tryops-evaluation-index/classify.go index b434a24..49a0628 100644 --- a/native/go/tryops-evaluation-index/classify.go +++ b/native/go/tryops-evaluation-index/classify.go @@ -71,6 +71,8 @@ func titleForPath(path string, data map[string]interface{}) string { return "Native Go job runner" case "tryops.vulnerability_scan.v1": return "Vulnerability scan coverage" + case "tryops.live_supply_chain.v1": + return "Live Syft/Trivy/Cosign supply-chain evidence" case "tryops.native_ci_contract.v1": return "Native CI supply-chain contract" case "tryops.native_dependency_lock_contract.v1": diff --git a/native/go/tryops-evaluation-index/index.go b/native/go/tryops-evaluation-index/index.go index 265f3f3..484c8cc 100644 --- a/native/go/tryops-evaluation-index/index.go +++ b/native/go/tryops-evaluation-index/index.go @@ -77,6 +77,7 @@ func selectHighlights(reports []artifactReport) map[string]artifactReport { "demo_acceptance": {schema: "tryops.professor_demo_acceptance.v1", preferredSub: "demo_acceptance/professor_demo_acceptance.json"}, "demo_video": {schema: "tryops.professor_demo_video.v1", preferredSub: "demo_video/professor_demo_video.json"}, "vulnerability": {schema: "tryops.vulnerability_scan.v1", preferredSub: "security/vulnerability_scan_report.json"}, + "live_supply_chain": {schema: "tryops.live_supply_chain.v1", preferredSub: "ci/live_supply_chain_report.json"}, "ci_contract": {schema: "tryops.native_ci_contract.v1", preferredSub: "ci/native_ci_contract.json"}, "dependency_lock": {schema: "tryops.native_dependency_lock_contract.v1", preferredSub: "dependencies/native_dependency_lock_contract.json"}, "data_versioning": {schema: "tryops.dvc_minio_versioning.v1", preferredSub: "data_versioning/dvc_minio_report.json"}, diff --git a/native/go/tryops-evaluation-index/index_test.go b/native/go/tryops-evaluation-index/index_test.go index 3fd8317..9b12f95 100644 --- a/native/go/tryops-evaluation-index/index_test.go +++ b/native/go/tryops-evaluation-index/index_test.go @@ -109,6 +109,12 @@ func TestCategoryForPath(t *testing.T) { if ciCategory != "governance" { t.Fatalf("unexpected ci category: %q", ciCategory) } + liveSupplyChainCategory := categoryForPath("artifacts/eval/ci/live_supply_chain_report.json", map[string]interface{}{ + "schema_version": "tryops.live_supply_chain.v1", + }) + if liveSupplyChainCategory != "governance" { + t.Fatalf("unexpected live supply-chain category: %q", liveSupplyChainCategory) + } dependencyCategory := categoryForPath("artifacts/eval/dependencies/native_dependency_lock_contract.json", map[string]interface{}{ "schema_version": "tryops.native_dependency_lock_contract.v1", }) @@ -129,6 +135,25 @@ func TestCategoryForPath(t *testing.T) { } } +func TestSummaryLiveSupplyChain(t *testing.T) { + items := summaryLiveSupplyChain(map[string]interface{}{ + "passed": true, + "production_ready": true, + "syft": map[string]interface{}{ + "package_count": float64(614), + }, + "trivy": map[string]interface{}{ + "total_high_critical": float64(0), + }, + "cosign": map[string]interface{}{ + "verified": true, + }, + }) + if len(items) != 5 || items[2].Value != "614" || items[3].Value != "0" || items[4].Value != "true" { + t.Fatalf("unexpected live supply-chain summary: %#v", items) + } +} + func TestSummaryCIContract(t *testing.T) { items := summaryCIContract(map[string]interface{}{ "passed": true, diff --git a/native/go/tryops-evaluation-index/summary.go b/native/go/tryops-evaluation-index/summary.go index d760e58..9e94dee 100644 --- a/native/go/tryops-evaluation-index/summary.go +++ b/native/go/tryops-evaluation-index/summary.go @@ -36,6 +36,8 @@ func summaryForReport(path string, data map[string]interface{}) []summaryItem { return summaryVLLMProbe(data) case "tryops.vulnerability_scan.v1": return summaryVulnerability(data) + case "tryops.live_supply_chain.v1": + return summaryLiveSupplyChain(data) case "tryops.native_ci_contract.v1": return summaryCIContract(data) case "tryops.native_dependency_lock_contract.v1": @@ -263,6 +265,19 @@ func summaryVulnerability(data map[string]interface{}) []summaryItem { } } +func summaryLiveSupplyChain(data map[string]interface{}) []summaryItem { + syft := objectField(data, "syft") + trivy := objectField(data, "trivy") + cosign := objectField(data, "cosign") + return []summaryItem{ + {Label: "passed", Value: formatValue(data["passed"])}, + {Label: "production ready", Value: formatValue(data["production_ready"])}, + {Label: "syft packages", Value: formatValue(syft["package_count"])}, + {Label: "trivy high/critical", Value: formatValue(trivy["total_high_critical"])}, + {Label: "cosign verified", Value: formatValue(cosign["verified"])}, + } +} + func summaryCIContract(data map[string]interface{}) []summaryItem { missing := arrayField(data, "missing_required_tools") checks := arrayField(data, "checks") diff --git a/native/go/tryops-live-supply-chain/config.go b/native/go/tryops-live-supply-chain/config.go new file mode 100644 index 0000000..b5b94e6 --- /dev/null +++ b/native/go/tryops-live-supply-chain/config.go @@ -0,0 +1,42 @@ +package main + +import ( + "flag" + "os" + "path/filepath" +) + +func parseConfig() Config { + cfg := Config{} + flag.StringVar(&cfg.RootPath, "root", getenv("TRYOPS_ROOT", "."), "repository root") + flag.StringVar(&cfg.OutputPath, "output", getenv("TRYOPS_LIVE_SUPPLY_CHAIN_OUTPUT", "artifacts/eval/ci/live_supply_chain_report.json"), "JSON evidence output path") + flag.StringVar(&cfg.SyftSBOMPath, "syft-sbom", "artifacts/eval/ci/syft/filesystem.spdx.json", "Syft SPDX JSON output path") + flag.StringVar(&cfg.SyftVersionPath, "syft-version", "artifacts/eval/ci/syft/version.txt", "Syft version output path") + flag.StringVar(&cfg.TrivyReportPath, "trivy-report", "artifacts/eval/ci/trivy/filesystem.json", "Trivy JSON output path") + flag.StringVar(&cfg.TrivyVersionPath, "trivy-version", "artifacts/eval/ci/trivy/version.txt", "Trivy version output path") + flag.StringVar(&cfg.CosignVersionPath, "cosign-version", "artifacts/eval/ci/cosign/version.txt", "Cosign version output path") + flag.StringVar(&cfg.CosignPublicKey, "cosign-public-key", "artifacts/eval/ci/cosign/tryops-local.pub", "Cosign public key path") + flag.StringVar(&cfg.CosignSignature, "cosign-signature", "artifacts/eval/ci/cosign/sbom.spdx.json.sig", "Cosign signature path") + flag.StringVar(&cfg.CosignVerifyPath, "cosign-verify-output", "artifacts/eval/ci/cosign/verify-blob.txt", "Cosign verify output path") + flag.StringVar(&cfg.SignedBlobPath, "signed-blob", "artifacts/eval/ci/syft/filesystem.spdx.json", "blob signed by Cosign") + flag.StringVar(&cfg.SyftImage, "syft-image", getenv("TRYOPS_SYFT_IMAGE", "anchore/syft:v1.45.1"), "Syft container image used") + flag.StringVar(&cfg.TrivyImage, "trivy-image", getenv("TRYOPS_TRIVY_IMAGE", "aquasec/trivy:0.71.0"), "Trivy container image used") + flag.StringVar(&cfg.CosignImage, "cosign-image", getenv("TRYOPS_COSIGN_IMAGE", "ghcr.io/sigstore/cosign/cosign:v2.4.1"), "Cosign container image used") + flag.Parse() + cfg.RootPath = filepath.Clean(cfg.RootPath) + return cfg +} + +func getenv(name string, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func rootJoin(root string, path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(root, path) +} diff --git a/native/go/tryops-live-supply-chain/evaluate.go b/native/go/tryops-live-supply-chain/evaluate.go new file mode 100644 index 0000000..0072257 --- /dev/null +++ b/native/go/tryops-live-supply-chain/evaluate.go @@ -0,0 +1,151 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +func evaluate(cfg Config) Report { + checks := []Check{} + syft := summarizeSyft(cfg, &checks) + trivy := summarizeTrivy(cfg, &checks) + cosign := summarizeCosign(cfg, &checks) + + tools := []ToolEvidence{ + {Name: "syft", Image: cfg.SyftImage, VersionOutputPath: cfg.SyftVersionPath, Version: firstVersionLine(readText(cfg.RootPath, cfg.SyftVersionPath)), Executed: syft.PackageCount > 0}, + {Name: "trivy", Image: cfg.TrivyImage, VersionOutputPath: cfg.TrivyVersionPath, Version: firstVersionLine(readText(cfg.RootPath, cfg.TrivyVersionPath)), Executed: trivy.SchemaVersion > 0}, + {Name: "cosign", Image: cfg.CosignImage, VersionOutputPath: cfg.CosignVersionPath, Version: firstVersionLine(readText(cfg.RootPath, cfg.CosignVersionPath)), Executed: cosign.Verified}, + } + for _, tool := range tools { + checks = append(checks, Check{ + Name: "tool." + tool.Name + ".executed", + Passed: tool.Executed, + Detail: fmt.Sprintf("image=%s version=%s", tool.Image, tool.Version), + }) + } + + passed := true + for _, check := range checks { + if !check.Passed { + passed = false + break + } + } + return Report{ + SchemaVersion: schemaVersion, + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Passed: passed, + ProductionReady: passed, + CoverageLevel: "native_live_syft_trivy_cosign", + Tools: tools, + Syft: syft, + Trivy: trivy, + Cosign: cosign, + Checks: checks, + Research: []ResearchRef{ + {Name: "Syft CLI", URL: "https://oss.anchore.com/docs/reference/syft/cli/", Use: "SPDX SBOM generation for filesystem source"}, + {Name: "Trivy filesystem scanning", URL: "https://trivy.dev/docs/latest/configuration/reporting/", Use: "HIGH/CRITICAL vulnerability, secret, and misconfiguration gate"}, + {Name: "Cosign blob signing", URL: "https://docs.sigstore.dev/cosign/signing/signing_with_blobs/", Use: "local SBOM signature and verification evidence"}, + }, + Notes: []string{ + "Syft, Trivy, and Cosign are executed from pinned open-source container images so host PATH installation is not required.", + "Cosign uses a temporary local key for blob-signature proof; GitHub Actions keeps the keyless OCI image signing contract.", + "Trivy is run with HIGH/CRITICAL severity and scanners=vuln,secret,misconfig; any finding fails production readiness.", + }, + } +} + +func summarizeSyft(cfg Config, checks *[]Check) SyftSummary { + data, err := readJSON(cfg.RootPath, cfg.SyftSBOMPath) + if err != nil { + *checks = append(*checks, Check{Name: "syft.sbom.present", Passed: false, Detail: err.Error()}) + return SyftSummary{Path: cfg.SyftSBOMPath} + } + spdx := stringField(data, "spdxVersion") + packages := arrayField(data, "packages") + documentName := stringField(data, "name") + *checks = append(*checks, + Check{Name: "syft.sbom.spdx_schema", Passed: strings.HasPrefix(spdx, "SPDX-"), Detail: spdx}, + Check{Name: "syft.sbom.packages_present", Passed: len(packages) > 0, Detail: fmt.Sprintf("packages=%d", len(packages))}, + ) + return SyftSummary{Path: cfg.SyftSBOMPath, SPDX: spdx, PackageCount: len(packages), DocumentName: documentName} +} + +func summarizeTrivy(cfg Config, checks *[]Check) TrivySummary { + summary := TrivySummary{Path: cfg.TrivyReportPath, BySeverity: map[string]int{}} + data, err := readJSON(cfg.RootPath, cfg.TrivyReportPath) + if err != nil { + *checks = append(*checks, Check{Name: "trivy.report.present", Passed: false, Detail: err.Error()}) + return summary + } + summary.SchemaVersion = intField(data, "SchemaVersion") + results := arrayField(data, "Results") + summary.Results = len(results) + for _, item := range results { + result, _ := item.(map[string]interface{}) + for _, vuln := range arrayField(result, "Vulnerabilities") { + severity := severityOf(vuln) + summary.BySeverity[severity]++ + if highCritical(severity) { + summary.HighCriticalVulnerabilities++ + } + } + for _, misconfig := range arrayField(result, "Misconfigurations") { + severity := severityOf(misconfig) + summary.BySeverity[severity]++ + if highCritical(severity) { + summary.HighCriticalMisconfigurations++ + } + } + for _, secret := range arrayField(result, "Secrets") { + severity := severityOf(secret) + summary.BySeverity[severity]++ + if highCritical(severity) { + summary.HighCriticalSecrets++ + } + } + } + summary.TotalHighCritical = summary.HighCriticalVulnerabilities + summary.HighCriticalMisconfigurations + summary.HighCriticalSecrets + *checks = append(*checks, + Check{Name: "trivy.report.schema", Passed: summary.SchemaVersion > 0, Detail: fmt.Sprintf("schema=%d", summary.SchemaVersion)}, + Check{Name: "trivy.high_critical.vulnerabilities", Passed: summary.HighCriticalVulnerabilities == 0, Detail: fmt.Sprintf("%d", summary.HighCriticalVulnerabilities)}, + Check{Name: "trivy.high_critical.misconfigurations", Passed: summary.HighCriticalMisconfigurations == 0, Detail: fmt.Sprintf("%d", summary.HighCriticalMisconfigurations)}, + Check{Name: "trivy.high_critical.secrets", Passed: summary.HighCriticalSecrets == 0, Detail: fmt.Sprintf("%d", summary.HighCriticalSecrets)}, + ) + return summary +} + +func summarizeCosign(cfg Config, checks *[]Check) CosignSummary { + verifyOutput := readText(cfg.RootPath, cfg.CosignVerifyPath) + publicKey := readText(cfg.RootPath, cfg.CosignPublicKey) + signatureBytes := fileSize(cfg.RootPath, cfg.CosignSignature) + publicKeyBytes := fileSize(cfg.RootPath, cfg.CosignPublicKey) + signedBlobBytes := fileSize(cfg.RootPath, cfg.SignedBlobPath) + summary := CosignSummary{ + SignedBlobPath: cfg.SignedBlobPath, + PublicKeyPath: cfg.CosignPublicKey, + SignaturePath: cfg.CosignSignature, + VerifyOutputPath: cfg.CosignVerifyPath, + PublicKeyBytes: publicKeyBytes, + SignatureBytes: signatureBytes, + Verified: strings.Contains(verifyOutput, "Verified OK"), + TLogSkipped: strings.Contains(strings.ToLower(verifyOutput), "tlog"), + } + *checks = append(*checks, + Check{Name: "cosign.signed_blob.present", Passed: signedBlobBytes > 0, Detail: fmt.Sprintf("bytes=%d", signedBlobBytes)}, + Check{Name: "cosign.public_key.present", Passed: publicKeyBytes > 0 && strings.Contains(publicKey, "BEGIN PUBLIC KEY"), Detail: fmt.Sprintf("bytes=%d", publicKeyBytes)}, + Check{Name: "cosign.signature.present", Passed: signatureBytes > 0, Detail: fmt.Sprintf("bytes=%d", signatureBytes)}, + Check{Name: "cosign.verify_blob.ok", Passed: summary.Verified, Detail: strings.TrimSpace(verifyOutput)}, + ) + return summary +} + +func severityOf(item interface{}) string { + data, _ := item.(map[string]interface{}) + return strings.ToUpper(stringField(data, "Severity")) +} + +func highCritical(severity string) bool { + return severity == "HIGH" || severity == "CRITICAL" +} diff --git a/native/go/tryops-live-supply-chain/files.go b/native/go/tryops-live-supply-chain/files.go new file mode 100644 index 0000000..0b518a2 --- /dev/null +++ b/native/go/tryops-live-supply-chain/files.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" +) + +func readJSON(root string, path string) (map[string]interface{}, error) { + body, err := os.ReadFile(rootJoin(root, path)) + if err != nil { + return nil, err + } + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + return nil, err + } + return data, nil +} + +func readText(root string, path string) string { + body, err := os.ReadFile(rootJoin(root, path)) + if err != nil { + return "" + } + return string(body) +} + +func fileSize(root string, path string) int64 { + info, err := os.Stat(rootJoin(root, path)) + if err != nil { + return 0 + } + return info.Size() +} + +func writeReport(path string, report Report) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + body, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(body, '\n'), 0o644) +} + +func firstVersionLine(output string) string { + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + return line + } + return "" +} diff --git a/native/go/tryops-live-supply-chain/go.mod b/native/go/tryops-live-supply-chain/go.mod new file mode 100644 index 0000000..ed70fde --- /dev/null +++ b/native/go/tryops-live-supply-chain/go.mod @@ -0,0 +1,3 @@ +module tryops.dev/tryops-live-supply-chain + +go 1.22 diff --git a/native/go/tryops-live-supply-chain/jsonutil.go b/native/go/tryops-live-supply-chain/jsonutil.go new file mode 100644 index 0000000..318e6b9 --- /dev/null +++ b/native/go/tryops-live-supply-chain/jsonutil.go @@ -0,0 +1,26 @@ +package main + +func stringField(data map[string]interface{}, key string) string { + if value, ok := data[key].(string); ok { + return value + } + return "" +} + +func intField(data map[string]interface{}, key string) int { + switch value := data[key].(type) { + case float64: + return int(value) + case int: + return value + default: + return 0 + } +} + +func arrayField(data map[string]interface{}, key string) []interface{} { + if value, ok := data[key].([]interface{}); ok { + return value + } + return nil +} diff --git a/native/go/tryops-live-supply-chain/live_supply_chain_test.go b/native/go/tryops-live-supply-chain/live_supply_chain_test.go new file mode 100644 index 0000000..18fdb8a --- /dev/null +++ b/native/go/tryops-live-supply-chain/live_supply_chain_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +func TestEvaluateLiveSupplyChainPasses(t *testing.T) { + root := t.TempDir() + writeFixture(t, root, "artifacts/eval/ci/syft/filesystem.spdx.json", `{"spdxVersion":"SPDX-2.3","name":"tryops","packages":[{"name":"a"},{"name":"b"}]}`) + writeFixture(t, root, "artifacts/eval/ci/trivy/filesystem.json", `{"SchemaVersion":2,"Results":[{"Target":"repo","Vulnerabilities":[],"Misconfigurations":[],"Secrets":[]}]}`) + writeFixture(t, root, "artifacts/eval/ci/cosign/tryops-local.pub", "-----BEGIN PUBLIC KEY-----\nabc\n-----END PUBLIC KEY-----\n") + writeFixture(t, root, "artifacts/eval/ci/cosign/sbom.spdx.json.sig", "signature\n") + writeFixture(t, root, "artifacts/eval/ci/cosign/verify-blob.txt", "Verified OK\n") + writeFixture(t, root, "artifacts/eval/ci/syft/version.txt", "Application: syft\nVersion: 1.45.1\n") + writeFixture(t, root, "artifacts/eval/ci/trivy/version.txt", "Version: 0.71.0\n") + writeFixture(t, root, "artifacts/eval/ci/cosign/version.txt", "GitVersion: v2.4.1\n") + + report := evaluate(Config{RootPath: root, + SyftSBOMPath: "artifacts/eval/ci/syft/filesystem.spdx.json", + TrivyReportPath: "artifacts/eval/ci/trivy/filesystem.json", + CosignPublicKey: "artifacts/eval/ci/cosign/tryops-local.pub", + CosignSignature: "artifacts/eval/ci/cosign/sbom.spdx.json.sig", + CosignVerifyPath: "artifacts/eval/ci/cosign/verify-blob.txt", + SignedBlobPath: "artifacts/eval/ci/syft/filesystem.spdx.json", + SyftVersionPath: "artifacts/eval/ci/syft/version.txt", + TrivyVersionPath: "artifacts/eval/ci/trivy/version.txt", + CosignVersionPath: "artifacts/eval/ci/cosign/version.txt", + }) + if !report.Passed || !report.ProductionReady { + t.Fatalf("expected passing production report: %#v", report.Checks) + } + if report.Syft.PackageCount != 2 || report.Trivy.TotalHighCritical != 0 || !report.Cosign.Verified { + t.Fatalf("unexpected report summary: %#v", report) + } +} + +func TestEvaluateLiveSupplyChainFailsHighCritical(t *testing.T) { + root := t.TempDir() + writeFixture(t, root, "artifacts/eval/ci/syft/filesystem.spdx.json", `{"spdxVersion":"SPDX-2.3","packages":[{"name":"a"}]}`) + writeFixture(t, root, "artifacts/eval/ci/trivy/filesystem.json", `{"SchemaVersion":2,"Results":[{"Vulnerabilities":[{"Severity":"CRITICAL"}]}]}`) + writeFixture(t, root, "artifacts/eval/ci/cosign/tryops-local.pub", "-----BEGIN PUBLIC KEY-----\nabc\n-----END PUBLIC KEY-----\n") + writeFixture(t, root, "artifacts/eval/ci/cosign/sbom.spdx.json.sig", "signature\n") + writeFixture(t, root, "artifacts/eval/ci/cosign/verify-blob.txt", "Verified OK\n") + report := evaluate(Config{RootPath: root, + SyftSBOMPath: "artifacts/eval/ci/syft/filesystem.spdx.json", + TrivyReportPath: "artifacts/eval/ci/trivy/filesystem.json", + CosignPublicKey: "artifacts/eval/ci/cosign/tryops-local.pub", + CosignSignature: "artifacts/eval/ci/cosign/sbom.spdx.json.sig", + CosignVerifyPath: "artifacts/eval/ci/cosign/verify-blob.txt", + SignedBlobPath: "artifacts/eval/ci/syft/filesystem.spdx.json", + }) + if report.Passed || report.ProductionReady { + t.Fatalf("expected high critical findings to fail") + } + if report.Trivy.HighCriticalVulnerabilities != 1 { + t.Fatalf("expected one critical vulnerability: %#v", report.Trivy) + } +} + +func writeFixture(t *testing.T, root string, path string, body string) { + t.Helper() + full := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/native/go/tryops-live-supply-chain/main.go b/native/go/tryops-live-supply-chain/main.go new file mode 100644 index 0000000..d4d7295 --- /dev/null +++ b/native/go/tryops-live-supply-chain/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "os" +) + +func main() { + cfg := parseConfig() + report := evaluate(cfg) + if err := writeReport(rootJoin(cfg.RootPath, cfg.OutputPath), report); err != nil { + fmt.Fprintf(os.Stderr, "write live supply-chain report: %v\n", err) + os.Exit(2) + } + fmt.Printf("live supply chain: passed=%t production_ready=%t syft_packages=%d trivy_high_critical=%d cosign_verified=%t\n", + report.Passed, + report.ProductionReady, + report.Syft.PackageCount, + report.Trivy.TotalHighCritical, + report.Cosign.Verified, + ) + if !report.Passed { + os.Exit(1) + } +} diff --git a/native/go/tryops-live-supply-chain/types.go b/native/go/tryops-live-supply-chain/types.go new file mode 100644 index 0000000..07fdb32 --- /dev/null +++ b/native/go/tryops-live-supply-chain/types.go @@ -0,0 +1,84 @@ +package main + +const schemaVersion = "tryops.live_supply_chain.v1" + +type Config struct { + RootPath string + OutputPath string + SyftSBOMPath string + SyftVersionPath string + TrivyReportPath string + TrivyVersionPath string + CosignVersionPath string + CosignPublicKey string + CosignSignature string + CosignVerifyPath string + SignedBlobPath string + SyftImage string + TrivyImage string + CosignImage string +} + +type Check struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Detail string `json:"detail"` +} + +type ToolEvidence struct { + Name string `json:"name"` + Image string `json:"image"` + VersionOutputPath string `json:"version_output_path"` + Version string `json:"version"` + Executed bool `json:"executed"` +} + +type SyftSummary struct { + Path string `json:"path"` + SPDX string `json:"spdx_version"` + PackageCount int `json:"package_count"` + DocumentName string `json:"document_name"` +} + +type TrivySummary struct { + Path string `json:"path"` + SchemaVersion int `json:"schema_version"` + Results int `json:"results"` + HighCriticalVulnerabilities int `json:"high_critical_vulnerabilities"` + HighCriticalMisconfigurations int `json:"high_critical_misconfigurations"` + HighCriticalSecrets int `json:"high_critical_secrets"` + TotalHighCritical int `json:"total_high_critical"` + BySeverity map[string]int `json:"by_severity"` +} + +type CosignSummary struct { + SignedBlobPath string `json:"signed_blob_path"` + PublicKeyPath string `json:"public_key_path"` + SignaturePath string `json:"signature_path"` + VerifyOutputPath string `json:"verify_output_path"` + PublicKeyBytes int64 `json:"public_key_bytes"` + SignatureBytes int64 `json:"signature_bytes"` + Verified bool `json:"verified"` + TLogSkipped bool `json:"tlog_skipped"` +} + +type ResearchRef struct { + Name string `json:"name"` + URL string `json:"url"` + Use string `json:"use"` +} + +type Report struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt string `json:"generated_at"` + Passed bool `json:"passed"` + ProductionReady bool `json:"production_ready"` + CoverageLevel string `json:"coverage_level"` + Tools []ToolEvidence `json:"tools"` + Syft SyftSummary `json:"syft"` + Trivy TrivySummary `json:"trivy"` + Cosign CosignSummary `json:"cosign"` + Checks []Check `json:"checks"` + Research []ResearchRef `json:"research"` + Notes []string `json:"notes"` +} diff --git a/src/tryops/api.py b/src/tryops/api.py index 35d94d9..824cfe5 100644 --- a/src/tryops/api.py +++ b/src/tryops/api.py @@ -22,16 +22,24 @@ from tryops.auth import authenticate_api_key, authorize_admin_payload, build_rbac_session from tryops.contracts import ModelCandidate from tryops.evaluation_artifacts import load_evaluation_index +from tryops.experiments import ( + DEFAULT_EXPERIMENT_ID, + normalize_experiment_analysis_variants, + normalize_experiment_variants, + normalize_guardrail_thresholds, + normalize_holdback, +) from tryops.guardrails import evaluate_egress_guardrails, evaluate_ingress_guardrails, merge_guardrail_verdicts from tryops.jobs import VTON_JOB_QUEUE, render_job_metrics from tryops.lineage import build_lineage_record +from tryops.native_experiment_stats import analyze_with_native_experiment_stats from tryops.observability import record_api_observation, render_prometheus_metrics, sanitize_payload_metadata from tryops.pipelines.llm_baseline import estimate_tokens, generate_baseline_response from tryops.pipelines.vton_baseline import run_naive_overlay_baseline from tryops.policy import evaluate_promotion from tryops.quota import check_and_record_quota from tryops.quota_read_model import load_quota_read_model -from tryops.routing import build_routing_decision +from tryops.routing import build_experiment_routing_decision, build_routing_decision from tryops.semantic_cache import GLOBAL_SEMANTIC_CACHE, build_cache_metadata from tryops.timeouts import RequestTimeoutError, run_with_timeout, timeout_details from tryops.vton_native_bridge import build_native_vton_execution_evidence @@ -353,21 +361,32 @@ def llm_generate(payload: dict[str, Any]) -> dict[str, Any]: _record(endpoint, request_id, "llm", clean["model_alias"], started, payload, response) return response try: - routing = build_routing_decision( - workload="llm", - request_id=request_id, - requested_alias=clean["model_alias"], - routing_mode=clean["routing_mode"], - canary_percent=clean["canary_percent"], - shadow=clean["shadow"], - fallback_enabled=clean["fallback_enabled"], - route_health={ - "baseline": "ready", - "champion": "ready" if clean["optimized_available"] else "unavailable", - "challenger": "ready" if clean["optimized_available"] else "unavailable", - "candidate": "ready" if clean["optimized_available"] else "unavailable", - }, - ) + if clean["routing_mode"] in {"experiment_ab", "experiment_bandit"}: + routing = build_experiment_routing_decision( + workload="llm", + request_id=request_id, + experiment_id=clean["experiment_id"], + variants=clean["experiment_variants"], + mode="ab" if clean["routing_mode"] == "experiment_ab" else "bandit", + holdback_percent=clean["experiment_holdback_percent"], + guardrail_thresholds=clean["experiment_guardrail_thresholds"], + ) + else: + routing = build_routing_decision( + workload="llm", + request_id=request_id, + requested_alias=clean["model_alias"], + routing_mode=clean["routing_mode"], + canary_percent=clean["canary_percent"], + shadow=clean["shadow"], + fallback_enabled=clean["fallback_enabled"], + route_health={ + "baseline": "ready", + "champion": "ready" if clean["optimized_available"] else "unavailable", + "challenger": "ready" if clean["optimized_available"] else "unavailable", + "candidate": "ready" if clean["optimized_available"] else "unavailable", + }, + ) quota = check_and_record_quota( user_id=clean["user_id"], plan=clean["quota_plan"], @@ -485,7 +504,7 @@ def llm_generate(payload: dict[str, Any]) -> dict[str, Any]: response["quota"] = quota _record(endpoint, request_id, "llm", routing["primary_alias"], started, payload, response) return response - except (KeyError, TypeError, ValueError) as exc: + except (KeyError, RuntimeError, TypeError, ValueError) as exc: response = structured_error( request_id=request_id, code="llm_generation_failed", @@ -495,6 +514,93 @@ def llm_generate(payload: dict[str, Any]) -> dict[str, Any]: _record(endpoint, request_id, "llm", clean["model_alias"], started, payload, response) return response + @app.get("/api/experiments/summary") + @app.get("/v1/experiments/summary") + def get_experiment_summary(api_key: str = None) -> dict[str, Any]: + auth = authenticate_api_key(api_key, required_scope="admin:read") + if not auth["allowed"]: + return _admin_auth_error("unknown", auth, "experiments") + routing_report = _optional_json_artifact("artifacts/eval/experiments/online_experiment_report.json") + analysis_report = _optional_json_artifact("artifacts/eval/experiments/online_experiment_analysis_report.json") + passed = bool(routing_report.get("passed")) and bool(analysis_report.get("passed")) + native_ready = ( + routing_report.get("decisions", {}) + .get("bandit", {}) + .get("experiment", {}) + .get("source") + == "native_cpp_cli" + ) and ( + analysis_report.get("native_experiment_stats", {}).get("source") == "native_cpp_cli" + ) + return { + "status": "ok", + "data": { + "schema_version": "tryops.experiment_console.v1", + "experiment_id": routing_report.get("decisions", {}) + .get("bandit", {}) + .get("experiment_id", DEFAULT_EXPERIMENT_ID), + "production_ready": passed and native_ready, + "routing_report": routing_report, + "analysis_report": analysis_report, + }, + } + + @app.post("/api/experiments/route") + @app.post("/v1/experiments/route") + def experiment_route(payload: dict[str, Any]) -> dict[str, Any]: + request_id = request_id_from_payload(payload) + auth = authorize_admin_payload(payload, required_scope="admin:read") + if not auth["allowed"]: + return _admin_auth_error(request_id, auth, "experiments") + try: + mode = str(payload.get("mode", "bandit")) + if mode not in {"ab", "bandit"}: + raise ValueError("mode must be 'ab' or 'bandit'") + decision = build_experiment_routing_decision( + workload=str(payload.get("workload", "llm")), + request_id=request_id, + experiment_id=str(payload.get("experiment_id", DEFAULT_EXPERIMENT_ID)), + variants=normalize_experiment_variants(payload.get("variants")), + mode=mode, + holdback_percent=float(payload.get("holdback_percent", 5.0)), + guardrail_thresholds=normalize_guardrail_thresholds(payload.get("guardrail_thresholds")), + ) + return {"status": "ok", "request_id": request_id, "data": decision, "auth": auth} + except (RuntimeError, TypeError, ValueError) as exc: + return structured_error( + request_id=request_id, + code="experiment_route_failed", + message=str(exc), + workload="experiments", + ) + + @app.post("/api/experiments/analyze") + @app.post("/v1/experiments/analyze") + def experiment_analyze(payload: dict[str, Any]) -> dict[str, Any]: + request_id = request_id_from_payload(payload) + auth = authorize_admin_payload(payload, required_scope="admin:read") + if not auth["allowed"]: + return _admin_auth_error(request_id, auth, "experiments") + try: + stats = analyze_with_native_experiment_stats( + holdback=normalize_holdback(payload.get("holdback")), + variants=normalize_experiment_analysis_variants(payload.get("variants")), + experiment_id=str(payload.get("experiment_id", DEFAULT_EXPERIMENT_ID)), + confidence=float(payload.get("confidence", 0.95)), + alpha=float(payload.get("alpha", 0.05)), + beta=float(payload.get("beta", 0.20)), + min_detectable_effect=float(payload.get("min_detectable_effect", 0.05)), + min_sample_size=float(payload.get("min_sample_size", 100.0)), + ) + return {"status": "ok", "request_id": request_id, "data": stats, "auth": auth} + except (RuntimeError, TypeError, ValueError) as exc: + return structured_error( + request_id=request_id, + code="experiment_analysis_failed", + message=str(exc), + workload="experiments", + ) + @app.get("/api/history") def get_history(api_key: str = None, kind: str = None, limit: int = 50) -> dict[str, Any]: auth = authenticate_api_key(api_key, required_scope="admin:read") @@ -761,6 +867,19 @@ def _record( conn.close() +def _optional_json_artifact(path: str) -> dict[str, Any]: + try: + return load_json_artifact(path) + except (OSError, ValueError) as exc: + return { + "schema_version": "tryops.optional_artifact_missing.v1", + "path": path, + "passed": False, + "status": "missing", + "reason": str(exc), + } + + def _write_vton_report_sidecar(report: dict[str, Any]) -> None: output_path = report.get("output", {}).get("path") if not output_path: diff --git a/src/tryops/api_contracts.py b/src/tryops/api_contracts.py index 28b2b38..47fd11d 100644 --- a/src/tryops/api_contracts.py +++ b/src/tryops/api_contracts.py @@ -6,6 +6,12 @@ from uuid import uuid4 from tryops.pipelines.llm_baseline import SUPPORTED_MODEL_ALIASES +from tryops.experiments import ( + DEFAULT_EXPERIMENT_HOLDBACK_PERCENT, + DEFAULT_EXPERIMENT_ID, + normalize_experiment_variants, + normalize_guardrail_thresholds, +) from tryops.quota import SUPPORTED_QUOTA_PLANS from tryops.routing import SUPPORTED_VTON_ALIASES @@ -109,8 +115,13 @@ def validate_llm_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], list[ errors.append({"field": "structured", "message": "structured must be a boolean"}) routing_mode = str(payload.get("routing_mode", "direct")) - if routing_mode not in {"direct", "canary"}: - errors.append({"field": "routing_mode", "message": "routing_mode must be 'direct' or 'canary'"}) + if routing_mode not in {"direct", "canary", "experiment_ab", "experiment_bandit"}: + errors.append( + { + "field": "routing_mode", + "message": "routing_mode must be 'direct', 'canary', 'experiment_ab', or 'experiment_bandit'", + } + ) canary_percent = payload.get("canary_percent", 0.0) try: @@ -155,6 +166,26 @@ def validate_llm_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], list[ errors.append({"field": "quota_plan", "message": f"quota_plan must be one of {sorted(SUPPORTED_QUOTA_PLANS)}"}) timeout_ms_int = _validate_timeout_ms(payload, errors) + experiment_id = str(payload.get("experiment_id", DEFAULT_EXPERIMENT_ID)).strip() or DEFAULT_EXPERIMENT_ID + holdback_percent = payload.get("experiment_holdback_percent", DEFAULT_EXPERIMENT_HOLDBACK_PERCENT) + try: + holdback_percent_float = float(holdback_percent) + except (TypeError, ValueError): + holdback_percent_float = -1.0 + errors.append({"field": "experiment_holdback_percent", "message": "experiment_holdback_percent must be numeric"}) + if holdback_percent_float < 0.0 or holdback_percent_float > 95.0: + errors.append({"field": "experiment_holdback_percent", "message": "experiment_holdback_percent must be between 0 and 95"}) + + try: + experiment_variants = normalize_experiment_variants(payload.get("experiment_variants")) + except ValueError as exc: + experiment_variants = [] + errors.append({"field": "experiment_variants", "message": str(exc)}) + try: + experiment_guardrail_thresholds = normalize_guardrail_thresholds(payload.get("experiment_guardrail_thresholds")) + except ValueError as exc: + experiment_guardrail_thresholds = {} + errors.append({"field": "experiment_guardrail_thresholds", "message": str(exc)}) return ( { @@ -172,6 +203,10 @@ def validate_llm_payload(payload: dict[str, Any]) -> tuple[dict[str, Any], list[ "user_id": user_id, "quota_plan": quota_plan, "timeout_ms": timeout_ms_int, + "experiment_id": experiment_id, + "experiment_holdback_percent": holdback_percent_float, + "experiment_variants": experiment_variants, + "experiment_guardrail_thresholds": experiment_guardrail_thresholds, }, errors, ) diff --git a/src/tryops/auth.py b/src/tryops/auth.py index c17d5a1..fdab0b5 100644 --- a/src/tryops/auth.py +++ b/src/tryops/auth.py @@ -16,7 +16,7 @@ FORBIDDEN_SECRET_FIELDS = {"api_key", "raw_key", "secret", "token", "password"} PUBLIC_NAV_ITEMS = ("demo", "llm", "vton") SCOPE_NAV_ITEMS = { - "admin:read": ("dashboard", "history", "runs", "registry", "evaluations"), + "admin:read": ("dashboard", "history", "runs", "registry", "evaluations", "experiments"), "lineage:read": ("governance",), "promotion:evaluate": ("incidents",), } diff --git a/src/tryops/experiments.py b/src/tryops/experiments.py new file mode 100644 index 0000000..f5c3146 --- /dev/null +++ b/src/tryops/experiments.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any + +from tryops.pipelines.llm_baseline import SUPPORTED_MODEL_ALIASES + + +DEFAULT_EXPERIMENT_ID = "tryops-llm-answer-quality" +DEFAULT_EXPERIMENT_HOLDBACK_PERCENT = 5.0 +DEFAULT_EXPERIMENT_THRESHOLDS = { + "max_block_rate": 0.02, + "max_latency_p95_ms": 120.0, + "max_error_rate": 0.01, +} +DEFAULT_EXPERIMENT_VARIANTS = [ + { + "name": "champion", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45.0, + "impressions": 1000.0, + "rewards": 820.0, + "guardrail_block_rate": 0.002, + "latency_p95_ms": 42.0, + "error_rate": 0.002, + }, + { + "name": "challenger", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45.0, + "impressions": 500.0, + "rewards": 465.0, + "guardrail_block_rate": 0.004, + "latency_p95_ms": 38.0, + "error_rate": 0.003, + }, + { + "name": "candidate", + "adapter": "tryops-rule-baseline", + "allocation_percent": 10.0, + "impressions": 50.0, + "rewards": 49.0, + "guardrail_block_rate": 0.080, + "latency_p95_ms": 35.0, + "error_rate": 0.003, + }, +] +DEFAULT_EXPERIMENT_ANALYSIS_VARIANTS = [ + { + "name": "champion", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45.0, + "impressions": 950.0, + "rewards": 786.0, + "guardrail_block_rate": 0.002, + "latency_p95_ms": 42.0, + "error_rate": 0.002, + }, + { + "name": "challenger", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45.0, + "impressions": 800.0, + "rewards": 760.0, + "guardrail_block_rate": 0.004, + "latency_p95_ms": 38.0, + "error_rate": 0.003, + }, +] +DEFAULT_EXPERIMENT_HOLDBACK = { + "name": "champion_holdback", + "impressions": 1000.0, + "rewards": 820.0, +} + + +def normalize_experiment_variants(value: object) -> list[dict[str, Any]]: + variants = value if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) else None + source = list(variants) if variants else DEFAULT_EXPERIMENT_VARIANTS + normalized = [_normalize_variant(item) for item in source] + if not normalized: + raise ValueError("experiment_variants cannot be empty") + return normalized + + +def normalize_experiment_analysis_variants(value: object) -> list[dict[str, Any]]: + variants = value if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)) else None + source = list(variants) if variants else DEFAULT_EXPERIMENT_ANALYSIS_VARIANTS + normalized = [_normalize_variant(item) for item in source] + if not normalized: + raise ValueError("experiment analysis variants cannot be empty") + return normalized + + +def normalize_guardrail_thresholds(value: object) -> dict[str, float]: + thresholds = dict(DEFAULT_EXPERIMENT_THRESHOLDS) + if isinstance(value, Mapping): + for key in thresholds: + if key in value: + thresholds[key] = _float_field(value, key) + return thresholds + + +def normalize_holdback(value: object) -> dict[str, Any]: + if not isinstance(value, Mapping): + return dict(DEFAULT_EXPERIMENT_HOLDBACK) + return { + "name": str(value.get("name", DEFAULT_EXPERIMENT_HOLDBACK["name"])), + "impressions": _float_field(value, "impressions", default=DEFAULT_EXPERIMENT_HOLDBACK["impressions"]), + "rewards": _float_field(value, "rewards", default=DEFAULT_EXPERIMENT_HOLDBACK["rewards"]), + } + + +def _normalize_variant(value: object) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("each experiment variant must be an object") + name = str(value.get("name", "")).strip() + if name not in SUPPORTED_MODEL_ALIASES: + raise ValueError(f"unsupported experiment variant '{name}'") + return { + "name": name, + "adapter": str(value.get("adapter", "tryops-rule-baseline")), + "allocation_percent": _float_field(value, "allocation_percent"), + "impressions": _float_field(value, "impressions"), + "rewards": _float_field(value, "rewards"), + "guardrail_block_rate": _float_field(value, "guardrail_block_rate"), + "latency_p95_ms": _float_field(value, "latency_p95_ms"), + "error_rate": _float_field(value, "error_rate"), + } + + +def _float_field(source: Mapping[str, Any], key: str, *, default: float = 0.0) -> float: + try: + return float(source.get(key, default)) + except (TypeError, ValueError) as exc: + raise ValueError(f"{key} must be numeric") from exc diff --git a/src/tryops/observability.py b/src/tryops/observability.py index 0ae604c..dc52669 100644 --- a/src/tryops/observability.py +++ b/src/tryops/observability.py @@ -246,20 +246,26 @@ def emit_structured_log(event: dict[str, Any], *, response: dict[str, Any] | Non record["native_envelope"] = build_native_trace_log_envelope(event, structured_log=record) _STRUCTURED_LOGS.append(record) if _LOG_PATH is not None: - _LOG_PATH.parent.mkdir(parents=True, exist_ok=True) - with _LOG_PATH.open("a", encoding="utf-8") as log_file: - log_file.write(json.dumps(record, sort_keys=True) + "\n") + _append_jsonl(_LOG_PATH, record) return record def emit_trace_span(span: dict[str, Any]) -> dict[str, Any]: if _TRACE_SPAN_PATH is not None: - _TRACE_SPAN_PATH.parent.mkdir(parents=True, exist_ok=True) - with _TRACE_SPAN_PATH.open("a", encoding="utf-8") as trace_file: - trace_file.write(json.dumps(span, sort_keys=True) + "\n") + _append_jsonl(_TRACE_SPAN_PATH, span) return span +def _append_jsonl(path: Path, record: dict[str, Any]) -> bool: + try: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as output_file: + output_file.write(json.dumps(record, sort_keys=True) + "\n") + return True + except OSError: + return False + + def structured_logs_snapshot() -> list[dict[str, Any]]: return list(_STRUCTURED_LOGS) diff --git a/tests/test_api_surface.py b/tests/test_api_surface.py index bfb9a2c..c34dbdd 100644 --- a/tests/test_api_surface.py +++ b/tests/test_api_surface.py @@ -46,6 +46,12 @@ def test_create_app_registers_versioned_routes_when_fastapi_is_available(self) - self.assertIn("/v1/quota/summary", paths) self.assertIn("/api/evaluations/summary", paths) self.assertIn("/v1/evaluations/summary", paths) + self.assertIn("/api/experiments/summary", paths) + self.assertIn("/v1/experiments/summary", paths) + self.assertIn("/api/experiments/route", paths) + self.assertIn("/v1/experiments/route", paths) + self.assertIn("/api/experiments/analyze", paths) + self.assertIn("/v1/experiments/analyze", paths) self.assertIn("/api/vton/comparison", paths) self.assertIn("/api/incidents/workflow", paths) self.assertIn("/v1/incidents/workflow", paths) @@ -84,6 +90,7 @@ def test_auth_session_route_returns_role_aware_nav(self) -> None: self.assertEqual(viewer["status"], "ok") self.assertEqual(viewer["data"]["principal"]["role"], "viewer") self.assertIn("dashboard", viewer["data"]["permissions"]["nav"]) + self.assertIn("experiments", viewer["data"]["permissions"]["nav"]) self.assertNotIn("incidents", viewer["data"]["permissions"]["nav"]) self.assertEqual(operator["data"]["principal"]["role"], "operator") self.assertIn("incidents", operator["data"]["permissions"]["nav"]) @@ -667,6 +674,99 @@ def test_llm_request_validation_and_shadow_contract(self) -> None: self.assertEqual(response["status"], "completed") self.assertIn("tokens_per_second", response["metrics"]) + def test_llm_experiment_routing_mode_validation_and_generation(self) -> None: + try: + app = create_app() + except RuntimeError as exc: + self.skipTest(str(exc)) + + clean, errors = validate_llm_payload( + { + "request_id": "req-experiment-llm", + "prompt": "Compare guarded A/B routing with bandit allocation.", + "model_alias": "champion", + "routing_mode": "experiment_bandit", + "max_tokens": 64, + "quota_plan": "free", + "user_id": "experiment-user", + } + ) + endpoint = _endpoint_for(app, "/api/llm/generate") + reset_quota_usage() + response = endpoint( + { + "request_id": "req-experiment-llm", + "prompt": clean["prompt"], + "model_alias": clean["model_alias"], + "routing_mode": clean["routing_mode"], + "max_tokens": clean["max_tokens"], + "quota_plan": clean["quota_plan"], + "user_id": clean["user_id"], + } + ) + + self.assertEqual(errors, []) + self.assertEqual(response["status"], "completed") + self.assertEqual(response["routing"]["mode"], "experiment_bandit") + self.assertEqual(response["routing"]["experiment"]["schema_version"], "tryops.native_experiment_router.v1") + self.assertIn(response["routing"]["primary_alias"], {"champion", "challenger"}) + + def test_experiment_route_endpoint_requires_auth_and_blocks_guardrail_candidate(self) -> None: + try: + app = create_app() + except RuntimeError as exc: + self.skipTest(str(exc)) + + endpoint = _endpoint_for(app, "/api/experiments/route") + payload = { + "request_id": "req-experiment-route", + "mode": "bandit", + "experiment_id": "tryops-llm-answer-quality", + "variants": [ + { + "name": "champion", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45, + "impressions": 1000, + "rewards": 820, + "guardrail_block_rate": 0.002, + "latency_p95_ms": 42, + "error_rate": 0.002, + }, + { + "name": "challenger", + "adapter": "tryops-rule-baseline", + "allocation_percent": 45, + "impressions": 500, + "rewards": 465, + "guardrail_block_rate": 0.004, + "latency_p95_ms": 38, + "error_rate": 0.003, + }, + { + "name": "candidate", + "adapter": "tryops-rule-baseline", + "allocation_percent": 10, + "impressions": 50, + "rewards": 49, + "guardrail_block_rate": 0.08, + "latency_p95_ms": 35, + "error_rate": 0.003, + }, + ], + "guardrail_thresholds": {"max_block_rate": 0.02, "max_latency_p95_ms": 120, "max_error_rate": 0.01}, + } + + denied = endpoint(payload) + allowed = endpoint({**payload, "api_key": "tryops-viewer-demo-key"}) + candidate = next(item for item in allowed["data"]["experiment"]["variants"] if item["name"] == "candidate") + + self.assertEqual(denied["status"], "rejected") + self.assertEqual(allowed["status"], "ok") + self.assertEqual(allowed["data"]["mode"], "experiment_bandit") + self.assertFalse(candidate["eligible"]) + self.assertIn("guardrail_block_rate", candidate["violations"]) + def test_structured_vton_validation_error_contract(self) -> None: clean, errors = validate_vton_payload({"request_id": "req-vton", "model_alias": "unknown"}) response = structured_error( diff --git a/tests/test_observability.py b/tests/test_observability.py index 03760a7..a42a31d 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -145,6 +145,36 @@ def test_structured_logs_and_trace_spans_are_sanitized_and_file_backed(self) -> self.assertNotIn("customer-456", str(file_records[0])) self.assertNotIn("customer-456", str(file_spans[0])) + def test_unwritable_observability_files_do_not_break_request_metrics(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "api_events_dir" + span_path = Path(temp_dir) / "api_spans_dir" + log_path.mkdir() + span_path.mkdir() + configure_structured_log_path(log_path) + configure_trace_span_path(span_path) + try: + event = record_api_observation( + endpoint="/v1/llm/generate", + request_id="req-unwritable", + workload="llm", + model_alias="baseline", + status="completed", + started_at=start_timer(), + payload={"prompt": "Explain TryOps.", "model_alias": "baseline", "user_id": "customer-789"}, + response={"status": "completed"}, + ) + memory_records = structured_logs_snapshot() + memory_spans = trace_spans_snapshot() + finally: + configure_structured_log_path(None) + configure_trace_span_path(None) + + self.assertEqual(event["request_id"], "req-unwritable") + self.assertEqual(len(memory_records), 1) + self.assertEqual(len(memory_spans), 1) + self.assertEqual(metrics_snapshot()["request_counters"][0]["count"], 1) + if __name__ == "__main__": unittest.main() diff --git a/web/src/App.tsx b/web/src/App.tsx index c282e9f..90e2c9b 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -2,6 +2,7 @@ import { useEffect, useMemo, useState } from "react"; import { TryOpsClient } from "./api"; import { AppShell } from "./components/AppShell"; import { DashboardView } from "./components/DashboardView"; +import { ExperimentView } from "./components/ExperimentView"; import { GovernanceView } from "./components/GovernanceView"; import { HistoryView } from "./components/HistoryView"; import { IncidentView } from "./components/IncidentView"; @@ -11,7 +12,7 @@ import { ProfessorDemoView } from "./components/ProfessorDemoView"; import { RegistryView } from "./components/RegistryView"; import { VtonStudio } from "./components/VtonStudio"; import { EvaluationView } from "./components/EvaluationView"; -import type { AuthSession, DashboardSummary, EvaluationIndex, ModelRecord, QuotaReadModel, RequestRecord, ViewKey } from "./types"; +import type { AuthSession, DashboardSummary, EvaluationIndex, ExperimentConsole, ModelRecord, QuotaReadModel, RequestRecord, ViewKey } from "./types"; const API_KEY_STORAGE = "tryops.console.api_key"; @@ -23,6 +24,7 @@ export function App() { const [requests, setRequests] = useState([]); const [models, setModels] = useState([]); const [evaluations, setEvaluations] = useState(); + const [experiments, setExperiments] = useState(); const [quota, setQuota] = useState(); const [session, setSession] = useState(); const [lastError, setLastError] = useState(); @@ -62,7 +64,8 @@ export function App() { client.quotaSummary(), client.history(), client.models(), - client.evaluations() + client.evaluations(), + client.experiments() ]); if (outcomes[0].status === "fulfilled") { setDashboard(outcomes[0].value); @@ -79,6 +82,9 @@ export function App() { if (outcomes[4].status === "fulfilled") { setEvaluations(outcomes[4].value); } + if (outcomes[5].status === "fulfilled") { + setExperiments(outcomes[5].value); + } const rejected = outcomes.find((outcome) => outcome.status === "rejected"); if (rejected?.status === "rejected") { setLastError(rejected.reason instanceof Error ? rejected.reason.message : "Console data refresh failed"); @@ -116,6 +122,8 @@ export function App() { return ; case "evaluations": return ; + case "experiments": + return ; case "governance": return ; case "incidents": diff --git a/web/src/api.ts b/web/src/api.ts index eb7ad02..2084c78 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -2,6 +2,10 @@ import type { AuthSession, DashboardSummary, EvaluationIndex, + ExperimentAnalysis, + ExperimentConsole, + ExperimentDecision, + ExperimentVariant, IncidentWorkflowReport, LlmGenerationResponse, ModelCandidate, @@ -21,7 +25,7 @@ export interface LlmPayload { model_alias: string; max_tokens: number; structured: boolean; - routing_mode: "direct" | "canary"; + routing_mode: "direct" | "canary" | "experiment_ab" | "experiment_bandit"; canary_percent: number; shadow: boolean; optimized_available: boolean; @@ -29,6 +33,10 @@ export interface LlmPayload { semantic_cache_enabled: boolean; user_id: string; quota_plan: string; + experiment_id?: string; + experiment_holdback_percent?: number; + experiment_variants?: ExperimentVariant[]; + experiment_guardrail_thresholds?: Record; } export interface VtonPayload { @@ -101,6 +109,48 @@ export class TryOpsClient { return response.data; } + async experiments(): Promise { + const response = await this.get<{ data: ExperimentConsole }>("/api/experiments/summary", true); + if (!response.data) { + throw new Error("Experiment evidence unavailable"); + } + return response.data; + } + + async routeExperiment(payload: { + mode: "ab" | "bandit"; + request_id: string; + experiment_id: string; + variants: ExperimentVariant[]; + holdback_percent: number; + guardrail_thresholds: Record; + }): Promise { + const response = await this.post<{ data: ExperimentDecision }>("/api/experiments/route", { + ...payload, + api_key: this.apiKey.trim(), + workload: "llm" + }); + if (!response.data) { + throw new Error("Experiment route unavailable"); + } + return response.data; + } + + async analyzeExperiment(payload: { + experiment_id: string; + holdback: { name: string; impressions: number; rewards: number }; + variants: ExperimentVariant[]; + }): Promise { + const response = await this.post<{ data: ExperimentAnalysis }>("/api/experiments/analyze", { + ...payload, + api_key: this.apiKey.trim() + }); + if (!response.data) { + throw new Error("Experiment analysis unavailable"); + } + return response.data; + } + async vtonComparison(): Promise { const response = await this.get<{ data: VtonComparisonReport }>("/api/vton/comparison", true); if (!response.data) { diff --git a/web/src/components/ExperimentView.tsx b/web/src/components/ExperimentView.tsx new file mode 100644 index 0000000..c93b407 --- /dev/null +++ b/web/src/components/ExperimentView.tsx @@ -0,0 +1,170 @@ +import { Play, RefreshCw } from "lucide-react"; +import { useMemo, useState } from "react"; +import type { TryOpsClient } from "../api"; +import { experimentVariants } from "../data"; +import { compactJson, formatNumber, formatOptionalMs } from "../format"; +import type { ExperimentAnalysis, ExperimentConsole, ExperimentDecision } from "../types"; +import { MetricTile } from "./MetricTile"; + +interface ExperimentViewProps { + client: TryOpsClient; + experiments?: ExperimentConsole; + onRefresh: () => void; +} + +export function ExperimentView({ client, experiments, onRefresh }: ExperimentViewProps) { + const [mode, setMode] = useState<"ab" | "bandit">("bandit"); + const [decision, setDecision] = useState(); + const [analysis, setAnalysis] = useState( + experiments?.analysis_report?.native_experiment_stats + ); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(); + + const reportDecision = mode === "ab" + ? experiments?.routing_report?.decisions?.ab + : experiments?.routing_report?.decisions?.bandit; + const activeDecision = decision ?? reportDecision; + const activeAnalysis = analysis ?? experiments?.analysis_report?.native_experiment_stats; + const variants = activeDecision?.experiment?.variants ?? []; + const eligibleCount = variants.filter((variant) => variant.eligible).length; + const blockedCount = variants.length - eligibleCount; + const selected = activeDecision?.experiment?.selected?.variant ?? activeDecision?.primary_alias; + const bestVariant = activeAnalysis?.best_variant ?? "-"; + const productionReady = experiments?.production_ready ?? false; + + const analysisRows = useMemo(() => activeAnalysis?.variants ?? [], [activeAnalysis]); + + async function runDecision() { + setBusy(true); + setError(undefined); + try { + const nextDecision = await client.routeExperiment({ + mode, + request_id: `req-console-experiment-${Date.now()}`, + experiment_id: experiments?.experiment_id ?? "tryops-llm-answer-quality", + variants: experimentVariants, + holdback_percent: 5, + guardrail_thresholds: { + max_block_rate: 0.02, + max_latency_p95_ms: 120, + max_error_rate: 0.01 + } + }); + const nextAnalysis = await client.analyzeExperiment({ + experiment_id: experiments?.experiment_id ?? "tryops-llm-answer-quality", + holdback: { + name: "champion_holdback", + impressions: 1000, + rewards: 820 + }, + variants: experimentVariants.filter((variant) => variant.name !== "candidate") + }); + setDecision(nextDecision); + setAnalysis(nextAnalysis); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Experiment route failed"); + } finally { + setBusy(false); + } + } + + return ( +
+
+
+
+

Online experiments

+

A/B and bandit routing

+
+
+ + + +
+
+
+ + + + 0 ? "amber" : "green"} /> +
+ {error ?
{error}
: null} +
+ +
+
+
+

Native route

+

{activeDecision?.experiment?.source ?? "unavailable"}

+
+ + {activeDecision?.experiment?.available ? "native" : "fallback"} + +
+
+ + + + + + + + + + + + + {variants.map((variant) => ( + + + + + + + + + ))} + +
VariantStatusTrafficRewardp95Violations
{variant.name} + + {variant.eligible ? "eligible" : "blocked"} + + {formatNumber(variant.traffic_percent)}%{formatNumber(variant.reward_rate)}{formatOptionalMs(variant.latency_p95_ms)}{variant.violations?.join(", ") || "-"}
+
+
+ +
+
+

Sequential verdict

+ {bestVariant} +
+
+ {analysisRows.map((variant) => ( + + ))} +
+
+ +
+
+

Raw contract

+
+
{compactJson({ decision: activeDecision, analysis: activeAnalysis })}
+
+
+ ); +} diff --git a/web/src/components/LlmPlayground.tsx b/web/src/components/LlmPlayground.tsx index 0d9e37b..8700092 100644 --- a/web/src/components/LlmPlayground.tsx +++ b/web/src/components/LlmPlayground.tsx @@ -15,6 +15,8 @@ export function LlmPlayground({ client, onMutate }: LlmPlaygroundProps) { const [prompt, setPrompt] = useState(samplePrompt); const [modelAlias, setModelAlias] = useState("baseline"); const [quotaPlan, setQuotaPlan] = useState("free"); + const [routingMode, setRoutingMode] = useState<"direct" | "canary" | "experiment_ab" | "experiment_bandit">("direct"); + const [canaryPercent, setCanaryPercent] = useState(10); const [maxTokens, setMaxTokens] = useState(180); const [structured, setStructured] = useState(true); const [shadow, setShadow] = useState(false); @@ -32,8 +34,8 @@ export function LlmPlayground({ client, onMutate }: LlmPlaygroundProps) { model_alias: modelAlias, max_tokens: maxTokens, structured, - routing_mode: "direct", - canary_percent: 0, + routing_mode: routingMode, + canary_percent: canaryPercent, shadow, optimized_available: modelAlias !== "baseline", fallback_enabled: true, @@ -103,6 +105,25 @@ export function LlmPlayground({ client, onMutate }: LlmPlaygroundProps) { ))} + +