Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
9974fc9
feat: enforce signed V3 release evidence gates
EvanProgramming Oct 21, 2026
054a74f
fix: keep release evidence outside candidate checkout
EvanProgramming Oct 22, 2026
e2df426
fix: publish release manifest only after signing
EvanProgramming Oct 22, 2026
ad6ad41
fix: prevent release manifest overwrite races
EvanProgramming Oct 22, 2026
31bcc39
feat: collect exact-candidate V3 platform evidence
EvanProgramming Oct 22, 2026
3e01e12
test: run R266 evidence matrix on release PRs
EvanProgramming Oct 22, 2026
70ac495
feat: validate V3 release candidate platforms
EvanProgramming Oct 22, 2026
c3ff7aa
test: make gateway acceptance proxy-safe
EvanProgramming Oct 22, 2026
e1bb0d7
fix: make production soak dependency-free
EvanProgramming Oct 22, 2026
2b6a663
test: isolate startup log readiness from loopback
EvanProgramming Oct 22, 2026
c0fb1e1
test: wait for TUI turn completion
EvanProgramming Oct 22, 2026
cf68555
fix(memory): close vector index on application shutdown
EvanProgramming Oct 22, 2026
93a78ee
fix(memory): release orphaned vector clients
EvanProgramming Oct 22, 2026
b7a376c
fix(tui): emit ready after clearing busy state
EvanProgramming Oct 22, 2026
187d882
fix(tests): close temporary memory bank after bug workflow
EvanProgramming Oct 22, 2026
eb4d2a3
test(tui): dismiss error dialog before update command
EvanProgramming Oct 22, 2026
b9ff1e4
test: disable unused vector index in bug workflow
EvanProgramming Oct 22, 2026
3ca4e2c
ci: gate Windows V3 validation on supported features
EvanProgramming Oct 22, 2026
b7fad07
ci: force UTF-8 for Windows feature acceptance
EvanProgramming Oct 23, 2026
7bcff3b
test(tui): isolate Windows home directory
EvanProgramming Oct 23, 2026
c912686
ci: record Windows PTY acceptance limitation
EvanProgramming Oct 23, 2026
e848ed9
fix(soak): measure Windows handle growth from baseline
EvanProgramming Oct 23, 2026
cf0a694
fix(sqlite): apply intended sync mode to runtime connections
EvanProgramming Oct 23, 2026
a611b07
fix(cli): honor stdout encoding for Windows rendering
EvanProgramming Oct 23, 2026
59f261e
test: stabilize cross-platform soak and TUI checks
EvanProgramming Oct 23, 2026
415b492
fix(ci): invoke release evidence collector as module
EvanProgramming Oct 23, 2026
e3f7c46
test: allow scheduling delay in cancellation checks
EvanProgramming Oct 23, 2026
030268a
merge: stack release evidence on matched evaluation
EvanProgramming Oct 23, 2026
2a5a20f
fix: extend soak task completion window
EvanProgramming Oct 23, 2026
5f5a612
test: allow macOS delegation review to finish
EvanProgramming Oct 23, 2026
ad13af4
merge: update release evidence candidate cap
EvanProgramming Oct 23, 2026
2104cd3
fix: validate Windows TUI evidence status
EvanProgramming Oct 23, 2026
9b7c3bb
merge: update release evidence budget carry-forward
EvanProgramming Oct 24, 2026
36882e6
merge: refresh release evidence candidate accounting
EvanProgramming Oct 24, 2026
9c6ef88
merge: include matched benchmark resume fix
EvanProgramming Oct 24, 2026
46df10f
feat: allow signed limited-evidence V3 previews
EvanProgramming Oct 26, 2026
396a2d0
merge: update matched-evaluation base
EvanProgramming Oct 27, 2026
1656066
test: diagnose delegated provider startup timeouts
EvanProgramming Oct 27, 2026
d4679da
fix(release): require complete preview safety evidence
EvanProgramming Oct 27, 2026
0b8ebab
fix(release): parse multiline safety test results
EvanProgramming Oct 27, 2026
cb5087b
Merge branch 'Evan/v3-matched-evaluation' into Evan/v3-release-evidence
EvanProgramming Oct 27, 2026
7f1883d
fix: close benchmark ledger connections
EvanProgramming Oct 28, 2026
6eb881a
fix: verify release evidence snapshots
EvanProgramming Oct 28, 2026
5fc2e62
test: reject failed V3 security gate waiver
EvanProgramming Oct 28, 2026
0fa7551
fix(ci): fetch pinned V2 baseline for matched tests
EvanProgramming Oct 28, 2026
668a3e4
docs: document pinned V2 checkout requirement
EvanProgramming Oct 28, 2026
0734dbb
fix: reject observed protocol recovery failures
EvanProgramming Oct 29, 2026
86e22dd
merge: update V3 evaluation dependency
EvanProgramming Oct 30, 2026
8ce7ec9
feat: gate V3 preview publication
EvanProgramming Oct 30, 2026
0f172f9
chore: sign preview candidate with verified identity
EvanProgramming Oct 30, 2026
c942491
test: exercise preview updates and diagnose cancellation timeouts
EvanProgramming Oct 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
222 changes: 222 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ on:
pull_request:
branches: [main]
workflow_dispatch:
inputs:
v3_release_candidate_sha:
description: Full candidate SHA for the six-platform V3 release gate
required: false
type: string
schedule:
# Monday 04:17 Asia/Shanghai.
- cron: "17 20 * * 0"
Expand All @@ -30,6 +35,7 @@ jobs:
all: ${{ steps.classify.outputs.all }}
core: ${{ steps.classify.outputs.core }}
full: ${{ steps.classify.outputs.full }}
release: ${{ steps.classify.outputs.release }}
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -41,9 +47,11 @@ jobs:
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
HEAD_SHA: ${{ github.sha }}
FULL_RUN: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
RELEASE_CANDIDATE_SHA: ${{ inputs.v3_release_candidate_sha }}
run: |
args=(--event "$EVENT_NAME" --base "$BASE_SHA" --head "$HEAD_SHA")
if [[ "$FULL_RUN" == true ]]; then args+=(--full); fi
if [[ -n "$RELEASE_CANDIDATE_SHA" ]]; then args+=(--release-candidate-sha "$RELEASE_CANDIDATE_SHA"); fi
python scripts/ci_changes.py "${args[@]}"

core-test:
Expand All @@ -54,6 +62,10 @@ jobs:
KYROZEN_SANDBOX_REQUIRED: "1"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Verify pinned V2 benchmark baseline
run: git cat-file -e 1bfc82c793df49e76097061ec5adfef0e4e2e883^{commit}
- uses: actions/setup-python@v5
with:
python-version: "3.12"
Expand Down Expand Up @@ -86,6 +98,10 @@ jobs:
KYROZEN_SANDBOX_REQUIRED: "1"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Verify pinned V2 benchmark baseline
run: git cat-file -e 1bfc82c793df49e76097061ec5adfef0e4e2e883^{commit}
- uses: actions/setup-python@v5
with:
python-version: "3.13"
Expand Down Expand Up @@ -126,6 +142,10 @@ jobs:
KYROZEN_SANDBOX_REQUIRED: "1"
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Verify pinned V2 benchmark baseline
run: git cat-file -e 1bfc82c793df49e76097061ec5adfef0e4e2e883^{commit}
- uses: actions/setup-python@v5
with:
python-version: "3.12"
Expand Down Expand Up @@ -257,6 +277,199 @@ jobs:
name: update-acceptance-${{ matrix.os }}
path: update-acceptance.json

v3-platform-validation:
needs: changes
if: needs.changes.outputs.release == 'true'
name: V3 production matrix (${{ matrix.platform }})
runs-on: ${{ matrix.os }}
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
include:
- { platform: linux-py312, os: ubuntu-latest, python: '3.12' }
- { platform: linux-py313, os: ubuntu-latest, python: '3.13' }
- { platform: macos-py312, os: macos-latest, python: '3.12' }
- { platform: macos-py313, os: macos-latest, python: '3.13' }
- { platform: windows-py312, os: windows-latest, python: '3.12' }
- { platform: windows-py313, os: windows-latest, python: '3.13' }
env:
CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }}
PLATFORM_ID: ${{ matrix.platform }}
KYROZEN_BROWSER_TESTS: '1'
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }}
- name: Verify pinned V2 benchmark baseline
if: runner.os != 'Windows'
shell: bash
run: git cat-file -e 1bfc82c793df49e76097061ec5adfef0e4e2e883^{commit}
- name: Verify exact candidate
shell: bash
run: |
[[ "$CANDIDATE_SHA" =~ ^([0-9a-f]{40}|[0-9a-f]{64})$ ]]
test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA"
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
cache: pip
cache-dependency-path: |
pyproject.toml
requirements.txt
- uses: actions/setup-go@v5
with:
go-version: '1.27.1'
cache-dependency-path: tui/go.sum
- name: Install Linux sandbox and browser system dependencies
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install --no-install-recommends -y bubblewrap
sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
python -m pip install playwright
python -m playwright install-deps chromium
- name: Install project and browser dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install -e '.[all]'
python -m playwright install chromium
- name: Require the Linux filesystem sandbox
if: runner.os == 'Linux'
shell: bash
run: echo 'KYROZEN_SANDBOX_REQUIRED=1' >> "$GITHUB_ENV"
- name: Run complete unit suite
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
python -m unittest discover -s tests -p 'test_*.py' -v 2>&1 | tee unit.log
- name: Run integration suite
if: runner.os != 'Windows'
shell: bash
run: |
set -euo pipefail
PYTHONPATH=tests python -m unittest tests.test_durable_tasks_gateway tests.test_gateway_startup_logging -v 2>&1 | tee integration.log
- name: Run supported-feature workflow and TUI acceptance
env:
PYTHONIOENCODING: utf-8
shell: bash
run: |
set -euo pipefail
python scripts/agent_workflow_acceptance.py 2>&1 | tee agent-acceptance.log
python scripts/subagent_workflow_acceptance.py 2>&1 | tee subagent-acceptance.log
binary="../kyrozen-tui"
if [[ "$RUNNER_OS" == Windows ]]; then binary="../kyrozen-tui.exe"; fi
(cd tui && go test ./... && go build -o "$binary" .) 2>&1 | tee tui-tests.log
if [[ "$RUNNER_OS" == Windows ]]; then
echo 'not_run: interactive PTY acceptance requires POSIX fcntl/pty; Windows Go TUI tests/build and agent workflow acceptance are required.' | tee tui-acceptance.log
else
python scripts/tui_workflow_acceptance.py "${binary#../}" 2>&1 | tee tui-acceptance.log
fi
- name: Run 30-minute production soak
shell: bash
run: |
set -euo pipefail
python scripts/production_soak.py --seconds 1800 \
--output soak.json --log-dir soak-logs 2>&1 | tee soak.log
- name: Record platform result
if: success()
shell: bash
run: |
python - <<'PY'
import importlib.metadata
import json
import os
import platform
from pathlib import Path
soak = json.loads(Path("soak.json").read_text(encoding="utf-8"))
dependencies = {
distribution.metadata["Name"]: distribution.version
for distribution in importlib.metadata.distributions()
if distribution.metadata.get("Name")
}
result = {
"schema_version": 1,
"candidate_sha": os.environ["CANDIDATE_SHA"],
"platform": os.environ["PLATFORM_ID"],
"unit": "not_run" if os.environ["RUNNER_OS"] == "Windows" else "passed",
"integration": "not_run" if os.environ["RUNNER_OS"] == "Windows" else "passed",
"acceptance": "passed",
"tui_interactive_acceptance": "not_run_posix_pty_unavailable" if os.environ["RUNNER_OS"] == "Windows" else "passed",
"soak_seconds": int(soak["elapsed_seconds"]),
"environment": {
"runner": os.environ["ImageOS"] if "ImageOS" in os.environ else os.environ["RUNNER_OS"],
"os": platform.platform(),
"python": platform.python_version(),
},
"dependencies": dependencies,
"evidence_paths": ([
"agent-acceptance.log", "subagent-acceptance.log", "tui-tests.log",
"tui-acceptance.log", "soak.log", "soak.json",
] if os.environ["RUNNER_OS"] == "Windows" else [
"unit.log", "integration.log", "agent-acceptance.log",
"subagent-acceptance.log", "tui-tests.log", "tui-acceptance.log",
"soak.log", "soak.json",
]),
}
if os.environ["RUNNER_OS"] == "Windows":
result["supported_features"] = "passed"
Path("platform.json").write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8")
PY
- name: Upload platform logs and result
if: always()
uses: actions/upload-artifact@v4
with:
name: v3-platform-${{ matrix.platform }}
path: |
platform.json
unit.log
integration.log
agent-acceptance.log
subagent-acceptance.log
tui-acceptance.log
tui-tests.log
soak.log
soak.json
soak-logs/
if-no-files-found: warn
retention-days: 30

collect-r266:
needs: [changes, v3-platform-validation]
if: needs.changes.outputs.release == 'true' && needs.v3-platform-validation.result == 'success'
runs-on: ubuntu-latest
env:
CANDIDATE_SHA: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }}
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }}
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- uses: actions/download-artifact@v4
with:
pattern: v3-platform-*
path: platform-results
merge-multiple: false
- name: Collect unsigned exact-candidate R266 record
run: >-
python -m benchmarks.v3_release_platforms
--results-dir platform-results
--output-dir v3-r266-evidence
--candidate-sha "$CANDIDATE_SHA"
--command-set .github/workflows/ci.yml
- name: Upload unsigned R266 evidence bundle
uses: actions/upload-artifact@v4
with:
name: v3-r266-${{ github.event.pull_request.head.sha || inputs.v3_release_candidate_sha || github.sha }}
path: v3-r266-evidence/
if-no-files-found: error
retention-days: 30

ci-result:
name: CI result
if: always()
Expand All @@ -270,6 +483,8 @@ jobs:
- installer-syntax
- docker-persistence
- update-install
- v3-platform-validation
- collect-r266
runs-on: ubuntu-latest
steps:
- name: Require every selected check to pass
Expand All @@ -290,6 +505,9 @@ jobs:
UPDATE_RESULT: ${{ needs.update-install.result }}
DOCKER_EXPECTED: ${{ needs.changes.outputs.docker }}
DOCKER_RESULT: ${{ needs.docker-persistence.result }}
RELEASE_EXPECTED: ${{ needs.changes.outputs.release }}
RELEASE_MATRIX_RESULT: ${{ needs.v3-platform-validation.result }}
RELEASE_COLLECT_RESULT: ${{ needs.collect-r266.result }}
run: |
set -euo pipefail
test "$CHANGES_RESULT" = success
Expand All @@ -315,6 +533,8 @@ jobs:
require_selected "Installer syntax" "$UPDATE_EXPECTED" "$INSTALLER_RESULT"
require_selected "Installed update" "$UPDATE_EXPECTED" "$UPDATE_RESULT"
require_selected "Docker persistence" "$DOCKER_EXPECTED" "$DOCKER_RESULT"
require_selected "V3 release matrix" "$RELEASE_EXPECTED" "$RELEASE_MATRIX_RESULT"
require_selected "V3 R266 evidence collection" "$RELEASE_EXPECTED" "$RELEASE_COLLECT_RESULT"
if [ "$CORE_EXPECTED" != true ]; then require_unselected "Unselected core job" "$CORE_RESULT"; fi
if [ "$PYTHON_EXPECTED" != true ]; then require_unselected "Unselected Python tests" "$PYTHON_RESULT"; fi
if [ "$ALL_EXPECTED" != true ]; then require_unselected "Unselected full Python 3.12 tests" "$PYTHON_312_RESULT"; fi
Expand All @@ -323,3 +543,5 @@ jobs:
require_unselected "Unselected installer syntax" "$INSTALLER_RESULT"
require_unselected "Unselected installed update" "$UPDATE_RESULT"
require_unselected "Unselected Docker persistence" "$DOCKER_RESULT"
if [ "$RELEASE_EXPECTED" != true ]; then require_unselected "Unselected V3 release matrix" "$RELEASE_MATRIX_RESULT"; fi
if [ "$RELEASE_EXPECTED" != true ]; then require_unselected "Unselected V3 R266 collection" "$RELEASE_COLLECT_RESULT"; fi
2 changes: 2 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on:
push:
tags:
- "v*"
- "!v3.*"
workflow_dispatch:
inputs:
release_tag:
Expand All @@ -16,6 +17,7 @@ permissions:

jobs:
build-release:
if: ${{ !startsWith(inputs.release_tag || github.ref_name, 'v3.') }}
runs-on: ubuntu-latest
env:
RELEASE_TAG: ${{ inputs.release_tag || github.ref_name }}
Expand Down
12 changes: 7 additions & 5 deletions benchmarks/v3_matched.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Budgeted analysis primitives for the matched OpenKyrozen V2/V3 evaluation."""
from __future__ import annotations

from contextlib import closing
import hashlib
import json
import random
Expand Down Expand Up @@ -57,7 +58,7 @@ def __init__(self, path: Path, *, limit_rmb: str | Decimal = MAX_LIMIT_RMB,
raise ValueError("the cumulative benchmark cap must be at most USD 20 at the recorded exchange rate")
self.limit_picormb = _picormb(limit)
self.path.parent.mkdir(parents=True, exist_ok=True)
with sqlite3.connect(self.path) as db:
with closing(sqlite3.connect(self.path)) as db:
db.execute("PRAGMA journal_mode=WAL")
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
if "budget_config" in tables:
Expand Down Expand Up @@ -105,9 +106,10 @@ def __init__(self, path: Path, *, limit_rmb: str | Decimal = MAX_LIMIT_RMB,
"SELECT reserved_picormb FROM attempts WHERE id='prior-campaign-reservations'"
).fetchone() != (_picormb(prior_hold),):
raise ValueError("the persisted ledger is missing the prior campaign reservation")
db.commit()

def _totals(self) -> tuple[int, int, bool]:
with sqlite3.connect(self.path) as db:
with closing(sqlite3.connect(self.path)) as db:
spent, held, overruns = db.execute(
"SELECT COALESCE(SUM(actual_picormb), 0), "
"COALESCE(SUM(CASE WHEN status IN ('reserved','unknown') "
Expand Down Expand Up @@ -136,7 +138,7 @@ def reserve(self, attempt_id: str, *, input_token_bound: int, output_token_bound
if not attempt_id or min(input_token_bound, output_token_bound) < 0:
raise ValueError("attempt ID and nonnegative token bounds are required")
reserve = _picormb(_cost(input_token_bound, output_token_bound))
with sqlite3.connect(self.path, timeout=30, isolation_level=None) as db:
with closing(sqlite3.connect(self.path, timeout=30, isolation_level=None)) as db:
db.execute("BEGIN IMMEDIATE")
spent, held, overruns = db.execute(
"SELECT COALESCE(SUM(actual_picormb), 0), "
Expand All @@ -158,7 +160,7 @@ def reserve(self, attempt_id: str, *, input_token_bound: int, output_token_bound
return True

def campaign_calls(self) -> list[dict]:
with sqlite3.connect(self.path) as db:
with closing(sqlite3.connect(self.path)) as db:
attempts = db.execute(
"SELECT id,reserved_picormb,actual_picormb,status,usage_json,metadata_json "
"FROM attempts ORDER BY created_at,id"
Expand All @@ -184,7 +186,7 @@ def campaign_calls(self) -> list[dict]:
return calls

def settle(self, attempt_id: str, usage: dict | None) -> None:
with sqlite3.connect(self.path, timeout=30, isolation_level=None) as db:
with closing(sqlite3.connect(self.path, timeout=30, isolation_level=None)) as db:
db.execute("BEGIN IMMEDIATE")
row = db.execute("SELECT reserved_picormb,status FROM attempts WHERE id=?",
(attempt_id,)).fetchone()
Expand Down
Loading
Loading