From 8b4f6d853cc0f11f52cf6039de2a03aa0c2043f3 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:38:45 +0900 Subject: [PATCH 1/2] config: bind audited diagnostic pilot admission Pin existing obligations and exact budget genesis identities; add one frozen diagnostic case without changing credentials or provider rates. Agent-assisted-by: OpenAI Codex Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- .../pr133-diagnostic-case05.json | 30 +++++++++++++++++++ tools/skill_eval_auto_config.json | 19 ++++++++++-- tools/skill_eval_pr_cases.json | 22 ++++++++++++-- 3 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 docs/examples/skill-comparison/pr133-diagnostic-case05.json diff --git a/docs/examples/skill-comparison/pr133-diagnostic-case05.json b/docs/examples/skill-comparison/pr133-diagnostic-case05.json new file mode 100644 index 000000000..a89500cab --- /dev/null +++ b/docs/examples/skill-comparison/pr133-diagnostic-case05.json @@ -0,0 +1,30 @@ +[ + { + "id": "case-05", + "split": "holdout", + "request": "Diagnose this blocked request at the recorded decision boundary; propose the next read-only check.", + "provenance": "Synthetic finite-state observations; not a production trace or engine benchmark.", + "events": [ + { + "id": "E1", + "kind": "cache_lookup", + "value": "hit" + }, + { + "id": "E2", + "kind": "receive", + "value": "pending" + }, + { + "id": "E3", + "kind": "identity", + "value": "match" + }, + { + "id": "E4", + "kind": "allocator", + "value": "denied" + } + ] + } +] diff --git a/tools/skill_eval_auto_config.json b/tools/skill_eval_auto_config.json index 451640eda..8c7e5735c 100644 --- a/tools/skill_eval_auto_config.json +++ b/tools/skill_eval_auto_config.json @@ -1,5 +1,5 @@ { - "schema_version": 1, + "schema_version": 2, "repository_id": 1409534281, "label": "skill-eval:execute", "operators": [ @@ -57,7 +57,22 @@ "ref": "refs/heads/skill-eval-budget/1409534281/160", "genesis_commit": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668", "checkpoint_commit": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668" + }, + "133": { + "ref": "refs/heads/skill-eval-budget/1409534281/133", + "genesis_commit": "be3164546eb76d073ccd3ddd29cfd18794955bd0", + "checkpoint_commit": "be3164546eb76d073ccd3ddd29cfd18794955bd0" } }, - "review_evidence": "https://github.com/InferMatrixLab/InferRSI/issues/215#issuecomment-6467531341" + "review_evidence": "https://github.com/InferMatrixLab/InferRSI/issues/235#issuecomment-6486272232", + "daily_budget": { + "currency": "USD", + "amount": "1000", + "timezone": "Asia/Shanghai", + "anchor": { + "ref": "refs/heads/skill-eval-budget/1409534281/daily", + "genesis_commit": "1d3df4ea3924c2b4ce92f051becf566839d66c9c", + "checkpoint_commit": "1d3df4ea3924c2b4ce92f051becf566839d66c9c" + } + } } diff --git a/tools/skill_eval_pr_cases.json b/tools/skill_eval_pr_cases.json index 9d7cbe6cb..f96c64f7b 100644 --- a/tools/skill_eval_pr_cases.json +++ b/tools/skill_eval_pr_cases.json @@ -1,6 +1,9 @@ { "skill_root": "src/inferrsi/data/skill_library", - "model": {"name": "gpt-6.1-sol", "effort": "xhigh"}, + "model": { + "name": "gpt-6.1-sol", + "effort": "xhigh" + }, "limits": { "repeats": 1, "max_task_calls": 2, @@ -12,14 +15,27 @@ { "skill": "inference-benchmark-design", "entrypoint": "methods/inference-benchmark-design/SKILL.md", - "paths": ["src/inferrsi/data/skill_library/methods/inference-benchmark-design"], + "paths": [ + "src/inferrsi/data/skill_library/methods/inference-benchmark-design" + ], "cases": "../docs/examples/skill-comparison/cases.json" }, { "skill": "audit-fixed-workload-claim", "entrypoint": "methods/audit-fixed-workload-claim/SKILL.md", - "paths": ["src/inferrsi/data/skill_library/methods/audit-fixed-workload-claim"], + "paths": [ + "src/inferrsi/data/skill_library/methods/audit-fixed-workload-claim" + ], "cases": "../docs/examples/skill-comparison/cases.json" + }, + { + "skill": "optimization-search-policy", + "entrypoint": "methods/optimization-search-policy/SKILL.md", + "paths": [ + "src/inferrsi/data/skill_library/methods/optimization-search-policy" + ], + "cases": "../docs/examples/skill-comparison/pr133-diagnostic-case05.json", + "adapter": "kv-admission-diagnostic-v1" } ] } From a13e0a143a5b84e0d63a12b954bee13324efd2c2 Mon Sep 17 00:00:00 2001 From: Dong1017 <55723162+Dong1017@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:45:48 +0900 Subject: [PATCH 2/2] test: bind deployed daily ledger and preserve legacy fixture Retain strict anchor and expiry assertions while testing schema-one migration independently of live configuration. Agent-assisted-by: OpenAI Codex Signed-off-by: Dong1017 <55723162+Dong1017@users.noreply.github.com> --- tests/test_skill_eval_auto_daily.py | 3 +++ tests/test_skill_eval_deployment.py | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/test_skill_eval_auto_daily.py b/tests/test_skill_eval_auto_daily.py index effe6b2bf..c2eda0042 100644 --- a/tests/test_skill_eval_auto_daily.py +++ b/tests/test_skill_eval_auto_daily.py @@ -185,6 +185,9 @@ def test_missing_global_history_never_writes_pr_reservation(self): def test_configuration_migration_preserves_legacy_read_only(self): legacy = json.loads(auto.CONFIG.read_text()) + legacy["schema_version"] = 1 + legacy.pop("daily_budget", None) + legacy["anchors"].pop("133", None) path = self.output / "config.json" path.write_text(json.dumps(legacy)) self.assertEqual(auto.configuration(path, now=0), legacy) diff --git a/tests/test_skill_eval_deployment.py b/tests/test_skill_eval_deployment.py index 0e080c38b..8b25a27e7 100644 --- a/tests/test_skill_eval_deployment.py +++ b/tests/test_skill_eval_deployment.py @@ -24,13 +24,21 @@ def test_deployment_identity_and_expiry_are_closed(deployed): assert deployed["repository_id"] == 1409534281 assert deployed["operators"] == ["Dong1017"] assert deployed["label"] == "skill-eval:execute" - expected = {"159": "727d31e55f496c53fbf02cf4f624008487de27cb", + expected = {"133": "be3164546eb76d073ccd3ddd29cfd18794955bd0", + "159": "727d31e55f496c53fbf02cf4f624008487de27cb", "160": "1fdbd8adc35e2df7439c3cd74780f869d2d6e668"} assert set(deployed["anchors"]) == set(expected) for number, sha in expected.items(): assert deployed["anchors"][number] == { "ref": f"refs/heads/skill-eval-budget/1409534281/{number}", "genesis_commit": sha, "checkpoint_commit": sha} + assert deployed["schema_version"] == 2 + assert deployed["daily_budget"] == { + "currency": "USD", "amount": "1000", "timezone": "Asia/Shanghai", + "anchor": { + "ref": "refs/heads/skill-eval-budget/1409534281/daily", + "genesis_commit": "1d3df4ea3924c2b4ce92f051becf566839d66c9c", + "checkpoint_commit": "1d3df4ea3924c2b4ce92f051becf566839d66c9c"}} assert deployed["expires_at"] == OBSERVED + 86400 assert deployed["provider"]["pricing"]["expires_at"] == deployed["expires_at"] assert deployed["provider_sha256"] == pr.fingerprint(deployed["provider"])