diff --git a/MANIFEST.txt b/MANIFEST.txt index e28151b..049fc91 100644 --- a/MANIFEST.txt +++ b/MANIFEST.txt @@ -124,6 +124,7 @@ tools/tests/test_manifest.py tools/tests/test_mesh_consume.py tools/tests/test_model_fabric_release_readiness.py tools/tests/test_operational_exhaust_fusion.py +tools/tests/test_resource_contract_verdict.py tools/tests/test_serve.py tools/tests/test_service_desk_metrics.py tools/validate_client_runtime_dump_exposure.py diff --git a/docs/devops/resource-contract-telemetry.md b/docs/devops/resource-contract-telemetry.md index bbbe16c..eb32dfa 100644 --- a/docs/devops/resource-contract-telemetry.md +++ b/docs/devops/resource-contract-telemetry.md @@ -29,14 +29,22 @@ so the loop speaks one language end to end: | condition | verdict | meaning | |---|---|---| | `enforcement != observe` and `fired_count > 0` and peak gate-eligible | **PROVED** | the control has been observed to act on real load | -| `observedPeak.value > limit.value` and `fired_count == 0` | **VIOLATION** | never-fired control — a counterexample to the claim it is a control | +| `enforcement != observe` and `observedPeak.value > limit.value` and `fired_count == 0` | **VIOLATION** | never-fired control — a counterexample to the claim it is a control | | peak not gate-eligible (`source != measured` or `unobserved > 0`) | **INCONCLUSIVE** | sufficiency unestablished; no verdict on the control can be drawn | +| `enforcement == observe` and `observedPeak.value > limit.value` | **INCONCLUSIVE** | a declared gauge, not a gate — no teeth-claim to prove or violate | -The **VIOLATION** row is the load-bearing one. A limit exceeded in production that never once -enforced is not an absence of news — it is evidence the control is paper. The normalization rule -`never_fired_control_is_a_violation` forbids dropping it, summarizing it to a healthy counter, or -downgrading it to INCONCLUSIVE: the exceedance was *measured*, so sufficiency is not the question, -enforcement is. +The **VIOLATION** row is the load-bearing one. A limit that *claimed teeth* and was exceeded in +production yet never once enforced is not an absence of news — it is evidence the control is +paper. The rule `never_fired_control_is_a_violation` forbids dropping it, summarizing it to a +healthy counter, or downgrading it to INCONCLUSIVE. + +The `enforcement != observe` guard on that row is itself load-bearing. VIOLATION is a **broken +enforcement promise**, and `observe` mode makes no promise — it is the honest, declared "gauge, +not gate" (ResourceContract requires an `observeOnlyReason` for it). An observe-mode exceedance is +therefore **INCONCLUSIVE**, per `observe_mode_exceedance_is_inconclusive_not_violation`: flagging +it VIOLATION would accuse a gauge of failing to be a gate and punish honest declaration. The +exceedance is never hidden either way — it flows as a `resource-saturation` TelemetrySignal +regardless of verdict, so an operator can still decide to promote the gauge to an enforcing limit. ## Canonical objects and topics diff --git a/mappings/resource-contract-measurement-telemetry-v1.yaml b/mappings/resource-contract-measurement-telemetry-v1.yaml index e8a62e6..c061b79 100644 --- a/mappings/resource-contract-measurement-telemetry-v1.yaml +++ b/mappings/resource-contract-measurement-telemetry-v1.yaml @@ -93,12 +93,17 @@ canonical_objects: enforcement != observe AND fired_count > 0 AND the observed peak is gate-eligible — the control has been observed to act on real load. VIOLATION: >- - observed_peak.value > limit.value AND fired_count == 0 — the limit was exceeded in - production and never once enforced. A never-fired control is a counterexample to the - claim that it is a control, not the absence of one. + enforcement != observe AND observed_peak.value > limit.value AND fired_count == 0 — a + control that CLAIMED teeth (throttle/refuse/terminate) was exceeded in production and + never once enforced. A never-fired control is a counterexample to the claim that it is + a control, not the absence of one. The enforcement != observe guard is load-bearing: + VIOLATION is a broken ENFORCEMENT PROMISE, and observe mode makes no such promise. INCONCLUSIVE: >- the observed peak is not gate-eligible (measurement_source != measured, or members were - unobserved) — sufficiency is not established, so no verdict on the control can be drawn. + unobserved), OR the contract is observe-mode — a declared gauge, not a gate, which makes + no teeth-claim to prove or violate. In neither case can a verdict on enforcement teeth + be drawn. The exceedance itself is never hidden: it still flows as a resource-saturation + TelemetrySignal regardless of verdict. normalization_rules: - id: carry_gate_eligibility_unrecomputed @@ -108,10 +113,23 @@ normalization_rules: that owns the instrument; a consumer may lower it but never raise it. - id: never_fired_control_is_a_violation rule: >- - A ResourceContract whose observed peak exceeds its limit while fired_count is zero MUST - normalize to a SufficiencyVerdict of VIOLATION and be published to ops.learning.feedback.v1. - It MUST NOT be dropped, summarized to a healthy counter, or downgraded to INCONCLUSIVE — - the exceedance was measured, so sufficiency is not the question; enforcement is. + A ResourceContract whose enforcement is not observe, whose observed peak exceeds its limit, + and whose fired_count is zero MUST normalize to a SufficiencyVerdict of VIOLATION and be + published to ops.learning.feedback.v1. It MUST NOT be dropped, summarized to a healthy + counter, or downgraded to INCONCLUSIVE — the exceedance was measured and enforcement was + claimed, so the broken promise is the finding. The enforcement != observe qualifier is + required: an observe-mode contract makes no enforcement promise and so cannot violate one + (see observe_mode_exceedance_is_inconclusive_not_violation); its exceedance still flows as + telemetry, but yields no teeth-verdict. + - id: observe_mode_exceedance_is_inconclusive_not_violation + rule: >- + A ResourceContract with enforcement == observe whose observed peak exceeds its limit MUST + normalize to INCONCLUSIVE, never VIOLATION. Observe mode is the honest, declared "gauge not + gate" (ResourceContract requires an observeOnlyReason for it). Marking it VIOLATION would + accuse a gauge of failing to be a gate and would punish honest declaration — training the + learning loop to demand fixes for things explicitly declared not-controls. The exceedance + is still published as a resource-saturation TelemetrySignal so an operator can decide + whether to promote the gauge to an enforcing limit. - id: unmeasured_peak_is_inconclusive_not_healthy rule: >- A ResourceContract whose observed peak is not gate-eligible MUST normalize to INCONCLUSIVE, diff --git a/tools/tests/test_resource_contract_verdict.py b/tools/tests/test_resource_contract_verdict.py new file mode 100644 index 0000000..0b6643b --- /dev/null +++ b/tools/tests/test_resource_contract_verdict.py @@ -0,0 +1,92 @@ +"""Teeth tests for the ResourceContract/Measurement verdict algebra. + +The verdict function `expected_verdict` in tools/validate_resource_contract_telemetry.py is the +correctness core of the whole profile — it decides PROVED / VIOLATION / INCONCLUSIVE, which is +what sociosphere's learning loop consumes. On first review that function shipped with a real +precedence bug: an observe-mode contract that exceeded its limit was returned as VIOLATION, which +accuses a declared gauge of failing to be a gate. It went unnoticed because the algebra was only +exercised indirectly by the validator script's own example, never in the pytest suite. + +These tests pin every precedence branch in `make test`, so a regression is caught even if the +worked example file changes. Each case names the input axis it isolates. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +VALIDATOR = ROOT / "tools" / "validate_resource_contract_telemetry.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("rc_telemetry_validator", VALIDATOR) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +expected_verdict = _load().expected_verdict + + +# A gate-eligible, exceeded, never-fired throttle contract — the canonical VIOLATION. +BASE = dict( + peak_value=0.94, limit_value=0.5, fired_count=0, gate_eligible=True, enforcement="throttle", +) + + +def test_canonical_violation(): + """A throttle that was exceeded and never fired is a broken enforcement promise.""" + assert expected_verdict(**BASE) == "VIOLATION" + + +def test_gate_ineligible_wins_first(): + """An unmeasured/partial peak cannot certify anything — INCONCLUSIVE outranks everything, + even an apparent exceedance.""" + assert expected_verdict(**{**BASE, "gate_eligible": False}) == "INCONCLUSIVE" + + +def test_enforcement_acted_is_proved(): + """A control that claimed teeth and was observed to fire on real load.""" + assert expected_verdict(**{**BASE, "fired_count": 3}) == "PROVED" + + +def test_within_limit_is_inconclusive(): + """No exceedance, nothing fired: no counterexample and no teeth proven. No news is not + proof of teeth.""" + assert expected_verdict(**{**BASE, "peak_value": 0.10}) == "INCONCLUSIVE" + + +def test_observe_mode_exceedance_is_inconclusive_not_violation(): + """The bug this module exists to pin. An observe-mode contract makes NO enforcement promise, + so an exceedance cannot violate one. It must be INCONCLUSIVE, never VIOLATION.""" + assert expected_verdict(**{**BASE, "enforcement": "observe"}) == "INCONCLUSIVE" + + +def test_observe_mode_never_proves_teeth(): + """Even if an observe-mode contract shows fired_count > 0 (it should not, but defensively), + observe mode cannot PROVE enforcement teeth — it isn't an enforcing mode.""" + assert expected_verdict(**{**BASE, "enforcement": "observe", "fired_count": 5}) != "PROVED" + + +@pytest.mark.parametrize("mode", ["throttle", "refuse", "terminate"]) +def test_all_enforcing_modes_violate_when_exceeded_and_unfired(mode): + """The VIOLATION rule applies to every mode that claims teeth, not just throttle.""" + assert expected_verdict(**{**BASE, "enforcement": mode}) == "VIOLATION" + + +def test_verdict_is_always_one_of_three(): + """The algebra is total: every input shape yields exactly one of the three verdicts.""" + verdicts = { + expected_verdict( + peak_value=p, limit_value=0.5, fired_count=f, gate_eligible=g, enforcement=e + ) + for p in (0.10, 0.94) + for f in (0, 3) + for g in (True, False) + for e in ("observe", "throttle", "refuse", "terminate") + } + assert verdicts <= {"PROVED", "VIOLATION", "INCONCLUSIVE"} diff --git a/tools/validate_resource_contract_telemetry.py b/tools/validate_resource_contract_telemetry.py index 89b8341..372a5d8 100644 --- a/tools/validate_resource_contract_telemetry.py +++ b/tools/validate_resource_contract_telemetry.py @@ -5,23 +5,30 @@ does two things a token-count validator does not: 1. It INDEPENDENTLY recomputes the SufficiencyVerdict from the worked example's raw inputs - (observed peak vs limit, fired_count, gate-eligibility) and asserts the example's stated - verdict matches. A validator that only checked the verdict STRING is present would pass a - mislabeled example — the exact "declared, never checked" defect this whole plane exists to - catch. So the verdict is derived here, not read. - - 2. It MUTATION-TESTS that derivation: flip fired_count and the verdict must move to PROVED; - flip the peak's gate-eligibility and it must move to INCONCLUSIVE. If the same verdict came - back under all three input shapes, the rule would be a rubber stamp and this check would be - proving nothing. - -Dependency-light on purpose: the mapping is token-checked by substring presence (no pyyaml, no -`re`), and the verdict logic runs on the JSON example via stdlib json. + (observed peak vs limit, fired_count, gate-eligibility, enforcement mode) and asserts the + example's stated verdict matches. A validator that only checked the verdict STRING is + present would pass a mislabeled example — the exact "declared, never checked" defect this + whole plane exists to catch. So the verdict is derived here, not read. + + 2. It MUTATION-TESTS that derivation across all four verdict-relevant input axes, so the rule + is proven to DISCRIMINATE rather than rubber-stamp. `tools/tests/test_resource_contract_ + verdict.py` pins the same algebra in the pytest suite (`make test`) so a regression is + caught even if this example file changes. + +The mapping is checked with ANCHORED regex over COMMENT-STRIPPED lines, not bare substring +presence. That distinction is the point of this file: a required object, rule, or topic that is +deleted from the mapping's structure but still mentioned in prose or a comment must NOT pass — +otherwise the validator is satisfiable by its own documentation, which is the paper-control +shape it exists to reject. (Substring matching, and a docstring that claimed `re` while using +neither, were both flagged on this file's first review — fixed here.) + +Dependency-light on purpose: `re` + stdlib `json`, no pyyaml. """ from __future__ import annotations import json +import re import sys from pathlib import Path @@ -32,49 +39,78 @@ REQUIRED_FILES = [MAPPING, EXAMPLE, DOC] -# The mapping must DECLARE these — the canonical objects, the three verdicts, the load-bearing -# normalization rules, and the learning-feedback topic that closes the loop. -REQUIRED_MAPPING_TOKENS = [ - "profile: resource-contract-measurement-telemetry", - "signal_class: resource-saturation", - "TelemetrySignal", - "EvidenceArtifact", - "SufficiencyVerdict", - "measurement_gate_eligible", - "PROVED", - "VIOLATION", - "INCONCLUSIVE", - "never_fired_control_is_a_violation", - "unmeasured_peak_is_inconclusive_not_healthy", - "carry_gate_eligibility_unrecomputed", - "acting_enforcement_requires_resolvable_negative_control", - "preserve_scope_against_fanout", - "ops.telemetry.signals.v1", - "ops.evidence.artifacts.v1", - "ops.learning.feedback.v1", +# Anchored, structural patterns — each must match a real YAML key / list-item at line start +# (after comments are stripped), never merely appear in prose. This is what makes the check +# resistant to "removed from structure, still named in a comment" regressions. +REQUIRED_STRUCTURAL_PATTERNS = [ + r"^profile:\s*resource-contract-measurement-telemetry\b", + r"^\s*TelemetrySignal:\s*$", + r"^\s*EvidenceArtifact:\s*$", + r"^\s*SufficiencyVerdict:\s*$", + r"^\s*signal_class:\s*resource-saturation\b", + r"^\s*-\s*measurement_gate_eligible\b", + # the three verdicts as KEYS under `verdicts:`, not as words in a sentence + r"^\s*PROVED:", + r"^\s*VIOLATION:", + r"^\s*INCONCLUSIVE:", + # the five normalization rules, by id + r"^\s*-\s*id:\s*carry_gate_eligibility_unrecomputed\b", + r"^\s*-\s*id:\s*never_fired_control_is_a_violation\b", + r"^\s*-\s*id:\s*observe_mode_exceedance_is_inconclusive_not_violation\b", + r"^\s*-\s*id:\s*unmeasured_peak_is_inconclusive_not_healthy\b", + r"^\s*-\s*id:\s*acting_enforcement_requires_resolvable_negative_control\b", + r"^\s*-\s*id:\s*preserve_scope_against_fanout\b", + # the three ops topics, as mapping keys + r"^\s*ops\.telemetry\.signals\.v1:", + r"^\s*ops\.evidence\.artifacts\.v1:", + r"^\s*ops\.learning\.feedback\.v1:", ] +def _strip_yaml_comments(text: str) -> str: + """Drop `#` comments so structural checks cannot be satisfied by commentary. + + The mapping carries no `#` inside quoted values, so cutting each line at its first `#` is + sufficient and keeps this dependency-light (no YAML parser). Full-line comments become empty + lines; inline comments (e.g. `- measurement_gate_eligible # carried through`) lose the tail. + """ + out = [] + for line in text.splitlines(): + h = line.find("#") + out.append(line if h < 0 else line[:h]) + return "\n".join(out) + + def expected_verdict(*, peak_value, limit_value, fired_count, gate_eligible, enforcement): """The verdict algebra, reimplemented independently of the mapping prose and the example. - Precedence matters and is deliberate: - - not gate-eligible wins first: an unmeasured/partial peak cannot certify anything, so no - verdict on the control can be drawn -> INCONCLUSIVE. - - then an acting enforcement that has fired -> PROVED: the control was observed to act. - - then an exceeded limit that never fired -> VIOLATION: the never-fired control. This - holds in ANY mode, observe included — an observe-mode control must still record the - breach, so silence when the limit was exceeded is itself the failure being asserted - (see the mapping rule `never_fired_control_is_a_violation`, which carries no observe - carve-out either). - - otherwise (the limit held, or a control that did fire but — being observe-mode — proves - no teeth) there is no counterexample -> INCONCLUSIVE (no news is not proof of teeth). + Precedence, and why each step is where it is: + + 1. not gate-eligible -> INCONCLUSIVE. An unmeasured or partially-observed peak cannot + certify anything, so no verdict on the control can be drawn. This wins first. + + 2. enforcement != "observe" AND fired_count > 0 -> PROVED. A control that CLAIMS teeth + (throttle/refuse/terminate) and was observed to act on real load. + + 3. enforcement != "observe" AND peak > limit AND fired_count == 0 -> VIOLATION. A control + that CLAIMED teeth, was exceeded, and never once bit — a counterexample to the claim + that it is a control. The `enforcement != "observe"` guard is load-bearing: VIOLATION + is a BROKEN ENFORCEMENT PROMISE, and `observe` mode makes no such promise. It is the + honest, declared "I am a gauge, not a gate" (ResourceContract requires an + `observeOnlyReason` for it). Flagging an observe-mode exceedance as VIOLATION would + accuse a gauge of failing to be a gate, punish honest declaration, and train the + learning loop to demand fixes for things explicitly declared not-controls. The + exceedance is NOT hidden by this — it still flows as a resource-saturation + TelemetrySignal regardless of verdict; only the teeth-verdict is withheld. + + 4. otherwise -> INCONCLUSIVE. The limit held, or the contract is observe-mode: no + counterexample and no teeth proven. No news is not proof of teeth. """ if not gate_eligible: return "INCONCLUSIVE" if enforcement != "observe" and fired_count > 0: return "PROVED" - if peak_value > limit_value and fired_count == 0: + if enforcement != "observe" and peak_value > limit_value and fired_count == 0: return "VIOLATION" return "INCONCLUSIVE" @@ -83,7 +119,7 @@ def main() -> int: failures: list[str] = [] checks = 0 - # ── files + mapping tokens ────────────────────────────────────────────────── + # ── files ─────────────────────────────────────────────────────────────────── for f in REQUIRED_FILES: checks += 1 if not f.is_file(): @@ -93,13 +129,15 @@ def main() -> int: print(f" {m}", file=sys.stderr) return 1 - mapping_text = MAPPING.read_text(encoding="utf-8") - for tok in REQUIRED_MAPPING_TOKENS: + # ── mapping STRUCTURE (anchored, comment-stripped) ────────────────────────── + mapping_body = _strip_yaml_comments(MAPPING.read_text(encoding="utf-8")) + for pat in REQUIRED_STRUCTURAL_PATTERNS: checks += 1 - if tok not in mapping_text: - failures.append(f"mapping missing required token: {tok!r}") + if not re.search(pat, mapping_body, re.MULTILINE): + failures.append(f"mapping missing required STRUCTURE (anchored): {pat!r}") print(f" {'OK ' if not failures else 'FAIL'} mapping declares " - f"{len(REQUIRED_MAPPING_TOKENS)} required tokens (objects, verdicts, rules, topics)") + f"{len(REQUIRED_STRUCTURAL_PATTERNS)} required structural elements " + "(anchored keys/list-items over comment-stripped lines, not prose)") # ── the teeth: derive the example's verdict and check it matches ──────────── example = json.loads(EXAMPLE.read_text(encoding="utf-8")) @@ -119,15 +157,15 @@ def main() -> int: failures.append( f"example's stated verdict {stated!r} does not match the verdict derived from its " f"own inputs ({got!r}): peak={peak['value']} limit={rc['limit']['value']} " - f"fired={rc['firedCount']} gateEligible={peak['gateEligible']}" + f"fired={rc['firedCount']} gateEligible={peak['gateEligible']} " + f"enforcement={rc['enforcement']!r}" ) print(f" FAIL example verdict mislabeled: stated {stated}, derived {got}") else: print(f" OK example verdict {stated} is what its own inputs derive " - f"(peak {peak['value']} > limit {rc['limit']['value']}, fired {rc['firedCount']})") + f"(enforcement {rc['enforcement']}, peak {peak['value']} > " + f"limit {rc['limit']['value']}, fired {rc['firedCount']})") - # sanity: the shipped example must actually exercise the load-bearing VIOLATION row, or this - # profile's whole reason for existing is untested. checks += 1 if stated != "VIOLATION": failures.append( @@ -137,7 +175,7 @@ def main() -> int: else: print(" OK shipped example exercises the load-bearing VIOLATION row") - # ── mutation test: the derivation must discriminate, not rubber-stamp ──────── + # ── mutation test across ALL FOUR axes, incl. the observe-mode carve-out ──── print(" mutation test — the verdict moves when the inputs move") base = dict( peak_value=peak["value"], limit_value=rc["limit"]["value"], @@ -148,6 +186,8 @@ def main() -> int: ("fired_count -> 3 (enforcement acted)", {**base, "fired_count": 3}, "PROVED"), ("gate_eligible -> False (unmeasured peak)", {**base, "gate_eligible": False}, "INCONCLUSIVE"), ("peak within limit, never fired", {**base, "peak_value": 0.10}, "INCONCLUSIVE"), + ("enforcement -> observe, same exceedance (gauge, not gate)", + {**base, "enforcement": "observe"}, "INCONCLUSIVE"), ] for label, kw, want in mutations: checks += 1 @@ -156,7 +196,6 @@ def main() -> int: failures.append(f"mutation {label}: expected {want}, derived {got_m}") else: print(f" OK {label} -> {want}") - # if every mutation returned VIOLATION we'd never get here clean — that's the rubber-stamp guard if failures: print(f"\n{len(failures)} failure(s) of {checks} checks:", file=sys.stderr) @@ -164,10 +203,10 @@ def main() -> int: print(f" {m}", file=sys.stderr) return 1 - print(f"\nOK resource-contract-measurement-telemetry: {checks} checks. The mapping declares " - "the loop (telemetry/evidence/learning topics), and the worked example's VIOLATION " + print(f"\nOK resource-contract-measurement-telemetry: {checks} checks. The mapping's " + "structure is anchored (not satisfiable by prose), and the worked example's VIOLATION " "verdict is derived from its own inputs and mutation-tested to discriminate PROVED / " - "VIOLATION / INCONCLUSIVE.") + "VIOLATION / INCONCLUSIVE, including the observe-mode carve-out.") return 0