diff --git a/bench/load/README.md b/bench/load/README.md index c79f6188a..2678f8dc3 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -79,6 +79,348 @@ and native paired edit scenarios). This adds no hidden reader population. `--agents 4 --edit-agents 4` is the separate four-pure-reader/four-editor tail workload. The counts remain explicit in the report and workload hash. +An explicit corpus `edit_tool: "edit.write"` selects whole-file write intent for +the same narrow Go, TypeScript or Python intrafile-call fixture. It additionally +requires `edit_initial_content` containing the exact pinned initial file bytes. +Before opening probe sessions, the client compares those bytes with its clone. +The complete offered requests are precomputed: write number N contains the +initial file plus additions 0 through N. The client does not reread or regenerate +content after another edit, serialize offered arrivals, or change retry intent. +Out-of-order overlapping writes can overwrite later changes; ordinary stale, +readback, refusal and evidence failures remain in the full denominator. The same +seven oracles, per-edit sessions, own-ACK readbacks and receipt accounting apply. +Write, symbol and batch receipts are not interchangeable. `edit.symbol` uses +an exact splice within an actual indexed declaration. The one-file `edit.batch` +adapter offers exact whole-file before/after strings and checks the durable +transaction, shared acknowledgement and per-file graph ticket separately. +Implementation controls and a one-edit diagnostic do not establish the +100-edit, reference, traffic-matrix or no-regression gates. + +The optional `edit_kind` selects `append_leaf`, `rename`, `delete`, +`modify_body`, `modify_caller_body`, `change_signature`, `add_crossfile_call` +or the Go-only `relocate`; omission retains `add_intrafile_call`. These are explicit fixture cells, not arbitrary +source transformations. A leaf must be a numeric zero-argument function with +no incoming or outgoing calls in complete relation results. Rename/delete +fixtures precreate each unique `GortexEditProbe…Before` declaration in the +pinned `edit_initial_content`; offered splices target that exact declaration. +Whole-file write/batch requests retain the precomputed offered prefix, including +out-of-order or lost-write outcomes. No state is regenerated after dispatch. + +Rename retains the seven positive new-symbol probes and adds a distinct +old-source absence probe/readback. Delete requires the exact scoped producer +`symbol not found` refusal, complete nontruncated graph/search collections and +a surviving declaration. A missing body, transport failure or empty partial +page cannot stand in for absence. The same bound prober proves the original +symbol exists during untimed setup; only replies sent after the actual edit +wire start can earn post-edit absence. Expected refusals retain their own raw +identity, endpoint and report count; they are not ordinary successful RPCs. +These new operation cells require actual runtime evidence before acceptance; +other kinds and unsupported combinations fail explicitly. + +Body/signature fixtures retain an adjacent following declaration. Their relative +start-line gap proves the changed body even when concurrent earlier edits shift +both declarations. Signature changes preserve valid zero-argument callers; +TypeScript/Python coarse signature metadata alone is not parameter proof. +Crossfile-call fixtures require two root files, exact imports where applicable, +and an independently admitted callee in the other file. + +Go relocation uses `edit_tool: "refactor.move"` (also `move_symbol`) with +`edit_kind: "relocate"`, distinct root Go files in the same package, and pinned +`edit_destination` / `edit_destination_content`. The offered source declarations +already exist; no client-generated declaration is substituted for a real move. +The seven destination/call probes plus explicit old-source absence must all +succeed. Immediate own-ACK readbacks check the destination file, new symbol, +old-symbol refusal and surviving source file before timing queries begin. +Standalone move observations retain one shared RPC A and each file's actual +write and publication windows. A direct move reconcile and an independent +public-read repair keep separate route records and actual request causes; a +later direct no-op cannot supply an earlier reader's work. Exact coalesced +execution identities are evaluated once while retaining both route records; +conflicting windows refuse. Each measured window must partition its own actual +owner and endpoints. A reader window can begin before the move RPC, remains +non-additive with shared A, and must accept the original file bytes after that +file's writer return. Scheduled background is explicitly inapplicable, not a +measured zero; physical mutation and batch receipts cannot supply missing move +clocks. Baseline controls without observations can establish public +functional completion while graph receipt proof and phases remain unavailable. +Cleanup uses one public two-file batch only after completed outcomes, then +verifies both original byte strings. Unsupported language, package, path and +operation combinations refuse explicitly. These implementation controls do not +establish live relocation timing or the broader traffic/reference gates. + +`edit_expect: "refusal"` turns a TypeScript or Python relocation fixture +(`edit_kind: "relocate"`, `edit_tool: "refactor.move"`, distinct root source and +destination files of one language) into a live refusal cell. The actual move is +sent, and the cell passes only on the producer's exact scoped refusal +(`unsupported language: move_symbol only supports Go files (got "")`), +followed at most by the known session notes, with the same session's immediate +file readback unchanged, the original symbol's source present, and both files' +bytes unchanged on disk. Its latency is the `relocation_refusal` readback; the +record has no probes, receipt, phases or restore batch, and the post-window step +only rechecks the bytes. Go relocation is supported and is never offered as a +refusal. + +`edit_output: "toon"` opts this fixture into TOON for the offered edit ACK, +visibility probes and immediate readbacks. Omission or `"json"` retains the +existing JSON plan and request identity. The decoder evaluates the requested +text; a valid structured JSON sibling, JSON fallback, malformed/truncated TOON +or unsafe numeric value cannot supply format credit. Unknown modes refuse, and +`"gcx"` refuses as unavailable (below). +Timing-status queries, initial presence/callee admissions and cleanup are +separate JSON controls, preserving exact generation/nanosecond arithmetic. + +TOON traversal subgraphs expose `total` plus explicit truncation and emitted +rows; they omit the JSON `total_nodes` and untrimmed full `total_edges` fields. +The harness does not insert those missing producer fields. Complete-row checks +require the known producer operation, exact declared node total, explicit +nontruncation and actual emitted collections; missing, clipped or contradictory +rows remain failures. File summaries retain their full explicit counts, signature metadata and end lines. +Compact traversal rows omit signature/end-line metadata. The pointed TOON +`change_signature` fixture instead traverses its unchanged following caller and +checks the changed relative start-line gap plus actual call relation; full +summary and editing-context signature checks remain separate. Actual Go, +TypeScript and Python producer controls establish that wire shape. TOON source implementation and offline +controls do not establish live, reference, traffic or performance coverage. + +`edit_output: "compact"` requests compact text (`output.compact`) only from the +four probes whose operation has a compact producer: file summary, symbol +search, callers and call chain. Source, editing-context and text probes, +readbacks, the edit ACK and timing controls stay JSON. Compact text is one +` :` row per symbol (symbol search may add +` (in )`), and subgraph answers may follow their rows with a truncation +line, one edge-count footer and concurrency notes; any other line, a missing +final newline or a JSON answer is a decode failure, and an empty answer is zero +rows. Because the rows carry no node IDs, edge rows, signatures or freshness +fields, compact proves presence only: the marker (or, from the marker's call +chain, the callee) as a top-level function row at its owned fixture path, with +a nonzero edge footer for relations. Truncated rows never prove absence. Plans +refuse compact for every kind except `add_intrafile_call` and +`add_crossfile_call`. + +`edit_output: "cli"` sends every oracle request, with the same lowered facade +arguments, through one owned `gortex call --json --format json +--index --quiet` process using the run's config and environment, in its +own process group that is killed at the probe deadline or phase end. The edit, +readbacks and receipt status stay MCP JSON. CLI stdout is credited only as one +JSON object whose numbers stay exact after the CLI's float64 re-encoding; the +`{"tool","format":"text","text"}` prose wrapper is refused, and a nonzero exit +is a probe error with its stdout kept as evidence only. Probe cadence is bounded +by process start, daemon dial and exit rather than 25 ms, and a probe's start is +its process start. Rename, delete and relocation are refused in CLI mode, +because their expected absence must be the exact scoped not-found refusal, and +paired runs refuse CLI probes. + +Each report's `output_modes` accounts every mode: `json`, `toon`, `compact`, +`cli` and `gcx`, each `selected`, `not_selected` or `unavailable` with a +reason, with probe answers and decode failures counted by the actual mode of +each probe request, claims fresh, stale answers claimed fresh after the ACK and +pre-commit mismatches. GCX is unavailable: there is no GCX decoder here, and +`read_file` answers a GCX request with TOON, so a GCX answer is never decoded or +credited as TOON. A mode without answers, or one whose payload cannot carry a +freshness claim (compact, GCX), reports its stale claimed-fresh count +unavailable rather than 0. + +`--scenario edit-and-probe --profile agent --agents 0 --edit-agents 1 +--worktrees 0` selects an initial `add_intrafile_call` / `edit.file` / JSON +accounting cell. Go retains its existing AST validation. The explicit `.ts` +fixture cell accepts only top-level `[export] function Name(): number` +declarations returning a decimal integer or a plain zero-argument call; its +template must make the unique marker return the selected intrafile callee. +`testdata/edit-probe/typescript/main.ts` provides ten initial symbols for the +ordinary warmup gate. Other TypeScript syntax is not supported by this fixture +cell. Its token checks discard comments/strings as proof and require the actual +producer symbol ID and file/path. TypeScript implementation does not constitute +a completed runtime or reference gate. The `.py` fixture cell similarly permits +only top-level `def Name():` definitions with one four-space-indented return of +a decimal integer or plain zero-argument call. Its fixture is +`testdata/edit-probe/python/main.py`; nested bodies, strings and comment-only +calls cannot provide source proof. Both new language cells are implementation +coverage only until their actual runtime checks pass. The finite plan contains 100 precomputed Poisson +arrivals (mean 15 seconds); `--edit-count 1` is an explicit diagnostic only. +Each edit has its own session opened before dispatch, with independent 25 ms +probes and immediate same-editing-session file/source readbacks. Session setup +is outside the edit window. Future arrivals do not wait for earlier ACKs or +probe replies. Canceled/unstarted edits remain in the offered denominator. + +`--edit-agents N` (1 to 32) runs N fixed editors against the same single +repository. Each editor offers its own open-loop Poisson stream with the same +15 second mean; editor 0 keeps the one-editor seeding, so `--edit-agents 1` +offers exactly the original plan. The streams merge by due time (ties by editor +index) and the merge order is the global ordinal, so markers and mutation IDs +stay keyed by ordinal and each ordinal is owned by one editor for the whole run. +Plan steps and edit records name that editor. An edit and its immediate +file/source readbacks go through its own editor's session; probes and receipt +status stay on the edit's separate prober session, and run-level status, stats +and the restore batch use editor 0. A blocked editor ACK does not delay another +editor's arrival. Editors share one file. `edit.write` carries whole-file +content precomputed in ordinal order, so an out-of-order write would silently +drop another editor's marker until the next write restored it; it is refused +with more than one editor. An out-of-order `edit.batch` fails its `old_string` +precondition and stays a failed record, and `edit.file` anchor insertions do +not depend on order. The 1-edit diagnostic and +the paired no-regression cell stay one editor. Routing across several +repositories or linked worktrees is not implemented. + +`edit_sender` selects a native or lifecycle traffic cell; without it every plan +is unchanged, and paired runs refuse it. A native write goes around MCP, so its +record has no ACK, receipt, phases or derived time (each unavailable with its +reason) and accounting `not_applicable_native_write`. It keeps its own write +window and the file's size, mtime and inode before and after. Its immediate file +readback gates `t_readback_ms`, graph-backed readbacks are kept as +`observation_only`, and it completes once every probe is first-correct inside +the deadline. The daemon learns of a native write only through its watcher, so +a fresh claim after one is counted unassessed rather than stale. Native writes +touch only the registered fixture file, or the step's own created file. + +- `shell_after_tool` (100 edits, one editor, `edit.file` call addition): even + ordinals are MCP edits; odd ordinals insert their own marker with an in-place + truncate-and-write at their due time or the preceding tool edit's own ACK, + whichever is later. A tool edit without its own ACK makes the shell write an + explicit dependency failure, and nothing is written. +- `editor_save` (one editor, rename fixture through `edit.file`): the fixture + predeclares placeholders `GortexEditProbeP`, exactly the + marker's length. Each save writes same-size bytes to a temporary file beside + the fixture and renames it over the file; the placeholder must become absent. +- `create_delete` (100 edits, root-level Go fixture with pinned + `edit_initial_content`, JSON or TOON): even ordinals create + `gortex_edit_probe_.go` in the fixture's package, calling the + fixture's callee; odd ordinals delete that file once every probe of its create + was first-correct. A delete's probes start at its own activation, so an + absence answer sent before its create completed is not applicable to it. + Deletion is proved by the exact scoped not-found refusal and complete + search, text and caller collections; a missing file's summary, editing + context and call chain have no pinned contract and are not probed. + Created files still present are removed by one timed restore batch. + TypeScript and Python create/delete are not implemented: a gap, not an + inapplicable cell. +- `hot_file` (100 edits, `--edit-agents 2`, `edit.file` call addition): MCP + traffic in which both editors send each pair of edits to the one file at the + same due time, taken from the single-editor stream. A lost write leaves its + marker invisible and is a measured failure. +- `restart` (`--edit-count 1`, one editor, no readers, JSON `edit.file` call + addition): the edit ACK waits for its graph receipt, so the cell does not + wait for it. While the ACK is outstanding, the edit's status is polled by its + `mutation_id` every 5 ms; once the bytes are committed and the receipt is not + yet terminal, the harness stops waiting for the ACK and kills the owned + daemon. `t_ack_ms` is then unavailable and the abandoned call is counted + censored, not as an error. An ACK that returns first leaves one status read + just before the kill to decide: a receipt already terminal there qualifies + the record `receipt terminal before kill` and it fails. The daemon is + restarted on the same store and awaited until ready, and the editor and + prober sessions are reopened. Probes pause during the downtime. First-correct + times come only from the restarted daemon, from send and from ready, and + answers before the kill are kept apart. The status after the restart is + recorded verbatim, the committed marker must be on disk, and the interrupted + receipt reports no phases, accounting or derived time. The run-level receipt + after the workload is taken on the reopened editor. Process counters cover + the pre-kill daemon only. + +Each edit-and-probe report names its `role`. A standalone run is the candidate +unless `--edit-probe-role baseline` is given; the paired command assigns roles +by build order and refuses the flag. A candidate still fails any record whose +phase accounting is missing. Every record carries `phases_ms` for each ACK and +patch lap the daemon names, taken from the record's own latest receipt status +(the per-file observation status for batches). A build without `phases_ms` +reports each lap unavailable with that reason, a producer null keeps the +producer's phase state as its reason, and a relocation keeps its per-file move +accounting instead. Missing laps are never zero. + +Records also carry client times from edit send: `t_readback_ms` (last correct +immediate same-session readback), `t_file_graph_ms` (source, summary, editing +context and old-source absence), `t_relations_ms` (callers and call chain) and +`t_search_ms` (symbol and text search). A group time is its slowest member's +first-correct time; any member that never answered correctly leaves the group +unavailable. `t_derived_ms` is daemon-clock time from receipt origin to +`patch_derived_finished` in the record's own terminal publication timeline, +because no probe observes the derived step; it is not additive with client +times and stays unavailable for coalesced receipts and relocations. + +The report `summary` covers the complete offered denominator: outcome counts +and, for every per-record time, each readback, each `phases_ms` lap and each +oracle's first-correct time, nearest-rank p50/p95/p99/max with percentile +bootstrap 95% intervals (2000 resamples, seeded from the plan seed and the +measure name). A value missing from a record that did not complete (deadline, +run end or failure) is `censored`: the record ranks above every observed value, +so a quantile or interval bound whose nearest rank falls among censored records +is unavailable rather than understated. A value missing from a completed record +(a native write's ACK, a coalesced receipt's derived time), or a measure the +record does not carry, is `not_applicable`. An interval needs at least 2, 20 or +100 ranked records for p50, p95 or p99 and is unavailable below that. Nothing is +sampled as zero. + +`--agents R` (0 to 32) adds pure readers on their own sessions, running the +ordinary reading plan with agent pacing. With readers, the phase opens with a +30 second no-edit window before the edit origin; the offered edit schedule, +relative to that origin, is unchanged, and the run budget includes the window. +Each read is classed when it is issued: `no_edit_baseline` before the origin, +`under_edit` while any sent edit's receipt is pending, otherwise +`between_edits`. A sent edit is pending until the harness first observes its +receipt terminal; an edit without one to observe (a failed edit, or a native +write) stays pending until its loop ends. An edit sent while a read is in +flight does not reclass it, so a read can straddle an edit's start by up to its +own latency. The report's `reads` table gives, per operation and +class, calls, busy refusals, errors, censored calls and the latency quantiles +of successful calls, in which calls cut off by their deadline or the run end +rank as censored; busy and failed calls are not applicable. Paired runs refuse +readers. + +The report retains raw call identities, independent ACK/physical/requested +background accounting, and missing evidence. RPC return timestamps delimit +latency; raw JSONL persistence runs on owned per-edit writers with separately +reported encoding/write overhead. The initial tiny-fixture raw budget is +8 MiB per edit, shared by accepted pending and in-flight bytes. One wake signal +coalesces small-row bursts without a separate message-count refusal. Exhaustion or persistence failure is +`harness_evidence_incomplete`, never product failure or a successful check. +Reference inputs require a reviewed evidence budget before execution. One timed +public batch restores the file after the measurement window; unknown/pending +mutations prevent that cleanup rewrite. The native paired path uses the same +cell machinery; other cells and reference acceptance remain separate gates. +An owner-matching immutable physical return with explicitly missing or invalid +deep laps ends that edit as failed coverage once its graph and read evidence +have arrived. Receipt terminality alone still waits for late physical evidence. +A complete native duplicate P keeps collecting until independent publisher R +and requested-background B arrive; absent late evidence is not terminal proof +of missing work. A validated immutable R phase failure can end failed coverage. +An original-publication-bound `requested_background_unavailable` is also +terminal failed coverage, not a measured zero or a successful accounting check. +Unknown or foreign diagnostics do not stop late-evidence collection. + +Requested B supports ordered native timer handoffs only when each previous +timer's actual `Stop` succeeds before its callback starts. Watcher, path and +actual generation bind each handoff; generation gaps never infer event counts. +Repeated native arm, quiet, stop and coalescing-control intervals accumulate +into fixed named scalars, without interval history. A callback that already ran, +an observed successor or retry/backoff remains unsupported, with its first +actual immutable invalidation reason retained. Later events cannot erase an +already accepted historical B window. Such a diagnostic may end collection +before other late evidence arrives; it does not prove that evidence stays +unavailable forever. + +`mutation_status` can separately expose `read_assisted_publication` when an +already-needed read repair executes for the exact pending committed bytes. +Its monotonic window covers the actual coordinator execution, with that +drain's owner generation and bounded reader admission causes. A reused wire ID +is not a unique publisher identity. This window can overlap the original +receipt's quiet wait; it is not added to A, B or a later no-op watcher P. +Its own `phase_accounting` measures the actual gate, parse, apply, search, +resolve, mtimes, derived and optional analysis operations, with an online +`repair_control` interval around them. The harness validates that independent +integer partition before accepting it as publisher deep coverage; the no-op +P's unexecuted deep values remain null, never retroactively filled from R. +R's internal `requested_receipt` matches the original `publication_phases.key` +and requested ticket, not the public commit `receipt`, caller `mutation_id` or +coalesced physical winner. The relationship to additive requested-background +accounting remains unavailable. Missing, failed, broad, evicted, rebound or content-mismatched +observations do not supply that evidence. Fresh same-byte inert validation +does not claim the publication performed by an earlier operation. One observed +independent partition does not establish the 100-edit accounting, paired +no-regression, reference or larger language/tool/native acceptance gates. +R is retained only by the existing bounded mutation ledger. Evicting the +original publication recorder's index entry does not erase an accepted ledger +observation or make canonical P/B available; ledger eviction clears R, and a +freshness rebind refuses the old relation. Earlier returned snapshots remain +immutable. + Edit scenarios require at least one editor; zero total or negative counts are rejected. Zero pure readers are allowed only for `edit`/`many-edit`; `all` needs both populations, and other scenarios need at least one reader. Inactive editor @@ -138,6 +480,44 @@ index; queryability and enrichment are separate observed upper bounds with a recorded polling interval. Older daemons without `loaded_roots` retain their legacy receipt and must still pass source-backed per-root checks. +## Startup admission + +`--scenario startup` (never part of `all`) measures what agents see while a +daemon starts. Every manifest repository is configured; `--worktrees` linked +checkouts of the first one are discovered through it. Probing starts before +the first daemon start, so the window before the socket exists is observed +too. `--seed-store ` copies a store (for example one written by an +older schema) into the private run first, which makes the initial boot an +upgrade; the source is only read. `--restarts N` adds N warm restarts. Each +boot lasts until every configured repository is ready and enriched, plus +`--duration`. + +Every second, from the primary, the first linked worktree, and a directory +whose `.gortex.yaml` names a workspace that a private `servers.toml` roster +claims, the probe opens fresh sessions: `initialize` (the instructions +variant is recorded), `search_symbols` and `resources/read +gortex://index-health` on the legacy surface, and `workspace(operation: +"index")` on the compact one. The roster's only server never listens, so a +routed call falls back to the daemon itself. From the same directories it +asks the PreToolUse hook about a native Read every second, and runs `gortex +call search_symbols` through a pipe (from the start) and through a pty +(`script(1)`, once the socket answers), each until its first success in that +boot. Every answer keeps `error_code`, `retryable`, phase, percent, latency, +exit code, and whether any refusal text advises `gortex track` or +`gortex repos reconcile`. A watcher polls the readiness probe every 100 ms +and, for 2 s after the primary registers, the checkout coordinator count. + +`startup-probe.jsonl` holds every answer. The phase gate `startup_probe` +counts answers per surface and directory in three windows (no socket, +loading, ready) and records each boot's time to ready and time to first +success. Headline metrics include `startup_repo_not_tracked_before_ready`, +`startup_nonconforming_loading_refusals`, the track-advice and hook-deny +counts, `startup_restart_ready_to_first_ok_p95_ms`, +`startup_restart_time_to_ready_p95_ms`, `startup_initial_time_to_ready_ms`, +`startup_initial_first_ok_ms` and `startup_coordinators_within_2s_min`. +Restart figures are unavailable unless every restart produced them, and +`delta` compares two startup reports like any other. + ## Measurements and artifacts One JSON file contains per-operation successful p50/p95/p99/max, successful @@ -273,6 +653,38 @@ reports and failures, repeat on the same host before p95 repeatability claims, and balance standalone build order per scenario. Coupled ratios do not replace independent scaling measurements. +The initial Go `add_intrafile_call` / `edit.file` / JSON accounting cell can use +the same dispatcher with exactly two builds, baseline first and candidate +second: + +```sh +go run ./bench/load paired --build main=/private/bin/main \ + --build candidate=/private/bin/candidate --manifest /private/corpus.json \ + --scratch /private/paired-artifacts --host native --scenario edit-and-probe \ + --agents 0 --edit-agents 1 --worktrees 0 --edit-count 100 --seed 42 \ + --out /private/paired.json +``` + +Both daemons run concurrently with independent clones, stores and sockets. They +share the exact precomputed offered Poisson schedule. Edits and all seven oracle +requests use the existing rotating back-to-back dispatch and matched per-request +median ratios. Each build retains one separately opened prober session per edit; +setup precedes the measurement window. A blocked ACK or probe does not serialize +later offered edits. Actual lateness, errors and canceled/unstarted entries remain +in the full offered denominator. + +Each build starts its immediate file/source readbacks after its own ACK, without +waiting for its partner. Those readbacks, receipt polling and stats are build-local +and **unpaired**; their latencies are excluded from matched ratios. First-correct +times are coupled paired-probe observations: waiting for the partner response can +delay the next probe, so these are not independent 25 ms freshness-budget results. +Missing baseline phase instrumentation stays unavailable, while candidate A/P/B +and applicable independent R accounting remain required. Evidence loss cancels +that edit's paired probes and is a failed/censored observation, never a product +deadline or a successful pair. This initial cell does not establish reference +capacity, the larger tool/language/native matrix, or a no-regression verdict +without the actual qualified paired run. + `delta` rejects incompatible workload/corpus/target/host-limit identities rather than printing an unqualified verdict. It lists every comparable timing metric more than 15% slower, without introducing budgets or CI pass/fail lanes. diff --git a/bench/load/call.go b/bench/load/call.go index 23e112a88..a9be86946 100644 --- a/bench/load/call.go +++ b/bench/load/call.go @@ -54,8 +54,10 @@ func checkToolResult(data json.RawMessage) error { return errors.New("tool returned isError without error content") } var body struct { - Code string `json:"error_code"` - Message string `json:"message"` + Code string `json:"error_code"` + Message string `json:"message"` + Retriable bool `json:"retriable"` + Data map[string]any `json:"data"` } if err := json.Unmarshal(payload, &body); err != nil { return err @@ -64,7 +66,7 @@ func checkToolResult(data json.RawMessage) error { if body.Code == "invalid_argument" { code = -32602 } - return &rpcError{Code: code, Message: body.Code + ": " + body.Message} + return &rpcError{Code: code, Message: body.Code + ": " + body.Message, Data: body.Data, Retriable: body.Code == "repo_loading" && body.Retriable} } type toolCaller interface { @@ -74,6 +76,14 @@ type toolCaller interface { // Each logical call has its own product budget. The phase context may end // sooner; that observation is right-censored, never a product deadline. func callTool(ctx context.Context, client toolCaller, r request, args map[string]any, role string, agent, step, slot int, timeout time.Duration, sleep func(context.Context, time.Duration) error) (json.RawMessage, []Attempt, error) { + return callToolPolicy(ctx, client, r, args, role, agent, step, slot, timeout, sleep, false) +} + +func callEditProbeTool(ctx context.Context, client toolCaller, r request, args map[string]any, role string, agent, step, slot int, timeout time.Duration, sleep func(context.Context, time.Duration) error) (json.RawMessage, []Attempt, error) { + return callToolPolicy(ctx, client, r, args, role, agent, step, slot, timeout, sleep, true) +} + +func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[string]any, role string, agent, step, slot int, timeout time.Duration, sleep func(context.Context, time.Duration) error, retryTyped bool) (json.RawMessage, []Attempt, error) { logicalStart := time.Now() productEnd := logicalStart.Add(timeout) phaseEnd, phaseBounded := ctx.Deadline() @@ -94,28 +104,47 @@ func callTool(ctx context.Context, client toolCaller, r request, args map[string for retry := 0; ; retry++ { start := time.Now() data, err := client.call(callCtx, "tools/call", map[string]any{"name": r.Tool, "arguments": args}) + returned := time.Now() + if retryTyped { + if clock, ok := client.(interface{ responseReturnedAt() time.Time }); ok { + if observed := clock.responseReturnedAt(); !observed.IsZero() && !observed.Before(start) { + returned = observed + } + } + } if err == nil && callCtx.Err() != nil { err = callCtx.Err() } if err == nil { err = checkToolResult(data) } - a := Attempt{Tool: r.name(), Role: role, Request: hash, Agent: agent, Step: step, Slot: slot, Retry: retry, Started: start, DurationMS: float64(time.Since(start)) / float64(time.Millisecond), LogicalMS: float64(time.Since(logicalStart)) / float64(time.Millisecond), Outcome: "success"} + a := Attempt{Tool: r.name(), Role: role, Request: hash, Agent: agent, Step: step, Slot: slot, Retry: retry, Started: start, DurationMS: float64(returned.Sub(start)) / float64(time.Millisecond), LogicalMS: float64(returned.Sub(logicalStart)) / float64(time.Millisecond), Outcome: "success"} var remote *rpcError if err != nil { a.Outcome, a.Error = "error", err.Error() if errors.As(err, &remote) { a.Code = remote.Code } - if remote != nil && remote.Code == -32002 { + if remote != nil && remote.Code == -32002 && (!retryTyped || remote.retryEditProbe()) { a.Outcome = "busy" + } else if retryTyped && remote != nil && remote.retryEditProbe() { + a.Outcome = "retryable" + } + if _, expected := err.(*editProbeExpectedAbsence); expected { + a.Outcome = "expected_refusal" + a.Code = -32000 + a.ExpectedOutcome = "symbol_absent" } if callCtx.Err() != nil { a.Outcome = terminalOutcome() + err = errors.Join(err, callCtx.Err()) + } + if retryTyped && errors.Is(err, errEditProbeEvidenceIncomplete) { + a.Outcome = "harness_incomplete" } } attempts = append(attempts, a) - if a.Outcome != "busy" { + if a.Outcome != "busy" && a.Outcome != "retryable" { return data, attempts, err } if err := sleep(callCtx, busyBackoff(retry)); err != nil { diff --git a/bench/load/client.go b/bench/load/client.go index a5e972065..66f5371a5 100644 --- a/bench/load/client.go +++ b/bench/load/client.go @@ -17,12 +17,27 @@ import ( const maxWireBytes = 64 << 20 type rpcError struct { - Code int `json:"code"` - Message string `json:"message"` + Code int `json:"code"` + Message string `json:"message"` + Data map[string]any `json:"data,omitempty"` + Retriable bool `json:"-"` // typed tool-result refusal, not a JSON-RPC top-level field } func (e *rpcError) Error() string { return fmt.Sprintf("RPC %d: %s", e.Code, e.Message) } +func (e *rpcError) retryEditProbe() bool { + if unknown, _ := e.Data["delivery_unknown"].(bool); unknown { + return false + } + // These are actual front-door shapes: dispatcher -32002 in + // mcp_request_lifetime.go, and repo_loading data in + // cmd/gortex/daemon_mcp_repo_loading.go. Unknowns stay terminal. + if e.Code == -32002 { + return true + } + return e.Retriable || e.Code == -32000 && e.Data["error_code"] == "repo_loading" && e.Data["retryable"] == true +} + type wireReply struct { ID json.Number `json:"id"` Result json.RawMessage `json:"result"` @@ -51,6 +66,14 @@ type socketClient struct { version string } +// Per-call values returned to its caller, never a shared last-ID cache. Times +// retain monotonic readings until the scenario renders relative nanoseconds. +type wireCallIdentity struct { + ID int64 + onWriteStart func(time.Time) + Started, WriteStarted, WriteFinished, ReplyDelivered, Finished time.Time +} + func readWireLine(reader *bufio.Reader) ([]byte, error) { var data []byte for { @@ -66,6 +89,12 @@ func readWireLine(reader *bufio.Reader) ([]byte, error) { } func dialSocket(ctx context.Context, socket, cwd, mode string) (net.Conn, *bufio.Reader, string, error) { + return dialSocketSurface(ctx, socket, cwd, mode, "facade-v1", "hide") +} + +// dialSocketSurface negotiates the session's tool surface in the handshake, +// as the CLI does per connection; the load workload always uses the facade. +func dialSocketSurface(ctx context.Context, socket, cwd, mode, tools, toolsMode string) (net.Conn, *bufio.Reader, string, error) { conn, err := (&net.Dialer{}).DialContext(ctx, "unix", socket) if err != nil { return nil, nil, "", err @@ -80,7 +109,7 @@ func dialSocket(ctx context.Context, socket, cwd, mode string) (net.Conn, *bufio } stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) defer stop() - handshake := map[string]any{"version": 1, "mode": mode, "cwd": cwd, "client": "gortex-load", "tools": "facade-v1", "tools_mode": "hide"} + handshake := map[string]any{"version": 1, "mode": mode, "cwd": cwd, "client": "gortex-load", "tools": tools, "tools_mode": toolsMode} if err := json.NewEncoder(conn).Encode(handshake); err != nil { _ = conn.Close() return nil, nil, "", err @@ -190,6 +219,10 @@ func (c *socketClient) Close() error { } func (c *socketClient) send(ctx context.Context, value any, advisory bool) error { + return c.sendObserved(ctx, value, advisory, nil) +} + +func (c *socketClient) sendObserved(ctx context.Context, value any, advisory bool, observation *wireCallIdentity) error { data, err := json.Marshal(value) if err != nil { return err @@ -227,7 +260,16 @@ func (c *socketClient) send(ctx context.Context, value any, advisory bool) error _ = c.conn.Close() close(interrupted) }) + if observation != nil { + observation.WriteStarted = time.Now() + if observation.onWriteStart != nil { + observation.onWriteStart(observation.WriteStarted) + } + } _, err = io.Copy(c.conn, bytes.NewReader(append(data, '\n'))) + if observation != nil { + observation.WriteFinished = time.Now() + } if !stop() { <-interrupted } @@ -242,6 +284,26 @@ func (c *socketClient) send(ctx context.Context, value any, advisory bool) error } func (c *socketClient) call(ctx context.Context, method string, params any) (json.RawMessage, error) { + return c.callObserved(ctx, method, params, nil) +} + +func (c *socketClient) callWithIdentity(ctx context.Context, method string, params any) (json.RawMessage, wireCallIdentity, error) { + return c.callWithWriteObserver(ctx, method, params, nil) +} + +func (c *socketClient) callWithWriteObserver(ctx context.Context, method string, params any, onWriteStart func(time.Time)) (json.RawMessage, wireCallIdentity, error) { + var observation wireCallIdentity + observation.onWriteStart = onWriteStart + data, err := c.callObserved(ctx, method, params, &observation) + observation.onWriteStart = nil + return data, observation, err +} + +func (c *socketClient) callObserved(ctx context.Context, method string, params any, observation *wireCallIdentity) (json.RawMessage, error) { + if observation != nil { + observation.Started = time.Now() + defer func() { observation.Finished = time.Now() }() + } if err := ctx.Err(); err != nil { return nil, err } @@ -253,6 +315,9 @@ func (c *socketClient) call(ctx context.Context, method string, params any) (jso } c.next++ id := c.next + if observation != nil { + observation.ID = id + } ch := make(chan wireReply, 1) c.pending[id] = ch c.mu.Unlock() @@ -261,7 +326,7 @@ func (c *socketClient) call(ctx context.Context, method string, params any) (jso delete(c.pending, id) c.mu.Unlock() }() - if err := c.send(ctx, map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}, false); err != nil { + if err := c.sendObserved(ctx, map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}, false, observation); err != nil { return nil, err } select { @@ -273,6 +338,9 @@ func (c *socketClient) call(ctx context.Context, method string, params any) (jso cancel() return nil, ctx.Err() case reply := <-ch: + if observation != nil { + observation.ReplyDelivered = time.Now() + } if ctx.Err() != nil { return reply.Result, ctx.Err() } diff --git a/bench/load/client_test.go b/bench/load/client_test.go index 03e4dc5a4..2f3c9c701 100644 --- a/bench/load/client_test.go +++ b/bench/load/client_test.go @@ -427,3 +427,63 @@ func TestInvalidationSubscriptionRecordsAvailabilityAndActualArguments(t *testin }) } } + +func TestRecordedCallsKeepExactIdentityAcrossMultiplexedReplies(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + return map[string]any{"wire_id": frame.ID, "ordinal": frame.Params["ordinal"]}, nil + }) + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + type result struct { + ordinal int + data json.RawMessage + identity wireCallIdentity + err error + } + results := make(chan result, 32) + var calls sync.WaitGroup + for ordinal := range 32 { + calls.Go(func() { + data, id, err := client.callWithIdentity(ctx, "identity", map[string]any{"ordinal": ordinal}) + results <- result{ordinal, data, id, err} + }) + } + calls.Wait() + close(results) + seen := make(map[int64]bool) + for result := range results { + if result.err != nil { + t.Fatal(result.err) + } + var decoded struct { + WireID int64 `json:"wire_id"` + Ordinal int `json:"ordinal"` + } + if err := json.Unmarshal(result.data, &decoded); err != nil { + t.Fatal(err) + } + id := result.identity + if id.ID != decoded.WireID || decoded.Ordinal != result.ordinal || seen[id.ID] { + t.Fatalf("misattributed multiplexed reply%+v %+v", result, decoded) + } + seen[id.ID] = true + if id.Started.IsZero() || id.WriteStarted.Before(id.Started) || id.WriteFinished.Before(id.WriteStarted) || id.ReplyDelivered.Before(id.WriteFinished) || id.Finished.Before(id.ReplyDelivered) { + t.Fatalf("incomplete/reversed call readings%+v", id) + } + } +} + +func TestRecordedCanceledCallDoesNotFabricateWireOrReply(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + client := &socketClient{} + data, id, err := client.callWithIdentity(ctx, "not-sent", nil) + if !errors.Is(err, context.Canceled) || data != nil || id.ID != 0 || !id.WriteStarted.IsZero() || !id.ReplyDelivered.IsZero() || id.Started.IsZero() || id.Finished.Before(id.Started) { + t.Fatalf("fabricated canceled call data=%s identity=%+v err=%v", data, id, err) + } +} diff --git a/bench/load/edit_output_decode.go b/bench/load/edit_output_decode.go new file mode 100644 index 000000000..ffc12113d --- /dev/null +++ b/bench/load/edit_output_decode.go @@ -0,0 +1,283 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "math" + "regexp" + "strconv" + "strings" + + toon "github.com/toon-format/toon-go" +) + +// The strict edit output decoder. It depends only on the standard library and +// the pinned TOON decoder, never on edit-probe plans or runtime state, so other +// edit-loop measurements can decode the same MCP envelopes the same way: one +// successful text payload plus only the known ordered auxiliary session notes, +// requested JSON never credited from TOON (or TOON from JSON), exact numbers +// only, and a harness-private marker recording which wire was decoded. + +// This typed, harness-local proof cannot be supplied by a JSON response. It +// records the selected wire decoder, never fabricates producer count fields. +type editProbeTOONWire struct{ operation string } + +const editProbeTOONWireKey = "\x00load.toon.wire" + +// There is no GCX decoder here, and read_file answers a GCX request with TOON. +// A GCX answer is reported unavailable, never decoded or credited as TOON. +var errEditOutputGCXUnavailable = errors.New("GCX output unavailable: no GCX decoder, and read_file answers a GCX request with TOON") + +func decodeEditProbeOutput(raw json.RawMessage, format, operation string) (map[string]any, error) { + switch format { + case "", "json": + case "toon": + case "compact": + return decodeEditProbeCompactOutput(raw, operation) + case "cli": + return decodeEditProbeCLIOutput(raw) + case "gcx": + return nil, errEditOutputGCXUnavailable + default: + return nil, errors.New("unsupported edit output mode") + } + text, err := editOutputPayload(raw, false) + if err != nil { + return nil, err + } + if format == "" || format == "json" { + decoder := json.NewDecoder(bytes.NewReader(text)) + decoder.UseNumber() + var body map[string]any + if !json.Valid(text) { + return nil, errors.New("requested JSON returned non-JSON payload") + } + if err := decoder.Decode(&body); err != nil { + return nil, err + } + return body, nil + } + if json.Valid(text) { + return nil, errors.New("requested TOON returned JSON; no format credit") + } + decoded, err := toon.Decode(text, toon.WithStrictMode(true)) + if err != nil { + return nil, err + } + object, ok := decoded.(map[string]any) + if !ok || len(object) == 0 { + return nil, errors.New("TOON probe payload must be a nonempty object") + } + if !editProbeTOONNumbersSafe(object) { + return nil, errors.New("TOON numeric value cannot be proved exact") + } + // The pinned decoder uses float64. Only exact safe integer values can + // provide IDs/counts; timing-status queries stay JSON/UseNumber separately. + normalized, err := json.Marshal(object) + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(normalized)) + decoder.UseNumber() + var body map[string]any + if err := decoder.Decode(&body); err != nil { + return nil, err + } + body[editProbeTOONWireKey] = editProbeTOONWire{operation: operation} + return body, nil +} + +// One successful text payload, followed only by the known ordered auxiliary +// session notes. Only compact text may be empty: it is a zero-row answer. +func editOutputPayload(raw json.RawMessage, allowEmpty bool) ([]byte, error) { + var envelope struct { + IsError bool `json:"isError"` + Content []struct { + Type string `json:"type"` + Text *string `json:"text"` + } `json:"content"` + } + if err := json.Unmarshal(raw, &envelope); err != nil { + return nil, err + } + if envelope.IsError || len(envelope.Content) < 1 || len(envelope.Content) > 3 || envelope.Content[0].Type != "text" || envelope.Content[0].Text == nil || *envelope.Content[0].Text == "" && !allowEmpty { + return nil, errors.New("output requires one successful payload and only known auxiliary notes") + } + lastNoteKind := 0 + for _, item := range envelope.Content[1:] { + kind := 0 + if item.Text != nil { + kind = editProbeSessionNoteKind(*item.Text) + } + if item.Type != "text" || kind <= lastNoteKind { + return nil, errors.New("unknown or duplicate auxiliary output content") + } + lastNoteKind = kind + } + return []byte(*envelope.Content[0].Text), nil +} + +// Compact text is one line per symbol: kind, qualified name, path:line, and +// for symbol search an optional enclosing owner. Subgraph answers may follow +// their rows with a truncation line, one edge-count footer and concurrency +// notes. There are no node IDs, edge rows, signatures or freshness fields. +type editProbeCompactRow struct { + Kind, Qualified, Path, Owner string + Line uint64 +} + +// This typed, harness-local answer cannot be supplied by a JSON response. +type editProbeCompactAnswer struct { + operation string + rows []editProbeCompactRow + truncated bool + edges int // -1 without an edge footer +} + +const editProbeCompactKey = "\x00load.compact.rows" + +// Only these operations have a compact text producer; every other request +// stays JSON. +var editProbeCompactOperations = map[string]bool{"read.summary": true, "search.symbols": true, "relations.callers": true, "trace.call_chain": true} + +var ( + editProbeCompactRowLine = regexp.MustCompile(`^([a-z_]+) (\S+) (\S+):([1-9][0-9]*)( \(in (\S+)\))?$`) + editProbeCompactTruncated = regexp.MustCompile(`^\.\.\. truncated \([0-9]+ total\)$`) + editProbeCompactEdges = regexp.MustCompile(`^edges: ([0-9]+)( of [0-9]+)? total(, [0-9]+ (EXTRACTED|INFERRED|AMBIGUOUS))*$`) + editProbeCompactConcurrency = regexp.MustCompile(`^concurrency \S+:( sync_guarded)?( cross_concurrent)?$`) + editProbeCompactBudgetMarker = "... trimmed to byte budget; pass max_bytes:0 for the full result" +) + +func decodeEditProbeCompactOutput(raw json.RawMessage, operation string) (map[string]any, error) { + if !editProbeCompactOperations[operation] { + return nil, errors.New("operation has no compact text producer") + } + text, err := editOutputPayload(raw, true) + if err != nil { + return nil, err + } + if json.Valid(text) { + return nil, errors.New("requested compact text returned JSON; no format credit") + } + if len(text) > 0 && text[len(text)-1] != '\n' { + return nil, errors.New("compact text must end each line with a newline") + } + answer := editProbeCompactAnswer{operation: operation, edges: -1} + subgraph := operation != "search.symbols" + stage := 0 // rows, then truncation, then the edge footer, then concurrency notes + for _, line := range strings.Split(string(text), "\n")[:bytes.Count(text, []byte("\n"))] { + switch match := editProbeCompactRowLine.FindStringSubmatch(line); { + case match != nil && stage == 0 && (match[6] == "" || !subgraph): + number, err := strconv.ParseUint(match[4], 10, 64) + if err != nil { + return nil, err + } + answer.rows = append(answer.rows, editProbeCompactRow{Kind: match[1], Qualified: match[2], Path: match[3], Owner: match[6], Line: number}) + case subgraph && stage < 1 && (editProbeCompactTruncated.MatchString(line) || line == editProbeCompactBudgetMarker): + answer.truncated, stage = true, 1 + case subgraph && stage < 2 && editProbeCompactEdges.MatchString(line): + edges, err := strconv.Atoi(editProbeCompactEdges.FindStringSubmatch(line)[1]) + if err != nil { + return nil, err + } + answer.edges, stage = edges, 2 + case subgraph && editProbeCompactConcurrency.MatchString(line): + stage = 3 + default: + return nil, fmt.Errorf("compact line outside the %s grammar: %q", operation, line) + } + } + return map[string]any{editProbeCompactKey: answer}, nil +} + +// `gortex call --format json` prints the tool's first text payload as JSON, +// re-encoded through float64, and wraps a prose answer as +// {"tool","format":"text","text"}. Only a JSON object with exactly +// representable numbers is an answer; the wrapper is never JSON credit. +func decodeEditProbeCLIOutput(stdout []byte) (map[string]any, error) { + decoder := json.NewDecoder(bytes.NewReader(stdout)) + decoder.UseNumber() + var body map[string]any + if err := decoder.Decode(&body); err != nil { + return nil, fmt.Errorf("CLI stdout is not one JSON object: %w", err) + } + if decoder.More() { + return nil, errors.New("CLI stdout has more than one JSON value") + } + if _, wrapped := body["text"].(string); wrapped && len(body) == 3 && body["format"] == "text" && body["tool"] != nil { + return nil, errors.New("CLI wrapped a prose answer; no JSON credit") + } + if !editProbeCLINumbersExact(body) { + return nil, errors.New("CLI numeric value cannot be proved exact after float64 re-encoding") + } + return body, nil +} + +func editProbeCLINumbersExact(value any) bool { + switch value := value.(type) { + case json.Number: + number, err := value.Float64() + return err == nil && editProbeTOONNumbersSafe(number) + case map[string]any: + for _, nested := range value { + if !editProbeCLINumbersExact(nested) { + return false + } + } + case []any: + for _, nested := range value { + if !editProbeCLINumbersExact(nested) { + return false + } + } + } + return true +} + +// These are the optional text-only notes appended by mcp/momentum.go. Selecting +// the first payload never changes the retained raw envelope or its byte budget. +func editProbeSessionNoteKind(text string) int { + const nudge = ". Every location returned so far is real and citeable — if you are localizing or answering a question, consider writing your conclusion from what you already hold and fetching only what it still lacks. This note appears once.)" + const escalation = " granular reads in a row. You likely have enough — answer NOW with your current best candidate and its evidence: name the file and the symbol, citing the locations already returned. A confident partial answer beats an exhausted turn budget with no answer." + const alternative = " If one more look is genuinely needed, a single explore call with the request in plain words returns the ranked neighborhood — symbols, source, call paths — in one response." + for _, form := range []struct { + prefix, suffix string + minimum, kind int + }{ + {"(Session note: graph read #", nudge, 6, 1}, + {"(Session note: ", escalation + " This note will not repeat.)", 10, 2}, + {"(Session note: ", escalation + alternative + " This note will not repeat.)", 10, 2}, + } { + if !strings.HasPrefix(text, form.prefix) || !strings.HasSuffix(text, form.suffix) { + continue + } + digits := strings.TrimSuffix(strings.TrimPrefix(text, form.prefix), form.suffix) + n, err := strconv.Atoi(digits) + if err == nil && n >= form.minimum && strconv.Itoa(n) == digits { + return form.kind + } + } + return 0 +} + +func editProbeTOONNumbersSafe(value any) bool { + switch value := value.(type) { + case float64: + return !math.IsNaN(value) && !math.IsInf(value, 0) && math.Abs(value) < 1<<53 + case map[string]any: + for _, nested := range value { + if !editProbeTOONNumbersSafe(nested) { + return false + } + } + case []any: + for _, nested := range value { + if !editProbeTOONNumbersSafe(nested) { + return false + } + } + } + return true +} diff --git a/bench/load/edit_probe_absence.go b/bench/load/edit_probe_absence.go new file mode 100644 index 000000000..c1de43bad --- /dev/null +++ b/bench/load/edit_probe_absence.go @@ -0,0 +1,166 @@ +package main + +import ( + "encoding/json" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "strings" +) + +// Expected absence is a declared producer refusal with its own identity and +// endpoint. It is never an empty successful body, a transport error or a +// blanket permission to ignore missing symbols. +type editProbeExpectedAbsence struct{ symbol, message string } + +func (e *editProbeExpectedAbsence) Error() string { return e.message } + +func editProbeExactNotFound(raw json.RawMessage, scopedID string) (string, bool) { + if scopedID == "" || strings.ContainsAny(scopedID, "\r\n\x00") { + return "", false + } + var response struct { + IsError bool `json:"isError"` + Structured any `json:"structuredContent"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &response) != nil || !response.IsError || response.Structured != nil || len(response.Content) != 1 || response.Content[0].Type != "text" { + return "", false + } + text := response.Content[0].Text + first, _, _ := strings.Cut(text, "\n") + exact := "symbol not found: " + scopedID + if first != exact { + // handleGetSymbolSource appends this exact same-line delimiter. The + // complete old ID must precede it; a prefix match is not absence proof. + hint, suggested := strings.CutPrefix(first, exact+" — did you mean: ") + if !suggested || strings.TrimSpace(hint) == "" { + return "", false + } + } + return text, true +} + +func (c *editProbeCaller) expectedAbsence(data json.RawMessage, identity wireCallIdentity) error { + logical := c.request.ExpectedAbsentSymbol + if logical == "" || c.request.Tool != "read" || c.request.Operation != "source" || (c.role != "probe" && c.role != "readback") { + return nil + } + target, ok := c.request.Arguments["target"].(map[string]any) + if !ok || target["symbol"] != logical { + return nil + } + c.state.mu.Lock() + proven := c.state.absenceBefore[logical] + editStarted := c.state.editWrite + c.state.mu.Unlock() + // A reply from a request sent before the real edit wire start does not + // establish post-edit absence, even when its delivery happens afterward. + if !proven || editStarted.IsZero() || identity.WriteStarted.IsZero() || identity.WriteStarted.Before(editStarted) { + return nil + } + scoped := c.runtime.p.Repos[0].Prefix + "/" + logical + if message, matched := editProbeExactNotFound(data, scoped); matched { + return &editProbeExpectedAbsence{symbol: scoped, message: message} + } + return nil +} + +// Before-state proof is obtained on the same bound prober session during +// untimed setup. A not-found response cannot establish its own applicability. +func editProbePresenceProof(raw json.RawMessage, c Corpus, logical string) bool { + separator := strings.LastIndex(logical, "::") + if separator < 1 || logical[:separator] != c.File || !token.IsIdentifier(logical[separator+2:]) { + return false + } + if checkToolResult(raw) != nil { + return false + } + body, err := decodeEditProbeResult(raw) + if err != nil { + return false + } + name := logical[separator+2:] + id, _ := body["id"].(string) + path, _ := body["file_path"].(string) + source, _ := body["source"].(string) + if !editProbeSourceID(id, c.File, name) || !editProbeFixturePath(path, c.File) || body["name"] != name { + return false + } + switch filepath.Ext(c.File) { + case ".ts": + valid, correct := editProbeTypeScriptFixture(source, name, "") + _, calls := editProbeTypeScriptFixture(source, name, c.Symbol[strings.LastIndex(c.Symbol, "::")+2:]) + if c.EditKind == "modify_caller_body" { + _, alternate := editProbeTypeScriptFixture(source, name, "Alternate") + calls = calls || alternate + } + return valid && (correct || calls) + case ".py": + valid, correct := editProbePythonFixture(source, name, "") + _, calls := editProbePythonFixture(source, name, c.Symbol[strings.LastIndex(c.Symbol, "::")+2:]) + if c.EditKind == "modify_caller_body" { + _, alternate := editProbePythonFixture(source, name, "Alternate") + calls = calls || alternate + } + return valid && (correct || calls) + case ".go": + parsed, err := parser.ParseFile(token.NewFileSet(), "before.go", source, 0) + if err != nil { + parsed, err = parser.ParseFile(token.NewFileSet(), "before.go", "package fixture\n"+source, 0) + } + if err != nil { + return false + } + count := 0 + for _, decl := range parsed.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Recv == nil && fn.Name.Name == name && fn.Body != nil { + count++ + } + } + return count == 1 + } + return false +} + +// A new call edge alone cannot prove rename reconciliation: a stale old +// caller may remain beside it. Retain both positive new-edge and complete +// old-identity exclusion checks on the actual relation result. +func editProbeRenameCensus(body map[string]any, c Corpus, step editProbeStep) bool { + truncated, ok := body["truncated"].(bool) + if !ok || truncated { + return false + } + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + if !nok || !eok { + return false + } + old := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + for _, raw := range nodes { + node, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := node["id"].(string) + if editProbeSourceID(id, c.File, old) { + return false + } + } + for _, raw := range edges { + edge, ok := raw.(map[string]any) + if !ok { + return false + } + from, fok := edge["from"].(string) + to, tok := edge["to"].(string) + if !fok || !tok || editProbeSourceID(from, c.File, old) || editProbeSourceID(to, c.File, old) { + return false + } + } + return true +} diff --git a/bench/load/edit_probe_absence_test.go b/bench/load/edit_probe_absence_test.go new file mode 100644 index 000000000..ba1b95c80 --- /dev/null +++ b/bench/load/edit_probe_absence_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" +) + +func absentSourceWire(message string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": message}}}) + return raw +} + +func TestEditProbeExpectedAbsenceMatchesExactProducerRefusal(t *testing.T) { + const id = "owned/file.go::Old" + for _, test := range []struct { + name string + raw json.RawMessage + want bool + }{ + {"exact", absentSourceWire("symbol not found: " + id), true}, + {"suggestions", absentSourceWire("symbol not found: " + id + "\nDid you mean another symbol?"), true}, + {"producer_same_line_suggestions", absentSourceWire("symbol not found: " + id + " — did you mean: owned/new.go::Old, owned/file.go::Other"), true}, + {"same_line_empty_hint", absentSourceWire("symbol not found: " + id + " — did you mean: "), false}, + {"same_line_foreign_prefix", absentSourceWire("symbol not found: " + id + "Extra — did you mean: owned/new.go::Old"), false}, + {"same_line_unknown_suffix", absentSourceWire("symbol not found: " + id + "; permission denied"), false}, + {"foreign", absentSourceWire("symbol not found: other/file.go::Old"), false}, + {"prefix", absentSourceWire("symbol not found: " + id + "Extra"), false}, + {"freshness", absentSourceWire("repo is still loading"), false}, + {"denied", absentSourceWire("permission denied: " + id), false}, + {"empty", json.RawMessage(`{"isError":false,"content":[]}`), false}, + {"structured", json.RawMessage(`{"isError":true,"structuredContent":{"error":"not found"},"content":[{"type":"text","text":"symbol not found: owned/file.go::Old"}]}`), false}, + {"malformed", json.RawMessage(`{`), false}, + } { + t.Run(test.name, func(t *testing.T) { + _, got := editProbeExactNotFound(test.raw, id) + if got != test.want { + t.Fatalf("matched=%v want=%v", got, test.want) + } + }) + } +} + +func TestEditProbeExpectedAbsenceRequiresPriorPresenceAndPostEditWire(t *testing.T) { + const logical = "file.go::Old" + now := time.Now() + for _, name := range []string{"valid", "no_presence", "before_edit", "no_edit", "no_wire", "wrong_target", "wrong_role", "foreign_refusal"} { + t.Run(name, func(t *testing.T) { + state := &editProbeState{absenceBefore: map[string]bool{logical: true}, editWrite: now} + caller := &editProbeCaller{runtime: &editProbeRuntime{p: &privateRun{Repos: []privateRepo{{Prefix: "owned"}}}}, state: state, role: "probe", request: request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: logical, Arguments: map[string]any{"target": map[string]any{"symbol": logical}}}} + identity := wireCallIdentity{WriteStarted: now.Add(time.Millisecond), Finished: now.Add(2 * time.Millisecond)} + raw := absentSourceWire("symbol not found: owned/" + logical) + switch name { + case "no_presence": + state.absenceBefore = nil + case "before_edit": + identity.WriteStarted = now.Add(-time.Millisecond) + case "no_edit": + state.editWrite = time.Time{} + case "no_wire": + identity.WriteStarted = time.Time{} + case "wrong_target": + caller.request.Arguments["target"] = map[string]any{"symbol": "other.go::Old"} + case "wrong_role": + caller.role = "status" + case "foreign_refusal": + raw = absentSourceWire("symbol not found: foreign/" + logical) + } + err := caller.expectedAbsence(raw, identity) + var absent *editProbeExpectedAbsence + if got := errors.As(err, &absent); got != (name == "valid") { + t.Fatalf("expected refusal=%v err=%v", got, err) + } + }) + } +} + +func TestEditProbeExpectedRefusalIsNotAnOrdinarySuccess(t *testing.T) { + metrics := summarize([]Attempt{{Tool: "read.source", Role: "probe", Outcome: "expected_refusal", ExpectedOutcome: "symbol_absent", LogicalMS: 12}}, 1, "probe")["read.source"] + if metrics.ExpectedRefusals != 1 || metrics.Successes != 0 || len(metrics.Errors) != 0 { + t.Fatalf("absence became ordinary success/error: %+v", metrics) + } +} + +func TestEditProbeAbsenceRequiresActualOwnedDeclarationBeforeEdit(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeSymbolCorpus(t, language) + logical := c.Symbol + name := logical[strings.LastIndex(logical, "::")+2:] + source := c.EditAnchor + for _, variant := range []string{"actual", "comment_only", "wrong_id", "wrong_file", "wrong_name", "empty", "not_found", "is_error_json"} { + t.Run(variant, func(t *testing.T) { + body := map[string]any{"id": "owned/" + logical, "file_path": "owned/" + c.File, "name": name, "source": source} + switch variant { + case "comment_only": + body["source"] = "// " + source + if language == "python" { + body["source"] = "# " + strings.ReplaceAll(source, "\n", "\n# ") + } + case "wrong_id": + body["id"] = "owned/other.go::" + name + case "wrong_file": + body["file_path"] = "owned/other.go" + case "wrong_name": + body["name"] = "Other" + case "empty": + body["source"] = "" + } + payload, _ := json.Marshal(body) + raw, _ := json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": string(payload)}}}) + if variant == "not_found" { + raw = absentSourceWire("symbol not found: owned/" + logical) + } + if variant == "is_error_json" { + raw, _ = json.Marshal(map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": string(payload)}}}) + } + if got := editProbePresenceProof(raw, c, logical); got != (variant == "actual") { + t.Fatalf("presence proof=%v", got) + } + }) + } + }) + } +} + +func TestEditProbeExpectedAbsenceSurvivesPerformWithoutEvidenceFailure(t *testing.T) { + const logical = "file.go::Old" + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": "symbol not found: owned/" + logical}}}, nil + }) + client, err := openMCP(t.Context(), socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.absenceBefore = map[string]bool{logical: true} + state.editWrite = time.Now().Add(-time.Millisecond) + run := &editProbeRuntime{p: &privateRun{Repos: []privateRepo{{Prefix: "owned"}}}, phase: &Phase{}, origin: time.Now(), options: runOptions{Deadline: time.Second}} + req := request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: logical, Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": logical}}} + _, _, attempts, err := run.performObserved(t.Context(), state, client, req, "oracle:old_source_absent", "probe", 0) + if _, ok := err.(*editProbeExpectedAbsence); !ok { + t.Fatalf("clean expected refusal wrapped/lost: %T %v", err, err) + } + if len(attempts) != 1 || attempts[0].Outcome != "expected_refusal" { + t.Fatalf("attempts=%+v", attempts) + } +} + +type canceledExpectedAbsenceCaller struct{ cancel context.CancelFunc } + +func (c canceledExpectedAbsenceCaller) call(context.Context, string, any) (json.RawMessage, error) { + c.cancel() + return absentSourceWire("symbol not found: owned/file.go::Old"), &editProbeExpectedAbsence{symbol: "owned/file.go::Old", message: "symbol not found: owned/file.go::Old"} +} +func TestEditProbeCensoredExpectedAbsenceCannotBeCorrect(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + _, attempts, err := callEditProbeTool(ctx, canceledExpectedAbsenceCaller{cancel}, request{Tool: "read", Operation: "source"}, nil, "probe", 0, 0, 0, time.Second, waitContext) + if _, ok := err.(*editProbeExpectedAbsence); ok { + t.Fatal("censored result retained eligible absence type") + } + if !errors.Is(err, context.Canceled) || len(attempts) != 1 || attempts[0].Outcome != "phase_censored" { + t.Fatalf("attempts=%+v err=%v", attempts, err) + } +} + +func TestEditProbeDisappearanceBeforeAckRetainsActualFirstCorrect(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + defer unblock() + const logical = "file.go::Old" + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Params["name"] == "edit" { + close(entered) + select { + case <-release: + case <-ctx.Done(): + } + return stubContent(map[string]any{"mutation_receipt": "committed"}), nil + } + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": "symbol not found: owned/" + logical}}}, nil + }) + editor, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer editor.Close() + prober, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer prober.Close() + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.absenceBefore = map[string]bool{logical: true} + state.started = time.Now() + state.record.FirstCorrect = map[string]Metric{"old_source_absent": unavailable("ms", "not observed")} + run := &editProbeRuntime{p: &privateRun{Repos: []privateRepo{{Prefix: "owned"}}}, phase: &Phase{}, origin: state.started, options: runOptions{Deadline: time.Second}} + done := make(chan error, 1) + go func() { + _, _, err := run.perform(ctx, state, editor, request{Tool: "edit", Operation: "symbol", Arguments: map[string]any{"operation": "symbol", "target": map[string]any{"symbol": logical}, "match": "old", "replacement": ""}}, "edit", "edit", 0) + done <- err + }() + defer func() { + unblock() + select { + case <-done: + case <-ctx.Done(): + } + }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("edit was not sent") + } + req := request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: logical, Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": logical}}} + raw, identity, err := run.perform(ctx, state, prober, req, "oracle:old_source_absent", "probe", 1) + if _, ok := err.(*editProbeExpectedAbsence); !ok { + t.Fatalf("pre-ACK disappearance lost: %v", err) + } + if err := run.observeProbe(state, editProbeOracle{name: "old_source_absent", call: req}, identity.WriteStarted, raw, identity, err, true); err != nil { + t.Fatal(err) + } + if state.record.FirstCorrect["old_source_absent"].Value == nil { + t.Fatal("actual early disappearance was gated behind ACK") + } + select { + case <-done: + t.Fatal("ACK was not held") + default: + } +} diff --git a/bench/load/edit_probe_background_test.go b/bench/load/edit_probe_background_test.go new file mode 100644 index 000000000..022367a74 --- /dev/null +++ b/bench/load/edit_probe_background_test.go @@ -0,0 +1,85 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func TestEditProbeAcceptsOnlyExplicitMeasuredNativeHandoffCoverage(t *testing.T) { + for _, name := range []string{"chain", "missing_count", "zero_count", "one_count", "unknown", "single_contradiction"} { + t.Run(name, func(t *testing.T) { + body := editProbeRepairAccountingFixture() + body["applied_generation"] = json.Number("3") + winner := body["coalesced_publication"].(map[string]any) + winner["applied_generation"] = json.Number("3") + winner["publication_phases"].(map[string]any)["ticket"] = json.Number("3") + winner["timings"].(map[string]any)["phase_timing"].(map[string]any)["patch_accounting"].(map[string]any)["attempt_ticket"] = json.Number("3") + background := body["requested_background"].(map[string]any) + background["applied_generation"] = json.Number("3") + background["coverage"] = "successful_pre_callback_native_handoffs" + background["observed_native_handoffs"] = json.Number("2") + switch name { + case "missing_count": + delete(background, "observed_native_handoffs") + case "zero_count": + background["observed_native_handoffs"] = json.Number("0") + case "one_count": + background["observed_native_handoffs"] = json.Number("1") + case "unknown": + background["coverage"] = "assumed_native_chain" + case "single_contradiction": + background["coverage"] = "one_canceled_timer_one_native_successor" + } + got := checkEditProbeAccounting(body, "receipt-a") + if (got.State == "observed_single_window_checks_pass") != (name == "chain") { + t.Fatalf("coverage%s: %+v", name, got) + } + }) + } +} + +func TestEditProbeOwnedBackgroundRefusalIsTerminalFailedCoverageNotMissingAsZero(t *testing.T) { + for _, name := range []string{"observed", "preexisting_stop_failed", "missing", "unknown_reason", "foreign_publication", "foreign_requested", "foreign_last", "missing_count", "wrong_source", "contradictory_B"} { + t.Run(name, func(t *testing.T) { + body := editProbeRepairAccountingFixture() + delete(body, "requested_background") + diagnostic := map[string]any{"scope": "requested_background_observation", "reason": "timer_callback_already_started", "publication_key": "original", "requested_generation": json.Number("1"), "last_observed_generation": json.Number("2"), "observed_native_handoffs": json.Number("1")} + body["requested_background_unavailable"] = diagnostic + switch name { + case "preexisting_stop_failed": + diagnostic["reason"] = "preexisting_timer_stop_not_successful" + diagnostic["last_observed_generation"] = json.Number("1") + diagnostic["observed_native_handoffs"] = json.Number("0") + case "missing": + delete(body, "requested_background_unavailable") + case "unknown_reason": + diagnostic["reason"] = "maybe_busy" + case "foreign_publication": + diagnostic["publication_key"] = "receipt-a" + case "foreign_requested": + diagnostic["requested_generation"] = json.Number("2") + case "foreign_last": + diagnostic["last_observed_generation"] = json.Number("3") + case "missing_count": + delete(diagnostic, "observed_native_handoffs") + case "wrong_source": + body["publication_phases"].(map[string]any)["source"] = "canonical_coalesced_attempt" + case "contradictory_B": + body["requested_background"] = map[string]any{} + } + got := checkEditProbeAccounting(body, "receipt-a") + if got.State == "observed_single_window_checks_pass" || got.B != nil { + t.Fatalf("unavailable B became coverage: %+v", got) + } + if (got.State == "terminal_unavailable_background") != (name == "observed" || name == "preexisting_stop_failed") { + t.Fatalf("refusal%s: %+v", name, got) + } + if editProbeFinished(true, true, true, got) != (name == "observed" || name == "preexisting_stop_failed") { + t.Fatalf("unowned/missing refusal stopped collection: %+v", got) + } + if editProbeFinished(false, true, true, got) || editProbeFinished(true, false, true, got) { + t.Fatal("refusal bypassed graph/oracle completion") + } + }) + } +} diff --git a/bench/load/edit_probe_batch.go b/bench/load/edit_probe_batch.go new file mode 100644 index 000000000..1eb30b96c --- /dev/null +++ b/bench/load/edit_probe_batch.go @@ -0,0 +1,279 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "path/filepath" + "strings" +) + +// Batch source authority is the durable transaction. The acknowledgement RPC +// and this file's graph ticket have separate clocks/retention and are never +// represented as one synthetic mutation_receipt. +type editProbeBatchReferences struct { + TransactionID string `json:"transaction_id"` + Observation string `json:"file_observation_receipt"` + Acknowledgement string `json:"acknowledgement_receipt"` + Path string `json:"path"` + RelativePath string `json:"relative_path"` + AfterSHA256 string `json:"after_sha256"` + GraphReceipt string `json:"reindex_receipt"` + GraphGeneration uint64 `json:"reindex_generation"` + ResultReceipt string `json:"result_reindex_receipt"` + ResultGeneration uint64 `json:"result_reindex_generation"` + AcknowledgementState string `json:"acknowledgement_state"` + TransactionError string `json:"transaction_error,omitempty"` +} + +// This explicit fixture offers whole-file before/after bytes. It is not the +// generic anchor-splice batch shape: the same precomputed prefix as edit.write +// makes its durable after-image provable without reading later server state. +func editProbeBatchRequest(c Corpus, before, marker, transactionID string) (request, string) { + _, after := editProbeWriteRequest(c, before, marker) + return editProbeBatchContentRequest(c, before, after, transactionID), after +} + +func editProbeBatchContentRequest(c Corpus, before, after, transactionID string) request { + return request{Tool: "edit", Operation: "batch", Arguments: map[string]any{ + "operation": "batch", + "changes": []any{map[string]any{"op": "edit_file", "path": c.File, "old_string": before, "new_string": after}}, + "options": map[string]any{"transaction_id": transactionID}, + }} +} + +// Only the pointed one-file edit_file shape is supported here. Missing timing +// references do not negate a genuine committed ACK: validateReferences below +// reports those independently, so unavailable accounting cannot become zero or +// remove the successful source acknowledgement from its denominator. +func decodeEditProbeBatchAcknowledgement(body map[string]any, transactionID, absPath, relativePath string, offered []byte) (editProbeBatchReferences, error) { + out := editProbeBatchReferences{} + if transactionID == "" || body["transaction_id"] != transactionID || body["status"] != "committed" || body["disk_status"] != "committed" { + return out, errors.New("batch transaction is not the requested committed source acknowledgement") + } + files, ok := body["files"].([]any) + if !ok || len(files) != 1 { + return out, errors.New("one-file batch acknowledgement requires exactly one durable file") + } + results, ok := body["results"].([]any) + if !ok || len(results) != 1 { + return out, errors.New("one-file batch acknowledgement requires exactly one applied result") + } + file, ok := files[0].(map[string]any) + if !ok { + return out, errors.New("invalid durable file") + } + result, ok := results[0].(map[string]any) + if !ok || result["op"] != "edit_file" || result["status"] != "applied" || result["path"] != relativePath { + return out, errors.New("batch result does not identify the applied requested edit_file") + } + path, _ := file["path"].(string) + rel, _ := file["relative_path"].(string) + if !filepath.IsAbs(absPath) || filepath.Clean(absPath) != absPath || path != absPath || rel != relativePath || relativePath == "" || file["after_absent"] == true || file["before_absent"] == true { + return out, errors.New("batch durable file identity does not match the existing requested file") + } + for _, name := range []string{"after_absent", "before_absent"} { + if value, present := file[name]; present { + if absent, ok := value.(bool); !ok || absent { + return out, errors.New("batch file absence flag contradicts existing-file edit") + } + } + } + after, _ := file["after_sha256"].(string) + bytes, err := hex.DecodeString(after) + if err != nil || len(bytes) != 32 || after != strings.ToLower(after) { + return out, errors.New("batch durable after-image SHA256 unavailable") + } + expected := sha256.Sum256(offered) + if len(offered) == 0 || after != hex.EncodeToString(expected[:]) { + return out, errors.New("batch durable after-image does not match exact offered whole-file bytes") + } + out.TransactionID, out.Path, out.RelativePath, out.AfterSHA256 = transactionID, path, rel, after + if value, present := body["error"]; present { + message, ok := value.(string) + if !ok { + return out, errors.New("invalid transaction error") + } + out.TransactionError = message + } + out.Acknowledgement, _ = body["acknowledgement_receipt"].(string) + out.AcknowledgementState, _ = body["acknowledgement_state"].(string) + out.GraphReceipt, _ = file["reindex_receipt"].(string) + if value, present := file["reindex_generation"]; present { + generation, err := strconvEditProbeUint(value) + if err != nil { + return out, errors.New("invalid batch file generation") + } + out.GraphGeneration = generation + } + out.ResultReceipt, _ = result["reindex_receipt"].(string) + if value, present := result["reindex_generation"]; present { + generation, err := strconvEditProbeUint(value) + if err != nil { + return out, errors.New("invalid batch result generation") + } + out.ResultGeneration = generation + } + if rows, ok := body["file_observations"].([]any); ok && len(rows) == 1 { + if row, ok := rows[0].(map[string]any); ok && row["path"] == relativePath { + out.Observation, _ = row["receipt"].(string) + } + } + return out, nil +} + +func (r editProbeBatchReferences) validateReferences() error { + if !strings.HasPrefix(r.Observation, "batch-file-") || r.Observation == "batch-file-" { + return errors.New("original batch file observation reference unavailable") + } + if r.Observation == r.TransactionID || r.Observation == r.Acknowledgement || r.Observation == r.GraphReceipt { + return errors.New("batch file timing identity borrowed from another domain") + } + + if !strings.HasPrefix(r.Acknowledgement, "batch-ack-") || r.Acknowledgement == "batch-ack-" || r.AcknowledgementState != "reference_only_query_mutation_status" { + return errors.New("original shared batch acknowledgement reference unavailable") + } + if r.GraphReceipt == "" || r.GraphGeneration == 0 { + return errors.New("original batch file graph reference unavailable") + } + if r.ResultReceipt != r.GraphReceipt || r.ResultGeneration != r.GraphGeneration { + return errors.New("batch result and durable file graph identities disagree") + } + if r.TransactionID == r.Acknowledgement || r.TransactionID == r.GraphReceipt || r.Acknowledgement == r.GraphReceipt { + return errors.New("batch transaction/acknowledgement/file identities are not distinct") + } + return nil +} + +// Validate the response-only observation against the durable ACK before any +// phase accounting. This does not replace the transaction's source verdict or +// synthesize a single-file mutation receipt from the shared RPC clock. +func (r editProbeBatchReferences) validateFileObservation(status map[string]any) error { + if err := r.validateReferences(); err != nil { + return err + } + if status["found"] != true || status["receipt_kind"] != "batch_file_observation" || + status["receipt"] != r.Observation || status["transaction_id"] != r.TransactionID || + status["path"] != r.RelativePath || status["batch_disk_verdict"] != "committed" || + status["after_sha256"] != r.AfterSHA256 || status["publication_receipt"] != r.GraphReceipt || + status["writer_return_observed"] != true || status["acknowledgement_state"] != "not_applicable_shared_transaction_rpc" { + return errors.New("batch file timing does not identify original durable file and publication") + } + generation, err := strconvEditProbeUint(status["publication_generation"]) + if err != nil || generation != r.GraphGeneration { + return errors.New("batch file timing publication generation mismatch") + } + return nil +} + +func (r editProbeBatchReferences) accounting(fileStatus, acknowledgementStatus map[string]any) editProbeAccounting { + if err := r.validateFileObservation(fileStatus); err != nil { + return editProbeAccounting{State: "unavailable", Reasons: []string{err.Error()}} + } + if acknowledgementStatus["found"] != true || acknowledgementStatus["receipt"] != r.Acknowledgement || acknowledgementStatus["receipt_kind"] != "batch_acknowledgement" { + return editProbeAccounting{State: "unavailable", Reasons: []string{"original shared batch ACK owner unavailable"}} + } + shared, _ := acknowledgementStatus["acknowledgement"].(map[string]any) + if shared == nil { + return editProbeAccounting{State: "unavailable", Reasons: []string{"shared batch ACK intervals unavailable"}} + } + return checkEditProbeAccountingWithAcknowledgement(fileStatus, r.Observation, "publication_generation", shared, true) +} + +func editProbeBatchIntent(req request) (string, []byte, error) { + if req.Tool != "edit" || req.Operation != "batch" { + return "", nil, errors.New("not the supported edit.batch request") + } + options, ok := req.Arguments["options"].(map[string]any) + if !ok { + return "", nil, errors.New("batch transaction option unavailable") + } + id, _ := options["transaction_id"].(string) + changes, ok := req.Arguments["changes"].([]any) + if id == "" || !ok || len(changes) != 1 { + return "", nil, errors.New("one-file batch offered intent unavailable") + } + change, ok := changes[0].(map[string]any) + if !ok || change["op"] != "edit_file" { + return "", nil, errors.New("unsupported batch operation") + } + before, beforeOK := change["old_string"].(string) + after, afterOK := change["new_string"].(string) + if !beforeOK || !afterOK || before == "" || after == "" { + return "", nil, errors.New("batch whole-file intent unavailable") + } + return id, []byte(after), nil +} + +// Poll the durable transaction separately from the shared A and original file +// P/B/R owners. This begins only after own-ACK immediate readbacks; neither +// scheduling nor the readback continuation waits for another offered edit. +func (r *editProbeRuntime) collectBatchEvidence(ctx context.Context, s *editProbeState, refs editProbeBatchReferences, offered []byte) { + perform := func(req request, purpose string, slot int) (map[string]any, error) { + raw, _, err := r.perform(ctx, s, s.client, req, purpose, "probe", slot) + if err != nil { + return nil, err + } + return decodeEditProbeResult(raw) + } + for ctx.Err() == nil { + tx, err := perform(request{Tool: "edit", Operation: "batch", Arguments: map[string]any{"operation": "batch", "options": map[string]any{"transaction_id": refs.TransactionID, "status_only": true}}}, "batch_transaction_status", 200) + if errors.Is(err, errEditProbeEvidenceIncomplete) { + return + } + if err == nil { + current, decodeErr := decodeEditProbeBatchAcknowledgement(tx, refs.TransactionID, refs.Path, refs.RelativePath, offered) + if decodeErr != nil { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "batch status source identity: "+decodeErr.Error()) + s.mu.Unlock() + return + } + // A durable recovery may legitimately replace its live publication. That + // does not authorize borrowing the replacement for the original clocks. + if current.GraphReceipt != refs.GraphReceipt || current.GraphGeneration != refs.GraphGeneration { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "batch publication was readmitted; original accounting unavailable") + s.mu.Unlock() + return + } + file, fileErr := perform(request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": refs.Observation}}}, "batch_file_status", 201) + ack, ackErr := perform(request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": refs.Acknowledgement}}}, "batch_ack_status", 202) + if errors.Is(fileErr, errEditProbeEvidenceIncomplete) || errors.Is(ackErr, errEditProbeEvidenceIncomplete) { + return + } + if fileErr == nil && ackErr == nil { + accounting := refs.accounting(file, ack) + terminal := tx["graph_status"] == "fresh" || tx["graph_status"] == "failed" + s.mu.Lock() + s.record.Accounting = accounting + s.record.PhasesMS = editProbePhases(file, "batch file status exposes no phases_ms") + s.record.Derived = editProbeDerived(file) + s.record.GraphTerminal = terminal + if terminal { + r.settlePendingLocked(s) + } + all := true + for _, value := range s.record.FirstCorrect { + if value.Value == nil { + all = false + } + } + if terminal && tx["graph_status"] != "fresh" { + s.record.Errors = append(s.record.Errors, "batch graph publication failed") + } + s.mu.Unlock() + if terminal && tx["graph_status"] != "fresh" { + return + } + if editProbeFinished(terminal, all, r.requireAccounting, accounting) { + return + } + } + } + if waitContext(ctx, editProbePeriod) != nil { + return + } + } +} diff --git a/bench/load/edit_probe_batch_test.go b/bench/load/edit_probe_batch_test.go new file mode 100644 index 000000000..0a362f4c2 --- /dev/null +++ b/bench/load/edit_probe_batch_test.go @@ -0,0 +1,445 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" +) + +func editProbeBatchACKAbsPath() string { + return filepath.Join(os.TempDir(), "edit-probe-owned-repo", "file.go") +} +func editProbeBatchOfferedBytes() []byte { + return []byte("package fixture\nfunc Added() { Callee() }\n") +} +func editProbeBatchOfferedSHA() string { + hash := sha256.Sum256(editProbeBatchOfferedBytes()) + return hex.EncodeToString(hash[:]) +} + +func editProbeBatchACKFixture() map[string]any { + return map[string]any{"file_observations": []any{map[string]any{"path": "repo/file.go", "receipt": "batch-file-5"}}, "transaction_id": "transaction-1", "status": "committed", "disk_status": "committed", "acknowledgement_receipt": "batch-ack-2", "acknowledgement_state": "reference_only_query_mutation_status", "files": []any{map[string]any{"path": editProbeBatchACKAbsPath(), "relative_path": "repo/file.go", "after_sha256": editProbeBatchOfferedSHA(), "reindex_receipt": "mutation-3", "reindex_generation": json.Number("4")}}, "results": []any{map[string]any{"op": "edit_file", "path": "repo/file.go", "status": "applied", "reindex_receipt": "mutation-3", "reindex_generation": json.Number("4")}}} +} + +func TestEditProbeBatchKeepsActualTransactionACKAndFileIdentities(t *testing.T) { + refs, err := decodeEditProbeBatchAcknowledgement(editProbeBatchACKFixture(), "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()) + if err != nil || refs.validateReferences() != nil { + t.Fatal(refs, err) + } + if refs.TransactionID != "transaction-1" || refs.Acknowledgement != "batch-ack-2" || refs.GraphReceipt != "mutation-3" || refs.GraphGeneration != 4 || refs.AfterSHA256 != editProbeBatchOfferedSHA() { + t.Fatal(refs) + } +} + +func TestEditProbeBatchRejectsWrongSourceVerdictOrFileJoin(t *testing.T) { + for _, name := range []string{"transaction", "rollback", "pending", "files", "results", "wrong_path", "wrong_relative", "wrong_result_path", "wrong_op", "failed_result", "absent", "malformed_absent", "hash", "foreign_after_hash", "generation"} { + t.Run(name, func(t *testing.T) { + body := editProbeBatchACKFixture() + file := body["files"].([]any)[0].(map[string]any) + result := body["results"].([]any)[0].(map[string]any) + switch name { + case "transaction": + body["transaction_id"] = "foreign" + case "rollback": + body["disk_status"] = "rolled_back" + case "pending": + body["status"] = "prepared" + case "files": + body["files"] = []any{file, file} + case "results": + body["results"] = []any{} + case "wrong_path": + file["path"] = "/foreign/file.go" + case "wrong_relative": + file["relative_path"] = "foreign/file.go" + case "wrong_result_path": + result["path"] = "foreign/file.go" + case "wrong_op": + result["op"] = "edit_symbol" + case "failed_result": + result["status"] = "failed" + case "absent": + file["after_absent"] = true + case "malformed_absent": + file["after_absent"] = "false" + case "hash": + file["after_sha256"] = strings.Repeat("b", 40) + case "foreign_after_hash": + file["after_sha256"] = strings.Repeat("b", 64) + case "generation": + file["reindex_generation"] = json.Number("1.5") + } + if _, err := decodeEditProbeBatchAcknowledgement(body, "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()); err == nil { + t.Fatal("accepted contradictory durable ACK", name) + } + }) + } +} + +func TestEditProbeBatchMissingReferencesDoNotEraseCommittedACK(t *testing.T) { + for _, name := range []string{"missing_A", "missing_graph", "wrong_result_graph", "wrong_result_generation", "swapped_domains"} { + t.Run(name, func(t *testing.T) { + body := editProbeBatchACKFixture() + file := body["files"].([]any)[0].(map[string]any) + result := body["results"].([]any)[0].(map[string]any) + switch name { + case "missing_A": + delete(body, "acknowledgement_receipt") + case "missing_graph": + delete(file, "reindex_receipt") + delete(file, "reindex_generation") + case "wrong_result_graph": + result["reindex_receipt"] = "foreign" + case "wrong_result_generation": + result["reindex_generation"] = json.Number("5") + case "swapped_domains": + file["reindex_receipt"] = "batch-ack-2" + result["reindex_receipt"] = "batch-ack-2" + } + refs, err := decodeEditProbeBatchAcknowledgement(body, "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()) + if err != nil { + t.Fatal("lost genuine committed source ACK", err) + } + if refs.validateReferences() == nil { + t.Fatal("accepted unavailable or foreign accounting", refs) + } + }) + } +} + +func TestEditProbeBatchRequestUsesPublicChangesAndTransactionOption(t *testing.T) { + c := Corpus{File: "file.go", EditAnchor: "// anchor\n", EditTemplate: "func {{marker}}() { Callee() }\n"} + before := "package fixture\n// anchor\n" + r, after := editProbeBatchRequest(c, before, "Marker", "transaction-1") + if r.name() != "edit.batch" { + t.Fatal(r) + } + expectedAfter := "package fixture\n// anchor\nfunc Marker() { Callee() }\n" + expected := map[string]any{"operation": "batch", "changes": []any{map[string]any{"op": "edit_file", "path": "file.go", "old_string": before, "new_string": expectedAfter}}, "options": map[string]any{"transaction_id": "transaction-1"}} + if after != expectedAfter { + t.Fatal("changed offered intent", after) + } + if !reflect.DeepEqual(r.Arguments, expected) { + t.Fatal("wrong public batch wire shape", r.Arguments) + } + if _, present := r.Arguments["mutation_id"]; present { + t.Fatal("substituted single-edit identity") + } +} + +func TestEditProbeBatchFileObservationRequiresOriginalDurableIdentity(t *testing.T) { + refs, err := decodeEditProbeBatchAcknowledgement(editProbeBatchACKFixture(), "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()) + if err != nil { + t.Fatal(err) + } + fixture := func() map[string]any { + return map[string]any{"found": true, "receipt_kind": "batch_file_observation", "receipt": "batch-file-5", "transaction_id": "transaction-1", "path": "repo/file.go", "batch_disk_verdict": "committed", "after_sha256": editProbeBatchOfferedSHA(), "publication_receipt": "mutation-3", "publication_generation": json.Number("4"), "writer_return_observed": true, "acknowledgement_state": "not_applicable_shared_transaction_rpc"} + } + if err := refs.validateFileObservation(fixture()); err != nil { + t.Fatal(err) + } + for _, key := range []string{"receipt", "transaction_id", "path", "after_sha256", "publication_receipt", "publication_generation", "batch_disk_verdict", "writer_return_observed", "receipt_kind", "found", "acknowledgement_state"} { + t.Run(key, func(t *testing.T) { + body := fixture() + delete(body, key) + if refs.validateFileObservation(body) == nil { + t.Fatal("accepted missing identity", key) + } + }) + } + for _, foreign := range []string{refs.TransactionID, refs.Acknowledgement, refs.GraphReceipt, "batch-file-other"} { + body := fixture() + body["receipt"] = foreign + if refs.validateFileObservation(body) == nil { + t.Fatal("borrowed receipt", foreign) + } + } +} + +func TestEditProbeBatchAccountingKeepsSharedACKOutsideFileAndWinnerWindows(t *testing.T) { + refs, err := decodeEditProbeBatchAcknowledgement(editProbeBatchACKFixture(), "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()) + if err != nil { + t.Fatal(err) + } + refs.GraphGeneration, refs.ResultGeneration = 1, 1 + file := editProbeAccountingFixture() + shared := map[string]any{"found": true, "receipt": refs.Acknowledgement, "receipt_kind": "batch_acknowledgement", "acknowledgement": file["acknowledgement"]} + file["acknowledgement"] = map[string]any{"rpc_id": "foreign-file-RPC"} + file["found"], file["receipt_kind"], file["receipt"] = true, "batch_file_observation", refs.Observation + file["transaction_id"], file["path"], file["batch_disk_verdict"] = refs.TransactionID, refs.RelativePath, "committed" + file["after_sha256"], file["publication_receipt"], file["publication_generation"] = refs.AfterSHA256, refs.GraphReceipt, json.Number("1") + file["writer_return_observed"], file["acknowledgement_state"] = true, "not_applicable_shared_transaction_rpc" + got := refs.accounting(file, shared) + if got.State != "observed_single_window_checks_pass" || got.RPCID != "rpc-a" || got.A == nil || got.P == nil || got.B == nil { + t.Fatalf("independent windows unavailable: %+v", got) + } + for _, key := range []string{"receipt", "receipt_kind", "found", "acknowledgement"} { + copy := map[string]any{} + for k, v := range shared { + copy[k] = v + } + delete(copy, key) + if refs.accounting(file, copy).State == "observed_single_window_checks_pass" { + t.Fatal("accepted missing shared ACK owner", key) + } + } + if file["acknowledgement"].(map[string]any)["rpc_id"] != "foreign-file-RPC" { + t.Fatal("mutated/merged original file status") + } +} + +func TestEditProbeBatchPlanKeepsOfferedPrefixAndScheduleAcrossFixtureLanguages(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeWriteCorpus(t, language) + write, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + c.EditTool = "edit.batch" + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Tool != "edit.batch" || len(plan.Steps) != 100 { + t.Fatal("wrong tool/denominator") + } + prior := c.EditInitialContent + for i, step := range plan.Steps { + id, offered, err := editProbeBatchIntent(step.Request) + if err != nil { + t.Fatal(err) + } + if id == step.MutationID || id == "" { + t.Fatal("borrowed single mutation identity") + } + if step.DueNS != write.Steps[i].DueNS || step.ActivateNS != write.Steps[i].ActivateNS || step.Marker != write.Steps[i].Marker { + t.Fatal("tool changed offered traffic") + } + if string(offered) != write.Steps[i].Request.Arguments["content"] { + t.Fatal("batch regenerated different offered bytes") + } + change := step.Request.Arguments["changes"].([]any)[0].(map[string]any) + if change["old_string"] != prior { + t.Fatal("batch does not require exact prior offered prefix") + } + prior = string(offered) + args := dispatchArgs(step.Request, privateRepo{Prefix: "owned-repo"}) + wire := args["changes"].([]any)[0].(map[string]any) + if wire["path"] != "owned-repo/"+c.File || change["path"] != c.File { + t.Fatal("wrong actual scoped path or mutated immutable plan") + } + } + if err := editProbeWriteAdmission(c, []byte(c.EditInitialContent+"unexpected")); err == nil { + t.Fatal("accepted different initial bytes") + } + }) + } +} + +func TestEditProbeBatchCollectorReadsThreeIndependentPublicOwners(t *testing.T) { + refs, err := decodeEditProbeBatchAcknowledgement(editProbeBatchACKFixture(), "transaction-1", editProbeBatchACKAbsPath(), "repo/file.go", editProbeBatchOfferedBytes()) + if err != nil { + t.Fatal(err) + } + refs.GraphGeneration, refs.ResultGeneration = 1, 1 + tx := editProbeBatchACKFixture() + tx["graph_status"] = "fresh" + tx["files"].([]any)[0].(map[string]any)["reindex_generation"] = json.Number("1") + tx["results"].([]any)[0].(map[string]any)["reindex_generation"] = json.Number("1") + file := editProbeAccountingFixture() + shared := map[string]any{"found": true, "receipt": refs.Acknowledgement, "receipt_kind": "batch_acknowledgement", "acknowledgement": file["acknowledgement"]} + delete(file, "acknowledgement") + file["found"], file["receipt_kind"], file["receipt"] = true, "batch_file_observation", refs.Observation + file["transaction_id"], file["path"], file["batch_disk_verdict"] = refs.TransactionID, refs.RelativePath, "committed" + file["after_sha256"], file["publication_receipt"], file["publication_generation"] = refs.AfterSHA256, refs.GraphReceipt, json.Number("1") + file["writer_return_observed"], file["acknowledgement_state"] = true, "not_applicable_shared_transaction_rpc" + var mu sync.Mutex + var owners []string + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + options, _ := args["options"].(map[string]any) + var owner string + var response map[string]any + if frame.Params["name"] == "edit" { + owner, _ = options["transaction_id"].(string) + response = tx + } else { + owner, _ = options["receipt"].(string) + switch owner { + case refs.Observation: + response = file + case refs.Acknowledgement: + response = shared + } + } + mu.Lock() + owners = append(owners, owner) + mu.Unlock() + return stubContent(response), nil + }) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + sink, err := os.CreateTemp(t.TempDir(), "batch-evidence-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer sink.Close() + state := &editProbeState{client: client, sink: sink, record: editProbeRecord{FirstCorrect: map[string]Metric{"oracle": measured(1, "ms")}}} + state.startRaw() + defer state.finishRaw() + run := &editProbeRuntime{options: runOptions{Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Prefix: "repo"}}}, phase: &Phase{Metrics: map[string]Metric{}}, origin: time.Now(), requireAccounting: true} + run.collectBatchEvidence(ctx, state, refs, editProbeBatchOfferedBytes()) + if ctx.Err() != nil || !state.record.GraphTerminal || state.record.Accounting.State != "observed_single_window_checks_pass" { + t.Fatalf("actual collector failed: %+v %v", state.record, ctx.Err()) + } + mu.Lock() + defer mu.Unlock() + if !reflect.DeepEqual(owners, []string{refs.TransactionID, refs.Observation, refs.Acknowledgement}) { + t.Fatal("owner route mismatch", owners) + } + if state.record.Receipt != "" { + t.Fatal("invented single-file mutation receipt") + } +} + +func TestBatchAccountingTextSpliceParseApplicabilityIsExplicitAndOwned(t *testing.T) { + base := editProbeAccountingFixture() + ack := base["acknowledgement"].(map[string]any) + ack["phase_states"].(map[string]any)["parse"] = "not_applicable" + ack["observed_execution_ms"].(map[string]any)["parse"] = nil + ack["parse_scope"] = "not_applicable_text_splice" + ack["caller_response_ready_ms"] = json.Number("7") + if got := checkEditProbeAccountingWithAcknowledgement(base, "receipt-a", "reindex_generation", ack, true); got.State != "observed_single_window_checks_pass" { + t.Fatal("explicit unexecuted batch parse not honored", got) + } + if got := checkEditProbeAccounting(base, "receipt-a"); got.State == "observed_single_window_checks_pass" { + t.Fatal("weakened ordinary applicable parse") + } + for _, name := range []string{"missing_scope", "unknown_scope", "missing_state", "unknown_state", "missing_value", "zero_instead_of_null"} { + t.Run(name, func(t *testing.T) { + raw, _ := json.Marshal(ack) + var copy map[string]any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(©); err != nil { + t.Fatal(err) + } + switch name { + case "missing_scope": + delete(copy, "parse_scope") + case "unknown_scope": + copy["parse_scope"] = "unknown" + case "missing_state": + delete(copy["phase_states"].(map[string]any), "parse") + case "unknown_state": + copy["phase_states"].(map[string]any)["parse"] = "unknown" + case "missing_value": + delete(copy["observed_execution_ms"].(map[string]any), "parse") + case "zero_instead_of_null": + copy["observed_execution_ms"].(map[string]any)["parse"] = json.Number("0") + } + if got := checkEditProbeAccountingWithAcknowledgement(base, "receipt-a", "reindex_generation", copy, true); got.State == "observed_single_window_checks_pass" { + t.Fatal("invented batch parse coverage", name) + } + }) + } +} + +func directRequestedBackgroundFixture() map[string]any { + status := editProbeAccountingFixture() + winner := status["coalesced_publication"].(map[string]any) + phase := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + phase["patch_accounting"].(map[string]any)["attempt_ticket"] = json.Number("1") + status["phase_timing"] = phase + delete(status, "coalesced_publication") + status["applied_generation"] = json.Number("1") + status["publication_phases"] = map[string]any{"key": "original", "ticket": json.Number("1"), "source": "mcp_edit", "clock": "monotonic"} + background := status["requested_background"].(map[string]any) + background["coverage"], background["observed_native_handoffs"] = "direct_requested_attempt", json.Number("0") + background["applied_generation"] = json.Number("1") + phases := background["phases_ms"].(map[string]any) + for _, name := range []string{"canceled_quiet_wait", "timer_stop", "coalescing_control", "successor_timer_arm", "successor_quiet_wait"} { + phases[name] = json.Number("0") + } + phases["requested_quiet_wait"] = json.Number("1") + background["background_wall_ms"] = json.Number("16") + return status +} +func directRequestedBackgroundWithPredecessorFixture() map[string]any { + status := directRequestedBackgroundFixture() + status["reindex_generation"], status["applied_generation"] = json.Number("3"), json.Number("3") + status["publication_phases"].(map[string]any)["ticket"] = json.Number("3") + status["phase_timing"].(map[string]any)["patch_accounting"].(map[string]any)["attempt_ticket"] = json.Number("3") + b := status["requested_background"].(map[string]any) + b["requested_generation"], b["applied_generation"] = json.Number("3"), json.Number("3") + b["admission_predecessor_generation"], b["admission_predecessor_stopped"] = json.Number("2"), true + return status +} +func TestDirectRequestedBackgroundConsumerRequiresExactOwnedWindow(t *testing.T) { + if got := checkEditProbeAccounting(directRequestedBackgroundFixture(), "receipt-a"); got.State != "observed_single_window_checks_pass" { + t.Fatal("valid direct B refused", got) + } + if got := checkEditProbeAccounting(directRequestedBackgroundWithPredecessorFixture(), "receipt-a"); got.State != "observed_single_window_checks_pass" { + t.Fatal("valid actual predecessor proof refused", got) + } + for _, name := range []string{"missing_quiet", "borrowed_native_quiet", "wrong_applied", "fake_handoff", "unknown_shape", "predecessor_stop_false", "foreign_predecessor", "malformed_predecessor", "missing_stop_proof"} { + t.Run(name, func(t *testing.T) { + status := directRequestedBackgroundFixture() + if strings.Contains(name, "predecessor") || name == "missing_stop_proof" { + status = directRequestedBackgroundWithPredecessorFixture() + } + b := status["requested_background"].(map[string]any) + phases := b["phases_ms"].(map[string]any) + switch name { + case "missing_quiet": + delete(phases, "requested_quiet_wait") + case "borrowed_native_quiet": + phases["successor_quiet_wait"] = json.Number("1") + case "wrong_applied": + b["applied_generation"] = json.Number("2") + case "fake_handoff": + b["observed_native_handoffs"] = json.Number("1") + case "unknown_shape": + b["coverage"] = "unknown" + case "predecessor_stop_false": + b["admission_predecessor_stopped"] = false + case "foreign_predecessor": + b["admission_predecessor_generation"] = json.Number("3") + case "malformed_predecessor": + b["admission_predecessor_generation"] = "not-a-generation" + case "missing_stop_proof": + delete(b, "admission_predecessor_stopped") + } + if got := checkEditProbeAccounting(status, "receipt-a"); got.State == "observed_single_window_checks_pass" { + t.Fatal("accepted contradictory direct B", name) + } + }) + } +} + +func TestCoalescedBackgroundCannotBorrowAnExecutedRequestedQuietLap(t *testing.T) { + status := editProbeAccountingFixture() + b := status["requested_background"].(map[string]any) + b["phases_ms"].(map[string]any)["requested_quiet_wait"] = json.Number("0") + if got := checkEditProbeAccounting(status, "receipt-a"); got.State != "observed_single_window_checks_pass" { + t.Fatal("new zero lap broke historical coalesced shape", got) + } + b["phases_ms"].(map[string]any)["requested_quiet_wait"] = json.Number("1") + if editProbeSupportedBackgroundCoverage(b) { + t.Fatal("coalesced shape borrowed an already-running requested callback") + } +} diff --git a/bench/load/edit_probe_body.go b/bench/load/edit_probe_body.go new file mode 100644 index 000000000..26568f756 --- /dev/null +++ b/bench/load/edit_probe_body.go @@ -0,0 +1,172 @@ +package main + +import ( + "path/filepath" + "strings" +) + +func editProbeFollowingDeclaration(file, name string) string { + switch filepath.Ext(file) { + case ".go": + return "func " + name + "() int { return 2 }" + case ".ts": + return "export function " + name + "(): number { return 2; }" + case ".py": + return "def " + name + "():\n return 2" + } + return "" +} + +func editProbeBodyOracles(c Corpus, step editProbeStep) []editProbeOracle { + target := map[string]any{"symbol": step.TargetSymbol} + file := map[string]any{"file": c.File} + oracles := []editProbeOracle{ + {"source", request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": target, "options": map[string]any{"context_lines": 0}}}}, + {"summary", request{Tool: "read", Operation: "summary", Arguments: map[string]any{"operation": "summary", "target": file}}}, + {"editing_context", request{Tool: "read", Operation: "editing_context", Arguments: map[string]any{"operation": "editing_context", "target": file}}}, + {"search_visible", request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": step.Marker, "options": map[string]any{"query_class": "symbol", "expand": "off", "limit": 1000, "max_per_file": 0}}}}, + {"text", request{Tool: "search", Operation: "text", Arguments: map[string]any{"operation": "text", "query": "return " + c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] + "()", "options": map[string]any{"limit": 1000}}}}, + {"callers", request{Tool: "relations", Operation: "callers", Arguments: map[string]any{"operation": "callers", "target": map[string]any{"symbol": c.Symbol}, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + {"call_chain", request{Tool: "trace", Operation: "call_chain", Arguments: map[string]any{"operation": "call_chain", "target": target, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + } + if c.EditKind == "change_signature" { + oracles[4].call.Arguments["query"] = "optional" + if c.EditOutput == "toon" { + // Compact traversal rows retain start lines, not signature/end-line + // metadata. The unchanged real caller supplies the relative witness. + oracles[5].call.Arguments["target"] = target + oracles[6].call.Arguments["target"] = map[string]any{"symbol": step.FollowingSymbol} + oracles[6].call.Arguments["options"] = map[string]any{"depth": 2, "limit": 1000} + } + } + return oracles +} + +// The precreated following declaration is an unchanged reference point. Any +// upstream edits move both rows equally; only this target's actual line growth +// changes their relative start-line gap. Presence of a precreated name alone +// cannot establish first-correct visibility. +func editProbeBodyGap(rows []any, c Corpus, step editProbeStep) bool { + targetName := step.TargetSymbol[strings.LastIndex(step.TargetSymbol, "::")+2:] + followingName := step.FollowingSymbol[strings.LastIndex(step.FollowingSymbol, "::")+2:] + var target, following uint64 + targetCount, followingCount := 0, 0 + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if editProbeSourceID(id, c.File, targetName) { + n, err := strconvEditProbeUint(row["start_line"]) + if err != nil || n == 0 { + return false + } + target = n + targetCount++ + } + if editProbeSourceID(id, c.File, followingName) { + n, err := strconvEditProbeUint(row["start_line"]) + if err != nil || n == 0 { + return false + } + following = n + followingCount++ + } + } + return targetCount == 1 && followingCount == 1 && following > target && following-target == uint64(step.ExpectedLineGap) +} + +func editProbeBodyCorrect(oracle string, body map[string]any, c Corpus, step editProbeStep, callee string) bool { + name := step.TargetSymbol[strings.LastIndex(step.TargetSymbol, "::")+2:] + expected := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", name)) + switch oracle { + case "source": + id, _ := body["id"].(string) + path, _ := body["file_path"].(string) + source, _ := body["source"].(string) + return body["name"] == name && editProbeSourceID(id, c.File, name) && editProbeFixturePath(path, c.File) && strings.TrimSpace(source) == expected && editProbeFixtureSource(c.File, editProbeBodyPlain(c, source), name, callee) + case "file": + path, _ := body["path"].(string) + source, _ := body["content"].(string) + return editProbeFixturePath(path, c.File) && strings.Count(source, expected) == 1 && editProbeFixtureSource(c.File, editProbeBodyPlain(c, source), name, callee) + case "summary", "editing_context", "search_visible": + key := map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"}[oracle] + rows, ok := body[key].([]any) + if !ok || !editProbeBodyGap(rows, c, step) { + return false + } + if c.EditKind == "change_signature" && oracle != "search_visible" { + return editProbeBodySignature(rows, c, step, oracle == "summary") + } + return true + case "text": + rows, ok := body["matches"].([]any) + if !ok { + return false + } + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + id, _ := row["symbol_id"].(string) + path, _ := row["path"].(string) + text, _ := row["text"].(string) + statement := "return " + callee + "()" + if filepath.Ext(c.File) == ".ts" { + statement += ";" + } + if c.EditKind == "change_signature" { + statement, _, _ = strings.Cut(expected, "\n") + } + if editProbeSourceID(id, c.File, name) && editProbeFixturePath(path, c.File) && strings.TrimSpace(text) == statement { + return true + } + } + return false + case "callers", "call_chain": + if c.EditKind == "modify_caller_body" && oracle == "call_chain" { + truncated, ok := body["truncated"].(bool) + _, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + _, eok := editProbeCompleteCount(body, "edges", "total_edges") + if !ok || truncated || !nok || !eok { + return false + } + } + edges, ok := body["edges"].([]any) + if !ok { + return false + } + found := false + for _, raw := range edges { + row, ok := raw.(map[string]any) + if !ok { + return false + } + from, _ := row["from"].(string) + to, _ := row["to"].(string) + if row["kind"] == "calls" && editProbeSourceID(from, c.File, name) { + if c.EditKind == "modify_caller_body" && editProbeSourceID(to, c.File, "Alternate") { + return false + } + if editProbeSourceID(to, c.File, callee) { + found = true + } + } + } + if c.EditKind == "change_signature" { + if c.EditOutput == "toon" { + operation := map[string]string{"callers": "relations.callers", "call_chain": "trace.call_chain"}[oracle] + proof, ok := body[editProbeTOONWireKey].(editProbeTOONWire) + rows, rowsOK := body["nodes"].([]any) + _, nodesComplete := editProbeCompleteCount(body, "nodes", "total_nodes") + _, edgesComplete := editProbeCompleteCount(body, "edges", "total_edges") + return found && ok && proof.operation == operation && rowsOK && nodesComplete && edgesComplete && editProbeBodyGap(rows, c, step) + } + return found && editProbeBodySpan(body, c, step) + } + return found + } + return false +} diff --git a/bench/load/edit_probe_body_test.go b/bench/load/edit_probe_body_test.go new file mode 100644 index 000000000..8b220c5c3 --- /dev/null +++ b/bench/load/edit_probe_body_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func editProbeBodyCorpus(t *testing.T, language, kind string, count int) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, language) + c.EditKind = kind + switch language { + case "go": + c.EditBeforeTemplate = "func {{marker}}() int { return 1 }" + c.EditTemplate = "func {{marker}}() int {\n return Example()\n}" + case "typescript": + c.EditBeforeTemplate = "export function {{marker}}(): number { return 1; }" + c.EditTemplate = "export function {{marker}}(): number {\n return Example();\n}" + case "python": + c.EditBeforeTemplate = "def {{marker}}():\n return 1" + c.EditTemplate = "def {{marker}}():\n\n return Example()" + } + if kind == "modify_caller_body" { + c.EditBeforeTemplate = strings.Replace(c.EditBeforeTemplate, "return 1", "return Alternate()", 1) + c.EditInitialContent += "\n\n" + editProbeFollowingDeclaration(c.File, "Alternate") + } + for i := 0; i < count; i++ { + marker := fmt.Sprintf("GortexEditProbe%016xN%03d", uint64(42), i) + c.EditInitialContent += "\n\n" + strings.ReplaceAll(c.EditBeforeTemplate, "{{marker}}", marker+"Before") + "\n\n" + editProbeFollowingDeclaration(c.File, marker+"After") + } + return c +} + +func TestEditProbeBodyChangesRetainTargetAndUnchangedRelativeWitness(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, kind := range []string{"modify_body", "modify_caller_body"} { + for _, tool := range []string{"edit.file", "edit.write", "edit.symbol", "edit.batch"} { + t.Run(language+"/"+kind+"/"+tool, func(t *testing.T) { + c := editProbeBodyCorpus(t, language, kind, 100) + c.EditTool = tool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + if step.TargetSymbol != c.File+"::"+step.Marker+"Before" || step.FollowingSymbol != c.File+"::"+step.Marker+"After" || step.ExpectedLineGap < 3 { + t.Fatal("body change lost precreated target/follower") + } + if len(editProbeOracles(c, step)) != 7 { + t.Fatal("body cell changed seven-oracle denominator") + } + } + }) + } + } + } +} + +func TestEditProbeBodyRelativeGapCannotBorrowUpstreamMovementOrNamePresence(t *testing.T) { + c := editProbeBodyCorpus(t, "go", "modify_body", 1) + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + for _, variant := range []string{"after", "upstream_shift_after", "before", "upstream_shift_before", "wrong_follower", "duplicate_target", "missing_line"} { + t.Run(variant, func(t *testing.T) { + start := 10 + gap := step.ExpectedLineGap + if variant == "before" || variant == "upstream_shift_before" { + gap -= 2 + } + if strings.HasPrefix(variant, "upstream_shift") { + start += 200 + } + target := map[string]any{"id": "owned/" + step.TargetSymbol, "start_line": json.Number(fmt.Sprint(start))} + follower := map[string]any{"id": "owned/" + step.FollowingSymbol, "start_line": json.Number(fmt.Sprint(start + gap))} + rows := []any{target, follower} + switch variant { + case "wrong_follower": + follower["id"] = "owned/other.go::Following" + case "duplicate_target": + rows = append(rows, target) + case "missing_line": + delete(target, "start_line") + } + want := variant == "after" || variant == "upstream_shift_after" + for _, oracle := range []string{"summary", "editing_context", "search_visible"} { + key := map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"}[oracle] + if got := editProbeCorrect(oracle, map[string]any{key: rows}, c, step, plan.Callee); got != want { + t.Fatalf("%s correct=%v", oracle, got) + } + } + }) + } +} + +func TestEditProbeBodySourceAndTextRequireActualNewCallInExactTarget(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeBodyCorpus(t, language, "modify_body", 1) + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + name := step.Marker + "Before" + after := strings.ReplaceAll(c.EditTemplate, "{{marker}}", name) + before := strings.ReplaceAll(c.EditBeforeTemplate, "{{marker}}", name) + for _, variant := range []string{"after", "before", "wrong_file", "wrong_id", "wrong_callee"} { + t.Run(variant, func(t *testing.T) { + body := map[string]any{"id": "owned/" + step.TargetSymbol, "file_path": "owned/" + c.File, "name": name, "source": after} + switch variant { + case "before": + body["source"] = before + case "wrong_file": + body["file_path"] = "owned/other.go" + case "wrong_id": + body["id"] = "owned/other.go::" + name + case "wrong_callee": + body["source"] = strings.ReplaceAll(after, "Example()", "Other()") + } + if got := editProbeCorrect("source", body, c, step, plan.Callee); got != (variant == "after") { + t.Fatalf("source correct=%v", got) + } + }) + } + statement := "return Example()" + if language == "typescript" { + statement += ";" + } + row := map[string]any{"symbol_id": "owned/" + step.TargetSymbol, "path": "owned/" + c.File, "text": statement} + if !editProbeCorrect("text", map[string]any{"matches": []any{row}}, c, step, plan.Callee) { + t.Fatal("actual enclosed new call refused") + } + row["symbol_id"] = "owned/" + c.File + "::Other" + if editProbeCorrect("text", map[string]any{"matches": []any{row}}, c, step, plan.Callee) { + t.Fatal("foreign target supplied text evidence") + } + }) + } +} diff --git a/bench/load/edit_probe_cli.go b/bench/load/edit_probe_cli.go new file mode 100644 index 000000000..292e16483 --- /dev/null +++ b/bench/load/edit_probe_cli.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "time" +) + +// A CLI probe is one owned `gortex call` process per oracle request. Its +// cadence is bounded by process start, daemon dial and exit, not the 25 ms +// probe period, and the process start stands in for the probe's wire start. +const editProbeCLIStderrLimit = 64 << 10 + +type editProbeCappedBuffer struct { + data []byte + limit int + overflow bool +} + +func (b *editProbeCappedBuffer) Write(data []byte) (int, error) { + if keep := b.limit - len(b.data); len(data) > keep { + b.overflow = true + b.data = append(b.data, data[:max(keep, 0)]...) + return len(data), nil + } + b.data = append(b.data, data...) + return len(data), nil +} + +// The edit ACK, readbacks and receipt status stay MCP JSON; only oracle +// requests go through the CLI, with the same lowered facade arguments. +func (r *editProbeRuntime) performCLI(ctx context.Context, s *editProbeState, req request, purpose string, stream int) (json.RawMessage, wireCallIdentity, error) { + budget := min(r.options.Deadline, 20*time.Second) + callCtx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + args, err := json.Marshal(dispatchArgs(req, r.p.Repos[0])) + if err != nil { + return nil, wireCallIdentity{}, err + } + argv := []string{"--config", r.p.Config, "call", req.Tool, "--json", string(args), "--format", "json", "--index", r.p.Repos[0].Path, "--quiet"} + cmd := exec.CommandContext(callCtx, r.p.Binary, argv...) + configureOwnedProcess(cmd) + cmd.Cancel = func() error { return killOwnedProcess(cmd) } + cmd.WaitDelay = time.Second + stdout := &editProbeCappedBuffer{limit: editProbeRawLimit} + stderr := &editProbeCappedBuffer{limit: editProbeCLIStderrLimit} + cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = r.p.Root, r.p.Env, stdout, stderr + identity := wireCallIdentity{Started: time.Now()} + identity.WriteStarted = identity.Started + err = cmd.Start() + if err == nil { + s.mu.Lock() + if s.firstProbeWrite.IsZero() || identity.WriteStarted.Before(s.firstProbeWrite) { + s.firstProbeWrite = identity.WriteStarted + } + s.mu.Unlock() + err = cmd.Wait() + err = errors.Join(err, cleanupExitedOwnedProcess(cmd)) + } + identity.Finished = time.Now() + outcome := "success" + switch { + case ctx.Err() != nil: + outcome, err = "phase_censored", errors.Join(ctx.Err(), err) + case callCtx.Err() != nil: + outcome, err = "product_deadline", errors.Join(callCtx.Err(), err) + case stdout.overflow: + outcome, err = "error", errors.Join(fmt.Errorf("%w: CLI stdout exceeded the per-edit evidence limit", errEditProbeEvidenceIncomplete), err) + case err != nil: + outcome, err = "error", fmt.Errorf("gortex call: %w: %s", err, stderr.data) + } + exit := -1 + if cmd.ProcessState != nil { + exit = cmd.ProcessState.ExitCode() + } + hash, _ := hashJSON(req) + attempt := Attempt{Tool: "cli:" + req.name(), Role: "probe", Request: hash, Step: s.step.Ordinal, Slot: stream, Started: identity.Started, DurationMS: float64(identity.Finished.Sub(identity.Started)) / float64(time.Millisecond), Outcome: outcome} + attempt.LogicalMS = attempt.DurationMS + if err != nil { + attempt.Error = err.Error() + } + r.mu.Lock() + r.phase.Attempts = append(r.phase.Attempts, attempt) + r.mu.Unlock() + row := map[string]any{"event": "cli", "purpose": purpose, "tool": req.name(), "argv": argv, "exit": exit, "stdout": string(stdout.data), "stderr": string(stderr.data), "stderr_truncated": stderr.overflow, "start_ns": editProbeOffset(identity.Started, r.origin), "end_ns": editProbeOffset(identity.Finished, r.origin), "output_mode": "cli"} + if err != nil { + row["error"] = err.Error() + } + if evidenceErr := s.emit(row); evidenceErr != nil { + err = errors.Join(err, evidenceErr) + } + if err != nil { + // A failed process gave no answer; its stdout stays in evidence only. + return nil, identity, err + } + return stdout.data, identity, nil +} diff --git a/bench/load/edit_probe_cli_test.go b/bench/load/edit_probe_cli_test.go new file mode 100644 index 000000000..0b81777ae --- /dev/null +++ b/bench/load/edit_probe_cli_test.go @@ -0,0 +1,179 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +// The test binary doubles as a scripted `gortex call` process. It records its +// argv and answers from the environment, or hangs until it is killed. +func init() { + switch os.Getenv("LOAD_EDIT_PROBE_CLI_FIXTURE") { + case "": + return + case "hang": + time.Sleep(time.Minute) + os.Exit(9) + } + argv, _ := json.Marshal(os.Args[1:]) + if os.WriteFile(os.Getenv("LOAD_EDIT_PROBE_CLI_ARGV"), argv, 0o600) != nil { + os.Exit(7) + } + fmt.Fprint(os.Stdout, os.Getenv("LOAD_EDIT_PROBE_CLI_STDOUT")) + fmt.Fprint(os.Stderr, "fixture stderr") + if os.Getenv("LOAD_EDIT_PROBE_CLI_EXIT") != "" { + os.Exit(1) + } + os.Exit(0) +} + +func editProbeCLIFixture(t *testing.T, deadline time.Duration, env ...string) (*editProbeRuntime, *editProbeState, editProbeOracle) { + t.Helper() + dir := t.TempDir() + corpus := editProbeCorpus() + corpus.EditOutput = "cli" + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step, state.record = plan.Steps[0], newEditProbeRecord(corpus, plan.Steps[0]) + origin := time.Now() + state.started, state.acknowledged = origin.Add(time.Millisecond), origin.Add(2*time.Millisecond) + p := &privateRun{Root: dir, Config: filepath.Join(dir, "config.yaml"), Binary: os.Args[0], Env: append(os.Environ(), env...), Repos: []privateRepo{{Corpus: corpus, Prefix: "owned", Path: filepath.Join(dir, "clone")}}} + run := &editProbeRuntime{options: runOptions{Deadline: deadline}, origin: origin, plan: plan, p: p, phase: &Phase{}} + return run, state, editProbeOracles(corpus, plan.Steps[0])[1] +} + +func TestEditProbeCLIProbeOwnsProcessAndDecodesStdout(t *testing.T) { + argvFile := filepath.Join(t.TempDir(), "argv.json") + marker := "GortexEditProbe0000000000000001N000" + answer := `{"nodes":[{"id":"owned/sample.go::` + marker + `"}],"truncated":false,"freshness":"fresh"}` + run, state, oracle := editProbeCLIFixture(t, 10*time.Second, "LOAD_EDIT_PROBE_CLI_FIXTURE=reply", "LOAD_EDIT_PROBE_CLI_ARGV="+argvFile, "LOAD_EDIT_PROBE_CLI_STDOUT="+answer) + if oracle.name != "summary" || state.step.Marker != marker { + t.Fatal(oracle.name, state.step.Marker) + } + data, identity, err := run.performCLI(t.Context(), state, oracle.call, "oracle:summary", 11) + if err != nil { + t.Fatal(err) + } + if err := run.observeProbe(state, oracle, identity.WriteStarted, data, identity, err, false); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(argvFile) + if err != nil { + t.Fatal(err) + } + var argv []string + if err := json.Unmarshal(raw, &argv); err != nil { + t.Fatal(err) + } + if len(argv) != 11 { + t.Fatalf("CLI argv %q", argv) + } + want := []string{"--config", run.p.Config, "call", "read", "--json", argv[5], "--format", "json", "--index", run.p.Repos[0].Path, "--quiet"} + lowered, err := json.Marshal(dispatchArgs(oracle.call, run.p.Repos[0])) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(argv, want) || argv[5] != string(lowered) { + t.Fatalf("CLI argv %q, want the lowered facade request %v", argv, dispatchArgs(oracle.call, run.p.Repos[0])) + } + if state.record.FirstCorrect["summary"].Value == nil || state.firstProbeWrite.IsZero() || len(run.phase.Attempts) != 1 || run.phase.Attempts[0].Tool != "cli:read.summary" || run.phase.Attempts[0].Role != "probe" || run.phase.Attempts[0].Outcome != "success" { + t.Fatalf("CLI answer not owned or credited: %+v %+v", state.record.FirstCorrect, run.phase.Attempts) + } + coverage := editProbeOutputCoverage(run.plan.Mode, run.modeCounts) + if cli := coverage["cli"]; cli.State != "selected" || cli.ProbeAnswers != 1 || cli.ClaimsFresh != 1 || cli.StaleClaimedFresh.Value == nil || *cli.StaleClaimedFresh.Value != 0 || coverage["json"].ProbeAnswers != 0 { + t.Fatalf("CLI answers counted as another mode: %+v", coverage) + } + // A tool refusal is only a nonzero exit and stderr text: a probe error. + run.p.Env = append(run.p.Env, "LOAD_EDIT_PROBE_CLI_EXIT=1") + data, identity, err = run.performCLI(t.Context(), state, oracle.call, "oracle:summary", 11) + if err == nil || !strings.Contains(err.Error(), "fixture stderr") || run.phase.Attempts[1].Outcome != "error" { + t.Fatalf("failed CLI call: %v %+v", err, run.phase.Attempts) + } + if err := run.observeProbe(state, oracle, identity.WriteStarted, data, identity, err, false); err != nil { + t.Fatal(err) + } + if state.record.ProbeErrors != 1 { + t.Fatalf("CLI exit not a probe error: %+v", state.record) + } +} + +func TestEditProbeCLIRefusesProseWrapperAndUnsafeNumbers(t *testing.T) { + if body, err := decodeEditProbeOutput([]byte("{\n \"total\": 9007199254740991,\n \"score\": 0.5\n}\n"), "cli", "read.summary"); err != nil || body["total"] != json.Number("9007199254740991") { + t.Fatalf("exact CLI JSON refused: %v %v", body, err) + } + for name, stdout := range map[string]string{ + "prose wrapper": `{"tool":"explore","format":"text","text":"# Answer"}`, + "unsafe integer": `{"total":9007199254740993}`, + "unsafe negative": `{"total":-9007199254740992}`, + "out of range": `{"total":1e400}`, + "nested unsafe": `{"nodes":[{"line":18014398509481984}]}`, + "array": `[1]`, + "two values": `{} {}`, + "empty": ``, + "TOON passed as -": "nodes[1]{id}:\n a\n", + "MCP envelope text": `{"content":[{"type":"text","text":"{}"}]}x`, + } { + if _, err := decodeEditProbeOutput([]byte(stdout), "cli", "read.summary"); err == nil { + t.Fatal("accepted", name) + } + } +} + +func TestEditProbeCLIDeadlineKillsProcessGroupAndCensors(t *testing.T) { + run, state, oracle := editProbeCLIFixture(t, 300*time.Millisecond, "LOAD_EDIT_PROBE_CLI_FIXTURE=hang") + started := time.Now() + _, _, err := run.performCLI(t.Context(), state, oracle.call, "oracle:summary", 11) + if !errors.Is(err, context.DeadlineExceeded) || time.Since(started) > 10*time.Second || run.phase.Attempts[0].Outcome != "product_deadline" { + t.Fatalf("hung CLI probe not killed at its deadline: %v after %s %+v", err, time.Since(started), run.phase.Attempts) + } + ctx, cancel := context.WithCancel(t.Context()) + time.AfterFunc(100*time.Millisecond, cancel) + run.options.Deadline = time.Minute + _, _, err = run.performCLI(ctx, state, oracle.call, "oracle:summary", 11) + if !errors.Is(err, context.Canceled) || run.phase.Attempts[1].Outcome != "phase_censored" { + t.Fatalf("phase end did not censor the CLI probe: %v %+v", err, run.phase.Attempts) + } +} + +func TestEditProbeCLIModeRefusesAbsenceKinds(t *testing.T) { + deleted := editProbeRenameCorpus(t, "go", 100) + deleted.EditKind = "delete" + for _, c := range []Corpus{editProbeRenameCorpus(t, "go", 100), deleted, moveProbeCorpus(42)} { + c.EditOutput = "cli" + if _, err := makeEditProbePlan(c, 42, 100); err == nil || !strings.Contains(err.Error(), "CLI probes cannot check "+c.EditKind) { + t.Fatalf("CLI admitted %s: %v", c.EditKind, err) + } + } + for _, c := range []Corpus{editProbeCorpus(), editProbeBodyCorpus(t, "go", "modify_body", 100)} { + baseline, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + operations := editProbeOperations(baseline, c) + c.EditOutput = "cli" + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + // The edit, its readbacks and every oracle request stay the JSON plan's. + if !reflect.DeepEqual(plan.Steps, baseline.Steps) || !reflect.DeepEqual(editProbeOperations(plan, c), operations) { + t.Fatal("CLI mode changed the offered requests") + } + if !strings.Contains(strings.Join(newEditProbeReport(plan, c, true).Limitations, "\n"), "CLI probe cadence is bounded by process start") { + t.Fatal("CLI limitations missing") + } + } +} diff --git a/bench/load/edit_probe_compact_test.go b/bench/load/edit_probe_compact_test.go new file mode 100644 index 000000000..21d572342 --- /dev/null +++ b/bench/load/edit_probe_compact_test.go @@ -0,0 +1,207 @@ +package main + +import ( + "encoding/json" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +func TestEditProbeCompactRowsRequireExactGrammarAndRefuseJSON(t *testing.T) { + body, err := decodeEditProbeOutput(outputWire("function Added fixture/main.go:5\nfunction Example fixture/main.go:3\n... truncated (9 total)\nedges: 2 of 4 total, 2 EXTRACTED\nconcurrency fixture/main.go::Added: sync_guarded\n", nil), "compact", "relations.callers") + if err != nil { + t.Fatal(err) + } + answer := body[editProbeCompactKey].(editProbeCompactAnswer) + if len(body) != 1 || len(answer.rows) != 2 || !answer.truncated || answer.edges != 2 || answer.rows[0] != (editProbeCompactRow{Kind: "function", Qualified: "Added", Path: "fixture/main.go", Line: 5}) { + t.Fatalf("compact subgraph rows: %+v", answer) + } + if editProbeClaimsFresh(body) { + t.Fatal("compact rows cannot claim freshness") + } + search, err := decodeEditProbeOutput(outputWire("method Marker fixture/main.go:9 (in Type)\n", nil), "compact", "search.symbols") + if err != nil || search[editProbeCompactKey].(editProbeCompactAnswer).rows[0].Owner != "Type" { + t.Fatalf("symbol search owner suffix: %v %+v", err, search) + } + empty, err := decodeEditProbeOutput(outputWire("", nil), "compact", "search.symbols") + if err != nil || len(empty[editProbeCompactKey].(editProbeCompactAnswer).rows) != 0 { + t.Fatalf("an empty compact answer is zero rows: %v", err) + } + if _, err := decodeEditProbeOutput(outputWire("", nil), "json", "search.symbols"); err == nil { + t.Fatal("empty JSON payload accepted") + } + for name, refused := range map[string]struct{ text, operation string }{ + "JSON body": {`{"nodes":[{"id":"fixture/main.go::Added"}]}`, "read.summary"}, + "missing newline": {"function Added fixture/main.go:5", "read.summary"}, + "unknown line": {"function Added fixture/main.go:5\nfreshness: fresh\n", "read.summary"}, + "row after footer": {"edges: 1 total\nfunction Added fixture/main.go:5\n", "trace.call_chain"}, + "two footers": {"function Added fixture/main.go:5\nedges: 1 total\nedges: 1 total\n", "trace.call_chain"}, + "subgraph owner": {"function Added fixture/main.go:5 (in Type)\n", "relations.callers"}, + "search footer": {"function Added fixture/main.go:5\nedges: 1 total\n", "search.symbols"}, + "zero line": {"function Added fixture/main.go:0\n", "read.summary"}, + "no compact producer": {"function Added fixture/main.go:5\n", "read.source"}, + "no compact producer2": {"function Added fixture/main.go:5\n", "search.text"}, + } { + if _, err := decodeEditProbeOutput(outputWire(refused.text, nil), "compact", refused.operation); err == nil { + t.Fatal("accepted", name) + } + } + failed := json.RawMessage(`{"isError":true,"content":[{"type":"text","text":"function Added fixture/main.go:5\n"}]}`) + if _, err := decodeEditProbeOutput(failed, "compact", "read.summary"); err == nil { + t.Fatal("tool failure counted as compact rows") + } + // A JSON response cannot supply the typed harness-local answer. + forged, err := decodeEditProbeOutput(outputWire(`{"\u0000load.compact.rows":{"rows":[]}}`, nil), "json", "read.summary") + if err != nil { + t.Fatal(err) + } + if _, ok := forged[editProbeCompactKey].(editProbeCompactAnswer); ok { + t.Fatal("JSON forged compact rows") + } +} + +func TestEditProbeCompactPresenceBindsMarkerPathAndCallee(t *testing.T) { + answer := func(operation string, edges int, rows ...editProbeCompactRow) map[string]any { + return map[string]any{editProbeCompactKey: editProbeCompactAnswer{operation: operation, rows: rows, edges: edges}} + } + for _, c := range []Corpus{editProbeCorpus(), editProbeCrossFileCorpus(t, "go")} { + c.EditOutput = "compact" + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + calleeFile := c.File + if c.EditKind == "add_crossfile_call" { + calleeFile, _, _ = editProbeCrossFileCallee(c) + } + marker := editProbeCompactRow{Kind: "function", Qualified: step.Marker, Path: "owned/" + c.File, Line: 9} + callee := editProbeCompactRow{Kind: "function", Qualified: plan.Callee, Path: "owned/" + calleeFile, Line: 3} + wrongPath, wrongKind, method := marker, marker, marker + wrongPath.Path, wrongKind.Kind, method.Qualified = "owned/other.go", "variable", "Type."+step.Marker + owned := marker + owned.Owner = "Type" + foreign := callee + foreign.Path = "owned/other" + filepath.Ext(calleeFile) + for _, check := range []struct { + oracle string + body map[string]any + want bool + }{ + {"summary", answer("read.summary", 1, callee, marker), true}, + {"summary", answer("read.summary", 1, wrongPath), false}, + {"summary", answer("read.summary", 1, wrongKind), false}, + {"summary", answer("read.summary", 1, method), false}, + {"summary", answer("search.symbols", -1, marker), false}, + {"search_visible", answer("search.symbols", -1, marker), true}, + {"search_visible", answer("search.symbols", -1, owned), false}, + {"callers", answer("relations.callers", 2, marker), true}, + {"callers", answer("relations.callers", -1, marker), false}, + {"callers", answer("relations.callers", 0, marker), false}, + {"call_chain", answer("trace.call_chain", 1, marker, callee), true}, + {"call_chain", answer("trace.call_chain", 1, marker), false}, + {"call_chain", answer("trace.call_chain", 1, marker, foreign), false}, + {"source", answer("read.summary", 1, marker), false}, + {"text", answer("read.summary", 1, marker), false}, + } { + if got := editProbeCorrect(check.oracle, check.body, c, step, plan.Callee); got != check.want { + t.Fatalf("%s %s: got %v want %v (%+v)", c.EditKind, check.oracle, got, check.want, check.body) + } + } + } +} + +func TestEditProbeCompactModeKeepsUnadvertisedOperationsJSON(t *testing.T) { + c := editProbeCorpus() + baseline, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + c.EditOutput = "compact" + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Mode != "compact" || len(plan.Steps) != len(baseline.Steps) { + t.Fatal(plan.Mode) + } + jsonOutput := map[string]any{"format": "json", "max_bytes": 0} + compactOutput := map[string]any{"format": "json", "compact": true, "max_bytes": 0} + for i, step := range plan.Steps { + if !reflect.DeepEqual(step, baseline.Steps[i]) { + t.Fatal("compact changed the offered edit, its schedule or identity") + } + for _, oracle := range editProbeOracles(c, step) { + want, output := "json", jsonOutput + if oracle.name == "summary" || oracle.name == "search_visible" || oracle.name == "callers" || oracle.name == "call_chain" { + want, output = "compact", compactOutput + } + if got := editProbeRequestOutput(oracle.call); got != want { + t.Fatalf("%s requested %s", oracle.name, got) + } + if got := dispatchArgs(oracle.call, privateRepo{Prefix: "owned"})["output"]; !reflect.DeepEqual(got, output) { + t.Fatalf("%s dispatched %v", oracle.name, got) + } + } + } + if got := dispatchArgs(request{Tool: "read", Operation: "file", Arguments: map[string]any{"operation": "file"}}, privateRepo{Prefix: "owned"})["output"]; !reflect.DeepEqual(got, jsonOutput) { + t.Fatalf("default dispatched output changed: %v", got) + } + for _, invalid := range []map[string]any{{"compact": false}, {"format": "toon", "compact": true}, {"compact": "true"}} { + if got := editProbeRequestOutput(request{Arguments: map[string]any{"output": invalid}}); got != "invalid" { + t.Fatalf("%v read as %s", invalid, got) + } + } + // Compact answers count under compact, whose freshness is unobservable: + // its stale claimed-fresh count is unavailable, never a measured zero. + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step, state.record = plan.Steps[0], newEditProbeRecord(c, plan.Steps[0]) + origin := time.Now() + state.started, state.acknowledged = origin.Add(time.Millisecond), origin.Add(2*time.Millisecond) + run := &editProbeRuntime{origin: origin, plan: plan, p: &privateRun{Repos: []privateRepo{{Corpus: c, Prefix: "owned"}}}} + identity := wireCallIdentity{WriteStarted: origin.Add(3 * time.Millisecond), Finished: origin.Add(4 * time.Millisecond)} + for _, oracle := range editProbeOracles(c, plan.Steps[0]) { + if oracle.name != "summary" { + continue + } + row := "function " + plan.Steps[0].Marker + " owned/" + c.File + ":9\n" + if err := run.observeProbe(state, oracle, identity.WriteStarted, outputWire(row, nil), identity, nil, false); err != nil { + t.Fatal(err) + } + if err := run.observeProbe(state, oracle, identity.WriteStarted, outputWire(`{"freshness":"fresh"}`, nil), identity, nil, false); err != nil { + t.Fatal(err) + } + } + if state.record.FirstCorrect["summary"].Value == nil || state.record.StaleClaimedFresh { + t.Fatalf("compact presence not credited, or a JSON answer to a compact request claimed freshness: %+v", state.record) + } + coverage := editProbeOutputCoverage(plan.Mode, run.modeCounts)["compact"] + if coverage.State != "selected" || coverage.ProbeAnswers != 2 || coverage.DecodeFailures != 1 || coverage.ClaimsFreshObservable || coverage.StaleClaimedFresh.Value != nil || coverage.StaleClaimedFresh.Reason == "" { + t.Fatalf("compact coverage: %+v", coverage) + } + report := newEditProbeReport(plan, c, true) + if !strings.Contains(strings.Join(report.Limitations, "\n"), "stale-claimed-fresh is unavailable") { + t.Fatal("compact limitations missing", report.Limitations) + } +} + +func TestEditProbeCompactRefusesAbsenceAndBodyKinds(t *testing.T) { + leaf := editProbeSymbolCorpus(t, "go") + leaf.EditKind = "append_leaf" + leaf.EditTemplate = strings.ReplaceAll(leaf.EditTemplate, "Example()", "7") + deleted := editProbeRenameCorpus(t, "go", 100) + deleted.EditKind = "delete" + for _, c := range []Corpus{leaf, editProbeRenameCorpus(t, "go", 100), deleted, editProbeBodyCorpus(t, "go", "modify_body", 100), editProbeSignatureCorpus(t, "go", 100), moveProbeCorpus(42)} { + if _, err := makeEditProbePlan(c, 42, 100); err != nil { + t.Fatal("JSON fixture invalid", c.EditKind, err) + } + c.EditOutput = "compact" + if _, err := makeEditProbePlan(c, 42, 100); err == nil || !strings.Contains(err.Error(), "compact output cannot check "+c.EditKind) { + t.Fatalf("compact admitted %s: %v", c.EditKind, err) + } + } +} diff --git a/bench/load/edit_probe_crossfile.go b/bench/load/edit_probe_crossfile.go new file mode 100644 index 000000000..3466856ef --- /dev/null +++ b/bench/load/edit_probe_crossfile.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" +) + +func editProbeCrossFileCallee(c Corpus) (file, name string, err error) { + sep := strings.LastIndex(c.EditCallee, "::") + if sep < 1 { + return "", "", errors.New("crossfile fixture requires exact edit_callee") + } + file, name = c.EditCallee[:sep], c.EditCallee[sep+2:] + if filepath.Base(c.File) != c.File || filepath.Base(file) != file || file == c.File || filepath.Ext(file) != filepath.Ext(c.File) || !simpleSymbolName(name) { + return "", "", errors.New("crossfile fixture requires distinct same-language root file and plain callee") + } + base := strings.TrimSuffix(file, filepath.Ext(file)) + if !simpleSymbolName(base) { + return "", "", errors.New("crossfile fixture module basename must be a plain identifier") + } + return file, name, nil +} + +func simpleSymbolName(s string) bool { + if s == "" { + return false + } + for i, r := range s { + if r == '_' || r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || i > 0 && r >= '0' && r <= '9' { + continue + } + return false + } + return true +} + +func editProbeCrossFileImport(c Corpus) string { + file, name, err := editProbeCrossFileCallee(c) + if err != nil { + return "" + } + module := strings.TrimSuffix(file, filepath.Ext(file)) + switch filepath.Ext(c.File) { + case ".ts": + return "import { " + name + " } from \"./" + module + "\";\n" + case ".py": + return "from " + module + " import " + name + "\n" + } + return "" +} +func editProbeCrossFilePlain(c Corpus, source string) string { + if c.EditKind != "add_crossfile_call" { + return source + } + prefix := editProbeCrossFileImport(c) + return strings.TrimPrefix(source, prefix) +} + +func (r *editProbeRuntime) admitCrossFileCallee(ctx context.Context) error { + c := r.p.Repos[0].Corpus + file, name, err := editProbeCrossFileCallee(c) + if err != nil { + return err + } + prefix := editProbeCrossFileImport(c) + if prefix != "" && !strings.HasPrefix(string(r.original), prefix) { + return errors.New("crossfile fixture missing exact explicit import") + } + content, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, file)) + if err != nil { + return err + } + if !editProbeLeafSource(file, string(content), name) { + return errors.New("crossfile fixture callee must be a real numeric leaf") + } + req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": c.EditCallee}, "options": map[string]any{"context_lines": 0}}} + raw, identity, err := r.editors[0].callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, r.p.Repos[0])}) + if err != nil { + return err + } + if err := checkToolResult(raw); err != nil { + return err + } + proof := c + proof.File = file + proof.Symbol = c.EditCallee + proof.EditKind = "" + if !editProbePresenceProof(raw, proof, c.EditCallee) { + return errors.New("crossfile callee public source admission failed") + } + record, err := json.Marshal(map[string]any{"symbol": c.EditCallee, "wire_id": identity.ID, "write_started": identity.WriteStarted, "finished": identity.Finished, "payload": raw}) + if err != nil { + return err + } + if r.phase.Gate == nil { + r.phase.Gate = make(map[string]json.RawMessage) + } + r.phase.Gate["edit_probe_crossfile_callee_admission"] = record + return nil +} diff --git a/bench/load/edit_probe_crossfile_test.go b/bench/load/edit_probe_crossfile_test.go new file mode 100644 index 000000000..6b115befa --- /dev/null +++ b/bench/load/edit_probe_crossfile_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "strings" + "testing" +) + +func editProbeCrossFileCorpus(t *testing.T, language string) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, language) + c.EditKind = "add_crossfile_call" + ext := c.File[strings.LastIndex(c.File, "."):] + c.EditCallee = "hub" + ext + "::Hub" + c.EditTemplate = strings.ReplaceAll(c.EditTemplate, "Example()", "Hub()") + c.EditInitialContent = editProbeCrossFileImport(c) + c.EditInitialContent + return c +} +func TestEditProbeCrossFileCallsBindActualOtherFile(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, tool := range []string{"edit.file", "edit.write", "edit.symbol", "edit.batch"} { + t.Run(language+"/"+tool, func(t *testing.T) { + c := editProbeCrossFileCorpus(t, language) + c.EditTool = tool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Callee != "Hub" || len(plan.Steps) != 100 { + t.Fatal("crossfile plan identity wrong") + } + step := plan.Steps[0] + oracles := editProbeOracles(c, step) + if oracles[5].call.Arguments["target"].(map[string]any)["symbol"] != c.EditCallee { + t.Fatal("callers queried original same-file anchor") + } + hubFile, _, _ := editProbeCrossFileCallee(c) + edge := map[string]any{"from": "owned/" + c.File + "::" + step.Marker, "to": "owned/" + hubFile + "::Hub", "kind": "calls"} + if !editProbeCorrect("call_chain", map[string]any{"edges": []any{edge}}, c, step, plan.Callee) { + t.Fatal("actual crossfile call refused") + } + edge["to"] = "owned/" + c.File + "::Hub" + if editProbeCorrect("call_chain", map[string]any{"edges": []any{edge}}, c, step, plan.Callee) { + t.Fatal("same-file homonym borrowed as crossfile evidence") + } + source := editProbeCrossFilePlain(c, c.EditInitialContent) + strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + if !editProbeCorrect("file", map[string]any{"path": "owned/" + c.File, "content": editProbeCrossFileImport(c) + source}, c, step, plan.Callee) { + t.Fatal("actual explicit import/call file refused") + } + }) + } + } +} +func TestEditProbeCrossFileRefusesAmbiguousImportAndCallee(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + base := editProbeCrossFileCorpus(t, language) + for _, variant := range []string{"same_file", "outside", "nested_edited", "wrong_language", "missing", "wrong_import"} { + t.Run(variant, func(t *testing.T) { + c := base + switch variant { + case "same_file": + c.EditCallee = c.File + "::Hub" + case "nested_edited": + c.File = "nested/" + c.File + c.Symbol = "nested/" + c.Symbol + case "outside": + c.EditCallee = "../hub.go::Hub" + case "wrong_language": + c.EditCallee = "hub.rs::Hub" + case "missing": + c.EditCallee = "" + case "wrong_import": + if language == "go" { + c.EditCallee = "bad-module.go::Hub" + } else { + c.EditInitialContent = strings.Replace(c.EditInitialContent, editProbeCrossFileImport(c), "", 1) + } + } + if _, err := makeEditProbePlan(c, 42, 1); err == nil { + t.Fatal("unsupported crossfile fixture admitted") + } + }) + } + }) + } +} diff --git a/bench/load/edit_probe_delete.go b/bench/load/edit_probe_delete.go new file mode 100644 index 000000000..b6643dce2 --- /dev/null +++ b/bench/load/edit_probe_delete.go @@ -0,0 +1,146 @@ +package main + +import ( + "encoding/json" + "strings" +) + +func editProbeDeleteOracles(c Corpus, step editProbeStep) []editProbeOracle { + old := map[string]any{"symbol": step.BeforeSymbol} + file := map[string]any{"file": c.File} + name := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + return []editProbeOracle{ + {"source", request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: step.BeforeSymbol, Arguments: map[string]any{"operation": "source", "target": old}}}, + {"summary", request{Tool: "read", Operation: "summary", Arguments: map[string]any{"operation": "summary", "target": file}}}, + {"editing_context", request{Tool: "read", Operation: "editing_context", Arguments: map[string]any{"operation": "editing_context", "target": file}}}, + {"search_visible", request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": name, "options": map[string]any{"query_class": "symbol", "expand": "off", "limit": 1000, "max_per_file": 0}}}}, + {"text", request{Tool: "search", Operation: "text", Arguments: map[string]any{"operation": "text", "query": name}}}, + {"callers", request{Tool: "relations", Operation: "callers", Arguments: map[string]any{"operation": "callers", "target": map[string]any{"symbol": c.Symbol}, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + {"call_chain", request{Tool: "trace", Operation: "call_chain", Arguments: map[string]any{"operation": "call_chain", "target": old, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + } +} + +func editProbeCompleteCount(body map[string]any, key, total string) ([]any, bool) { + value, present := body[key] + rows, ok := value.([]any) + // Actual producer nil slices serialize as explicit null. Only a present + // key plus its exact zero total can establish the empty collection. + if present && value == nil { + rows = []any{} + ok = true + } + if _, declared := body[total]; !declared && present && ok && editProbeTOONCompleteRows(body, key, total) { + return rows, true + } + count, err := strconvEditProbeUint(body[total]) + return rows, present && ok && err == nil && count == uint64(len(rows)) +} + +// Deletion has explicit absence oracles. A generic empty object, partial +// relation page or missing collection can never establish removal. Collections +// which should retain the stable callee must prove its concrete node identity. +func editProbeDeleteCorrect(oracle string, body map[string]any, c Corpus, step editProbeStep, callee string) bool { + old := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + untruncated := func() bool { v, ok := body["truncated"].(bool); return ok && !v } + cleanNodes := func(rows []any) bool { + survivor := false + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if editProbeSourceID(id, c.File, old) { + return false + } + if editProbeSourceID(id, c.File, callee) { + survivor = true + } + } + return survivor + } + cleanEdges := func(rows []any) bool { + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + from, fok := row["from"].(string) + to, tok := row["to"].(string) + if !fok || !tok || editProbeSourceID(from, c.File, old) || editProbeSourceID(to, c.File, old) { + return false + } + } + return true + } + switch oracle { + case "source": + return false // only the exact typed producer refusal qualifies + case "file": + source, _ := body["content"].(string) + path, _ := body["path"].(string) + if !editProbeFixturePath(path, c.File) || strings.Contains(source, old) { + return false + } + // The complete public file read must retain a real stable declaration; + // comments or an empty successful result cannot witness deletion. + payload, _ := json.Marshal(map[string]any{"id": c.Symbol, "file_path": c.File, "name": callee, "source": source}) + raw, _ := json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": string(payload)}}}) + return editProbePresenceProof(raw, c, c.Symbol) + case "summary", "callers": + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + return untruncated() && nok && eok && cleanNodes(nodes) && cleanEdges(edges) + case "editing_context": + nodes, ok := body["defines"].([]any) + file, fok := body["file"].(map[string]any) + id, _ := file["id"].(string) + if !ok || !fok || id != c.File && !strings.HasSuffix(id, "/"+c.File) { + return false + } + if truncated, exists := body["truncated"]; exists && truncated != false { + return false + } + return cleanNodes(nodes) + case "search_visible": + rows, ok := editProbeCompleteCount(body, "results", "total") + if !ok || !untruncated() || body["query_class"] != "symbol" || body["next_cursor"] != nil && body["next_cursor"] != "" { + return false + } + for _, key := range []string{"_truncated_by_budget", "_truncated_by_limit"} { + if value, present := body[key]; present && value != false { + return false + } + } + // Search may publish a complete decomposition/fallback population. Its + // unrelated rows are not the deleted symbol and must not become a false + // failure or a fake zero-result assertion. + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, ok := row["id"].(string) + if !ok || id == "" || editProbeSourceID(id, c.File, old) { + return false + } + } + return true + case "text": + rows, ok := editProbeCompleteCount(body, "matches", "count") + if body["query"] != old || !ok || len(rows) != 0 { + return false + } + for _, key := range []string{"truncated", "_truncated_by_limit"} { + if v, exists := body[key]; exists && v != false { + return false + } + } + return true + case "call_chain": + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + return untruncated() && nok && eok && len(nodes) == 0 && len(edges) == 0 + } + return false +} diff --git a/bench/load/edit_probe_delete_test.go b/bench/load/edit_probe_delete_test.go new file mode 100644 index 000000000..a02af057c --- /dev/null +++ b/bench/load/edit_probe_delete_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestEditProbeDeleteUsesRealEmptySpliceAndSeparateAbsence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeRenameCorpus(t, language, 100) + c.EditKind = "delete" + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + if !step.Deleted || step.Request.Arguments["replacement"] != "" || strings.Count(c.EditInitialContent, step.Request.Arguments["match"].(string)) != 1 { + t.Fatal("delete did not use actual exact empty symbol splice") + } + oracles := editProbeOracles(c, step) + if len(oracles) != 7 || oracles[0].call.ExpectedAbsentSymbol != step.BeforeSymbol { + t.Fatal("delete source absence not explicit") + } + if oracles[6].call.Arguments["target"].(map[string]any)["symbol"] != step.BeforeSymbol { + t.Fatal("deleted call-chain target changed") + } + } + }) + } +} + +func TestEditProbeDeleteRequiresCompleteSurvivorAndRelationEvidence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeRenameCorpus(t, language, 1) + c.EditKind = "delete" + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + old := step.Marker + "Before" + after := strings.Replace(c.EditInitialContent, step.Request.Arguments["match"].(string), "", 1) + nodes := []any{map[string]any{"id": "owned/" + c.Symbol}} + for _, oracle := range []string{"file", "summary", "editing_context", "search_visible", "text", "callers", "call_chain"} { + t.Run(oracle, func(t *testing.T) { + body := map[string]any{"content": after, "path": "owned/" + c.File, "nodes": nodes, "edges": []any{}, "total_nodes": json.Number("1"), "total_edges": json.Number("0"), "truncated": false, "defines": nodes, "file": map[string]any{"id": "owned/" + c.File}, "results": []any{}, "total": json.Number("0"), "query_class": "symbol", "matches": []any{}, "count": json.Number("0"), "query": old} + if oracle == "call_chain" { + body["nodes"] = []any{} + body["total_nodes"] = json.Number("0") + } + if !editProbeCorrect(oracle, body, c, step, plan.Callee) { + t.Fatalf("complete deletion proof refused: %v", body) + } + for _, bad := range []string{"empty", "truncated", "stale_old", "missing_collection"} { + t.Run(bad, func(t *testing.T) { + raw, _ := json.Marshal(body) + var invalid map[string]any + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + _ = decoder.Decode(&invalid) + switch bad { + case "empty": + invalid = map[string]any{} + case "truncated": + invalid["truncated"] = true + invalid["_truncated_by_limit"] = true + if oracle == "file" { + invalid["content"] = "" + } + case "stale_old": + invalid["content"] = c.EditInitialContent + invalid["nodes"] = append([]any{map[string]any{"id": "owned/" + step.BeforeSymbol}}, nodes...) + invalid["total_nodes"] = json.Number("2") + invalid["defines"] = invalid["nodes"] + invalid["results"] = []any{map[string]any{"id": "owned/" + step.BeforeSymbol}} + invalid["total"] = json.Number("1") + invalid["matches"] = []any{map[string]any{"text": old}} + invalid["count"] = json.Number("1") + case "missing_collection": + for _, key := range []string{"nodes", "defines", "results", "matches", "content"} { + delete(invalid, key) + } + } + if editProbeCorrect(oracle, invalid, c, step, plan.Callee) { + t.Fatalf("%s falsely established deletion", bad) + } + }) + } + }) + } + }) + } +} + +func TestEditProbeCompleteCountAllowsOnlyExplicitProducerNullWithExactZero(t *testing.T) { + for _, test := range []struct { + name string + body map[string]any + want bool + }{ + {"explicit_null", map[string]any{"nodes": nil, "total_nodes": json.Number("0")}, true}, + {"empty_array", map[string]any{"nodes": []any{}, "total_nodes": json.Number("0")}, true}, + {"missing_collection", map[string]any{"total_nodes": json.Number("0")}, false}, + {"missing_total", map[string]any{"nodes": nil}, false}, + {"null_nonzero", map[string]any{"nodes": nil, "total_nodes": json.Number("1")}, false}, + {"wrong_type", map[string]any{"nodes": "", "total_nodes": json.Number("0")}, false}, + } { + t.Run(test.name, func(t *testing.T) { + _, got := editProbeCompleteCount(test.body, "nodes", "total_nodes") + if got != test.want { + t.Fatalf("complete=%v", got) + } + }) + } +} diff --git a/bench/load/edit_probe_evidence.go b/bench/load/edit_probe_evidence.go new file mode 100644 index 000000000..719876e10 --- /dev/null +++ b/bench/load/edit_probe_evidence.go @@ -0,0 +1,620 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "math" + "math/big" + "path/filepath" + "strings" +) + +type editProbeOracle struct { + name string + call request +} + +func editProbeOracles(c Corpus, step editProbeStep) []editProbeOracle { + if step.ExpectedRefusal { + return nil // A refused relocation makes no change for any probe to observe. + } + if step.File != "" { + return editProbeLifecycleOracles(c, step) + } + if step.Relocated { + return editProbeOutputOracles(c, editProbeMoveOracles(c, step)) + } + if step.TargetSymbol != "" { + return editProbeOutputOracles(c, editProbeBodyOracles(c, step)) + } + if step.Deleted { + return editProbeOutputOracles(c, editProbeDeleteOracles(c, step)) + } + marker := map[string]any{"symbol": c.File + "::" + step.Marker} + file := map[string]any{"file": c.File} + oracles := []editProbeOracle{ + {"source", request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": marker}}}, + {"summary", request{Tool: "read", Operation: "summary", Arguments: map[string]any{"operation": "summary", "target": file}}}, + {"editing_context", request{Tool: "read", Operation: "editing_context", Arguments: map[string]any{"operation": "editing_context", "target": file}}}, + {"search_visible", request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": step.Marker}}}, + {"text", request{Tool: "search", Operation: "text", Arguments: map[string]any{"operation": "text", "query": step.Marker}}}, + {"callers", request{Tool: "relations", Operation: "callers", Arguments: map[string]any{"operation": "callers", "target": map[string]any{"symbol": c.Symbol}, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + {"call_chain", request{Tool: "trace", Operation: "call_chain", Arguments: map[string]any{"operation": "call_chain", "target": marker, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + } + if c.EditKind == "add_crossfile_call" { + oracles[5].call.Arguments["target"] = map[string]any{"symbol": c.EditCallee} + } + if c.EditKind == "append_leaf" { + oracles[5].call.Arguments["target"] = marker + } + if step.BeforeSymbol != "" { + oracles = append(oracles, editProbeOracle{name: "old_source_absent", call: request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: step.BeforeSymbol, Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": step.BeforeSymbol}}}}) + } + return editProbeOutputOracles(c, oracles) +} + +func decodeEditProbeResult(raw json.RawMessage) (map[string]any, error) { + payload, err := textPayload(raw, "content") + if err != nil { + return nil, err + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.UseNumber() + var value map[string]any + if err := decoder.Decode(&value); err != nil { + return nil, err + } + return value, nil +} + +func editProbeSourceCall(source, marker, callee string) bool { + file, err := parser.ParseFile(token.NewFileSet(), "probe.go", source, 0) + if err != nil { + file, err = parser.ParseFile(token.NewFileSet(), "probe.go", "package probe\n"+source, 0) + } + if err != nil { + return false + } + found := false + for _, decl := range file.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == marker && fn.Body != nil { + ast.Inspect(fn.Body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + if target, ok := call.Fun.(*ast.Ident); ok && target.Name == callee { + found = true + } + } + return true + }) + } + } + return found +} + +func editProbeSourceID(id, file, name string) bool { + return !strings.HasPrefix(id, "ext::") && (id == file+"::"+name || strings.HasSuffix(id, "/"+file+"::"+name)) +} + +func editProbeCorrect(oracle string, body map[string]any, c Corpus, step editProbeStep, callee string) bool { + if step.File != "" { + return editProbeLifecycleCorrect(oracle, body, c, step, callee) + } + if answer, ok := body[editProbeCompactKey].(editProbeCompactAnswer); ok { + return !step.Relocated && step.BeforeSymbol == "" && editProbeCompactCorrect(oracle, answer, c, step, callee) + } + if step.Relocated { + return editProbeMoveCorrect(oracle, body, c, step, callee) + } + if step.TargetSymbol != "" { + return editProbeBodyCorrect(oracle, body, c, step, callee) + } + if step.Deleted { + return editProbeDeleteCorrect(oracle, body, c, step, callee) + } + // Collections are producer-specific and top-level. Nested guidance/examples + // must never become proof that the requested graph fact was returned. + collection := func(key string, accept func(map[string]any) bool) bool { + rows, _ := body[key].([]any) + for _, value := range rows { + if row, ok := value.(map[string]any); ok && accept(row) { + return true + } + } + return false + } + switch oracle { + case "source": + source, _ := body["source"].(string) + if filepath.Ext(c.File) == ".ts" || filepath.Ext(c.File) == ".py" || (c.EditTool == "edit.write" || c.EditTool == "edit.batch" || c.EditTool == "edit.symbol" || c.EditKind != "") { + id, _ := body["id"].(string) + path, _ := body["file_path"].(string) + if !editProbeSourceID(id, c.File, step.Marker) || !editProbeFixturePath(path, c.File) { + return false + } + } + if c.EditKind == "append_leaf" { + return body["name"] == step.Marker && editProbeLeafSource(c.File, source, step.Marker) + } + return body["name"] == step.Marker && editProbeFixtureSource(c.File, editProbeCrossFilePlain(c, source), step.Marker, callee) + case "file": + source, _ := body["content"].(string) + if filepath.Ext(c.File) == ".ts" || filepath.Ext(c.File) == ".py" || (c.EditTool == "edit.write" || c.EditTool == "edit.batch" || c.EditTool == "edit.symbol" || c.EditKind != "") { + path, _ := body["path"].(string) + if !editProbeFixturePath(path, c.File) { + return false + } + } + if c.EditKind == "append_leaf" { + return editProbeLeafSource(c.File, source, step.Marker) + } + return editProbeFixtureSource(c.File, editProbeCrossFilePlain(c, source), step.Marker, callee) + case "text": + return collection("matches", func(row map[string]any) bool { + path, _ := row["path"].(string) + text, _ := row["text"].(string) + line, err := strconvEditProbeUint(row["line"]) + prefix := "func " + if filepath.Ext(c.File) == ".ts" { + textCallee := callee + if c.EditKind == "append_leaf" { + textCallee = "" + } + valid, correct := editProbeTypeScriptFixture(text, step.Marker, textCallee) + return err == nil && line > 0 && editProbeFixturePath(path, c.File) && valid && correct + } + if filepath.Ext(c.File) == ".py" { + return err == nil && line > 0 && editProbeFixturePath(path, c.File) && strings.TrimSuffix(text, "\r") == "def "+step.Marker+"():" + } + return err == nil && line > 0 && (path == c.File || strings.HasSuffix(path, "/"+c.File)) && strings.Contains(text, prefix+step.Marker) + }) + case "callers", "call_chain": + if step.BeforeSymbol != "" && !editProbeRenameCensus(body, c, step) { + return false + } + if c.EditKind == "append_leaf" { + return editProbeLeafRelations(body, c, step) + } + return collection("edges", func(row map[string]any) bool { + from, _ := row["from"].(string) + to, _ := row["to"].(string) + calleeFile := c.File + if c.EditKind == "add_crossfile_call" { + calleeFile, _, _ = editProbeCrossFileCallee(c) + } + return row["kind"] == "calls" && editProbeSourceID(from, c.File, step.Marker) && editProbeSourceID(to, calleeFile, callee) + }) + case "summary", "editing_context", "search_visible": + key := map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"}[oracle] + return collection(key, func(row map[string]any) bool { + id, _ := row["id"].(string) + return editProbeSourceID(id, c.File, step.Marker) + }) + default: + return false + } +} + +// Compact rows prove presence only: the marker, or from the marker's call +// chain the callee, as a top-level function row at its owned fixture path. +// Relation answers must also report at least one edge; they still do not show +// which edge or its tier. Truncated rows prove presence, never absence. +func editProbeCompactCorrect(oracle string, answer editProbeCompactAnswer, c Corpus, step editProbeStep, callee string) bool { + operation := map[string]string{"summary": "read.summary", "search_visible": "search.symbols", "callers": "relations.callers", "call_chain": "trace.call_chain"}[oracle] + if operation == "" || answer.operation != operation { + return false + } + name, file := step.Marker, c.File + if oracle == "call_chain" { + name = callee + if c.EditKind == "add_crossfile_call" { + file, _, _ = editProbeCrossFileCallee(c) + } + } + if (oracle == "callers" || oracle == "call_chain") && answer.edges < 1 { + return false + } + for _, row := range answer.rows { + if row.Kind == "function" && row.Qualified == name && row.Owner == "" && editProbeFixturePath(row.Path, file) { + return true + } + } + return false +} + +func editProbeClaimsFresh(body map[string]any) bool { + claims := func(row map[string]any) bool { + if row["graph_freshness"] == "fresh" || row["freshness_state"] == "fresh" { + return true + } + return row["state"] == "fresh" || row["fresh"] == true + } + freshness, _ := body["freshness"].(map[string]any) + return body["freshness"] == "fresh" || claims(body) || claims(freshness) +} + +func strconvEditProbeUint(value any) (uint64, error) { + number, ok := value.(json.Number) + if !ok { + return 0, errors.New("missing integer identity") + } + var n uint64 + if _, err := fmt.Sscanf(number.String(), "%d", &n); err != nil { + return 0, err + } + if fmt.Sprint(n) != number.String() { + return 0, errors.New("noncanonical integer identity") + } + return n, nil +} + +func editProbeNS(value any) (int64, error) { + if value == nil { + return 0, errors.New("missing duration") + } + ms, ok := new(big.Rat).SetString(fmt.Sprint(value)) + if !ok || ms.Sign() < 0 { + return 0, errors.New("invalid nonnegative duration") + } + ns := new(big.Rat).Mul(ms, big.NewRat(1000000, 1)) + if !ns.IsInt() || !ns.Num().IsInt64() { + return 0, errors.New("nonintegral/out-of-range nanoseconds") + } + return ns.Num().Int64(), nil +} + +type editProbeWindow struct { + WallNS int64 `json:"wall_ns"` + SumNS int64 `json:"sum_ns"` + ResidualNS int64 `json:"residual_ns"` + WithinFivePercent bool `json:"within_five_percent"` +} + +func checkEditProbeWindow(wall any, phases map[string]any) (editProbeWindow, error) { + ns, err := editProbeNS(wall) + if err != nil { + return editProbeWindow{WallNS: ns}, err + } + return checkEditProbeWindowNS(ns, phases) +} + +func checkEditProbeWindowNS(ns int64, phases map[string]any) (editProbeWindow, error) { + out := editProbeWindow{WallNS: ns} + if ns < 0 { + return out, errors.New("negative window") + } + if len(phases) == 0 { + return out, errors.New("no observed phase values") + } + for name, value := range phases { + if value == nil { + continue + } + n, err := editProbeNS(value) + if err != nil { + return out, fmt.Errorf("%s: %w", name, err) + } + if out.SumNS > math.MaxInt64-n { + return out, errors.New("phase subtotal overflow") + } + out.SumNS += n + } + if out.SumNS > out.WallNS { + return out, errors.New("phase subtotal exceeds its wall") + } + out.ResidualNS = out.WallNS - out.SumNS + out.WithinFivePercent = out.ResidualNS <= out.WallNS/20 + if !out.WithinFivePercent { + return out, errors.New("accounting residual exceeds five percent") + } + return out, nil +} + +type editProbeAccounting struct { + State string `json:"state"` + RPCID string `json:"bound_rpc_id"` + Requested uint64 `json:"requested_generation"` + Applied uint64 `json:"applied_generation"` + A *editProbeWindow `json:"acknowledgement"` + P *editProbeWindow `json:"physical"` + B *editProbeWindow `json:"requested_background"` + R *editProbeWindow `json:"read_assisted_publisher,omitempty"` + Reasons []string `json:"reasons"` +} + +func checkEditProbeAccounting(status map[string]any, receipt string) editProbeAccounting { + return checkEditProbeAccountingWithAcknowledgement(status, receipt, "reindex_generation", nil, false) +} + +// A symbol splice has no ACK parser gate, but only the actual offered tool +// and the retained status's matching producer identity can select that rule. +func checkEditProbeAccountingForTool(status map[string]any, receipt, tool string) editProbeAccounting { + if tool == "edit.symbol" { + if status["tool"] != "edit_symbol" { + return editProbeAccounting{State: "unavailable", Reasons: []string{"status tool does not match offered edit.symbol"}} + } + return checkEditProbeAccountingWithAcknowledgement(status, receipt, "reindex_generation", nil, true) + } + return checkEditProbeAccounting(status, receipt) +} + +// A batch's shared RPC owner is supplied separately from its per-file P/B/R +// owner. The windows remain independent; no map is merged into a synthetic +// physical mutation_status reply and A is never added to background work. +func checkEditProbeAccountingWithAcknowledgement(status map[string]any, receipt, requestedField string, sharedAcknowledgement map[string]any, parseMayBeInapplicable bool) editProbeAccounting { + out := editProbeAccounting{State: "unavailable"} + fail := func(reason string) { out.Reasons = append(out.Reasons, reason) } + if status["receipt"] != receipt { + fail("receipt identity mismatch") + } + requested, err := strconvEditProbeUint(status[requestedField]) + if err != nil || requested == 0 { + fail("requested generation unavailable") + } + out.Requested = requested + applied, err := strconvEditProbeUint(status["applied_generation"]) + if err != nil || applied < requested { + fail("applied generation unavailable") + } + out.Applied = applied + ack, _ := status["acknowledgement"].(map[string]any) + if sharedAcknowledgement != nil { + ack = sharedAcknowledgement + } + out.RPCID, _ = ack["rpc_id"].(string) + if out.RPCID == "" || ack["disjoint_observed_intervals"] != true { + fail("bound disjoint A identity unavailable") + } + aPhases, _ := ack["observed_execution_ms"].(map[string]any) + aStates, _ := ack["phase_states"].(map[string]any) + for _, name := range []string{"caller_result", "disk_write", "edit_control", "middleware", "parse", "publish", "reindex_wait", "reply_prepare"} { + // The actual batch text-splice handler explicitly records that it + // performs no parser gate. Preserve that null applicability state; + // missing/unknown work and an applicable parse never get this escape. + _, phasePresent := aPhases[name] + if name == "parse" && phasePresent && parseMayBeInapplicable && ack["parse_scope"] == "not_applicable_text_splice" && aStates[name] == "not_applicable" && aPhases[name] == nil { + continue + } + if aPhases[name] == nil || aStates[name] != "measured" { + fail("missing applicable A phase " + name) + } + } + a, e := checkEditProbeWindow(ack["caller_response_ready_ms"], aPhases) + if e != nil { + fail("A: " + e.Error()) + } else { + out.A = &a + } + original, _ := status["phase_timing"].(map[string]any) + publication, _ := status["publication_phases"].(map[string]any) + ownerKnown := status["receipt"] == receipt && requested > 0 && applied >= requested + physical := original + if applied > requested { + if original["patch_wall_ms"] != nil || original["patch_accounting"] != nil { + fail("coalesced receipt borrowed accepted P") + } + winner, _ := status["coalesced_publication"].(map[string]any) + wReq, _ := strconvEditProbeUint(winner["requested_generation"]) + wApplied, _ := strconvEditProbeUint(winner["applied_generation"]) + pub, _ := winner["publication_phases"].(map[string]any) + ticket, _ := strconvEditProbeUint(pub["ticket"]) + originalPublication, _ := status["publication_phases"].(map[string]any) + if wReq != requested || wApplied != applied || ticket != applied || winner["scope"] != "independent_physical_winner_serving_coalesced_receipt" || pub["source"] != "canonical_coalesced_attempt" || pub["key"] == originalPublication["key"] { + ownerKnown = false + fail("independent serving identity mismatch") + } + publication = pub + timings, _ := winner["timings"].(map[string]any) + physical, _ = timings["phase_timing"].(map[string]any) + } + pAccounting, _ := physical["patch_accounting"].(map[string]any) + pTicket, _ := strconvEditProbeUint(pAccounting["attempt_ticket"]) + pRetry, retryErr := strconvEditProbeUint(pAccounting["attempt_retry"]) + if pAccounting["observed_partition_complete"] != true || pTicket != applied || retryErr != nil || pRetry != 0 { + fail("applicable physical P unavailable") + } + pPhases, _ := pAccounting["phases_ms"].(map[string]any) + pStates, _ := pAccounting["phase_states"].(map[string]any) + duplicate := ownerKnown && editProbeObservedDuplicatePhysical(physical, publication, applied) + if repair, present := status["read_assisted_publication"]; present { + r, repairErr := checkEditProbeReadRepair(repair, status, receipt, requested) + if repairErr != nil { + fail("R: " + repairErr.Error()) + } else { + out.R = &r + } + } + if duplicate && out.R == nil { + fail("actual read-assisted publisher deep coverage unavailable") + } + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "attempt_setup", "patch_inspection", "executor_control", "attempt_finalize", "cohort_admission", "lane_wait"} { + if duplicate && out.R != nil && (name == "executor_control" || isEditProbeRepairDeepPhase(name)) { + continue // explicitly unexecuted in P; validated independently in R + } + if pPhases[name] == nil || pStates[name] != "measured" { + fail("missing applicable P phase " + name) + } + } + p, e := checkEditProbeWindow(physical["patch_wall_ms"], pPhases) + if e != nil { + fail("P: " + e.Error()) + } else { + out.P = &p + } + background, _ := status["requested_background"].(map[string]any) + bReq, _ := strconvEditProbeUint(background["requested_generation"]) + bApplied, _ := strconvEditProbeUint(background["applied_generation"]) + if background["observed_partition_complete"] != true || background["scope"] != "accepted_scheduling_call_entry_to_serving_attempt_return" || !editProbeSupportedBackgroundCoverage(background) || background["sum_state"] != "partial" || bReq != requested || bApplied != applied { + fail("requested B unavailable for actual topology/identity") + } + refusal, backgroundUnavailable := editProbeOwnedBackgroundRefusal(status, receipt, requested) + if backgroundUnavailable { + fail("requested B observed unavailable: " + refusal) + } + bPhases, _ := background["phases_ms"].(map[string]any) + if _, duplicate := bPhases["debounce"]; duplicate { + fail("whole debounce double-counted in B") + } + for _, name := range requestedBackgroundLapNamesForProbe { + if bPhases[name] == nil { + fail("missing B phase " + name) + } + } + for name, value := range pPhases { + if value != nil && fmt.Sprint(value) != fmt.Sprint(bPhases[name]) { + fail("B/P phase mismatch: " + name) + } + } + b, e := checkEditProbeWindow(background["background_wall_ms"], bPhases) + if e != nil { + fail("B: " + e.Error()) + } else { + out.B = &b + } + if len(out.Reasons) == 0 { + out.State = "observed_single_window_checks_pass" + } else if ownerKnown && backgroundUnavailable { + out.State = "terminal_unavailable_background" + } else if ownerKnown && (editProbeImmutablePartialPhysical(physical, publication, applied) || + duplicate && editProbeImmutablePartialReadRepair(status["read_assisted_publication"], status, receipt, requested)) { + out.State = "terminal_partial_physical" + } + return out +} + +// Receipt terminality can precede the actual serving return. Only the immutable +// late window proves that missing deep laps cannot arrive. A complete P with +// absent B must still wait: background publication follows physical publication. +// A complete duplicate P may precede R retention and B publication, so it +// cannot itself make absent publisher/background evidence terminal. +func editProbeImmutablePartialPhysical(physical, publication map[string]any, applied uint64) bool { + accounting, ok := editProbePhysicalAccounting(physical, publication, applied) + return ok && accounting["observed_partition_complete"] == false && + (accounting["sum_state"] == "missing_laps" || accounting["sum_state"] == "invalid_order") +} + +func editProbePhysicalAccounting(physical, publication map[string]any, applied uint64) (map[string]any, bool) { + if _, err := editProbeNS(physical["patch_wall_ms"]); err != nil || physical["clock"] != "monotonic" || publication["clock"] != "monotonic" || publication["terminal"] != true { + return nil, false + } + key, _ := publication["key"].(string) + ticket, err := strconvEditProbeUint(publication["ticket"]) + if key == "" || err != nil || ticket != applied || (publication["source"] != "mcp_edit" && publication["source"] != "canonical_coalesced_attempt") { + return nil, false + } + accounting, _ := physical["patch_accounting"].(map[string]any) + owner, err := strconvEditProbeUint(accounting["attempt_ticket"]) + _, retryErr := strconvEditProbeUint(accounting["attempt_retry"]) + if err != nil || owner != applied || retryErr != nil || accounting["scope"] != "accepted_winning_physical_attempt" { + return nil, false + } + return accounting, true +} + +func editProbeObservedDuplicatePhysical(physical, publication map[string]any, applied uint64) bool { + accounting, ok := editProbePhysicalAccounting(physical, publication, applied) + if !ok { + return false + } + retry, retryErr := strconvEditProbeUint(accounting["attempt_retry"]) + if retryErr != nil || accounting["observed_partition_complete"] != true || accounting["execution_disposition"] != "native_mtime_duplicate" || accounting["sum_state"] != "partial" || retry != 0 { + return false + } + phases, _ := accounting["phases_ms"].(map[string]any) + states, _ := accounting["phase_states"].(map[string]any) + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", "executor_control"} { + if phases[name] != nil || states[name] != "not_applicable" { + return false + } + } + for _, name := range []string{"attempt_setup", "patch_inspection", "attempt_finalize", "cohort_admission", "lane_wait"} { + if phases[name] == nil || states[name] != "measured" { + return false + } + } + window, err := checkEditProbeWindow(physical["patch_wall_ms"], phases) + return err == nil && window.ResidualNS == 0 +} + +func editProbeFinished(graphTerminal, allCorrect, requireAccounting bool, accounting editProbeAccounting) bool { + return graphTerminal && allCorrect && (!requireAccounting || accounting.State == "observed_single_window_checks_pass" || accounting.State == "terminal_partial_physical" || accounting.State == "terminal_unavailable_background") +} + +func editProbeSupportedBackgroundCoverage(background map[string]any) bool { + if _, generationPresent := background["admission_predecessor_generation"]; generationPresent || background["admission_predecessor_stopped"] != nil { + predecessor, e1 := strconvEditProbeUint(background["admission_predecessor_generation"]) + requested, e2 := strconvEditProbeUint(background["requested_generation"]) + if e1 != nil || e2 != nil || predecessor == 0 || predecessor >= requested || background["admission_predecessor_stopped"] != true { + return false + } + } + if background["coverage"] != "direct_requested_attempt" { + if phases, ok := background["phases_ms"].(map[string]any); ok { + if quiet, present := phases["requested_quiet_wait"]; present { + ns, e := editProbeNS(quiet) + if quiet == nil || e != nil || ns != 0 { + return false + } + } + } + } + count, present := background["observed_native_handoffs"] + handoffs, err := strconvEditProbeUint(count) + switch background["coverage"] { + case "one_canceled_timer_one_native_successor": + return !present || err == nil && handoffs == 1 // Historical single-handoff wire shape. + case "successful_pre_callback_native_handoffs": + return present && err == nil && handoffs >= 2 + case "direct_requested_attempt": + requested, e1 := strconvEditProbeUint(background["requested_generation"]) + applied, e2 := strconvEditProbeUint(background["applied_generation"]) + phases, _ := background["phases_ms"].(map[string]any) + quiet, observed := phases["requested_quiet_wait"] + for _, name := range []string{"canceled_quiet_wait", "timer_stop", "coalescing_control", "successor_timer_arm", "successor_quiet_wait"} { + value, exists := phases[name] + ns, e := editProbeNS(value) + if !exists || value == nil || e != nil || ns != 0 { + return false + } + } + return present && err == nil && handoffs == 0 && e1 == nil && e2 == nil && requested > 0 && applied == requested && observed && quiet != nil + default: + return false + } +} + +// Only an actual frozen refusal for this original publication can stop late-B +// collection. Missing B, unknown/malformed diagnostics and foreign owners wait. +// This is terminal failed coverage, never measured work or a successful edit. +func editProbeOwnedBackgroundRefusal(status map[string]any, receipt string, requested uint64) (string, bool) { + if receipt == "" || status["receipt"] != receipt || requested == 0 || status["requested_background"] != nil { + return "", false + } + original, _ := status["publication_phases"].(map[string]any) + key, _ := original["key"].(string) + ticket, ticketErr := strconvEditProbeUint(original["ticket"]) + if key == "" || ticketErr != nil || ticket != requested || original["source"] != "mcp_edit" || original["clock"] != "monotonic" { + return "", false + } + diagnostic, _ := status["requested_background_unavailable"].(map[string]any) + dReq, reqErr := strconvEditProbeUint(diagnostic["requested_generation"]) + last, lastErr := strconvEditProbeUint(diagnostic["last_observed_generation"]) + applied, appliedErr := strconvEditProbeUint(status["applied_generation"]) + handoffs, countErr := strconvEditProbeUint(diagnostic["observed_native_handoffs"]) + if diagnostic["scope"] != "requested_background_observation" || diagnostic["publication_key"] != key || reqErr != nil || dReq != requested || lastErr != nil || last < requested || appliedErr != nil || last > applied || countErr != nil || handoffs > 0 && last == requested { + return "", false + } + reason, _ := diagnostic["reason"].(string) + switch reason { + case "foreign_timer_identity", "invalid_timing_order", "timing_overflow", "handoff_count_overflow", "unobserved_timer_handoff", "repeated_timer_arm", "missing_timer_arm", "observed_successor", "preexisting_path_work", "preexisting_timer_stop_not_successful", "timer_stop_not_successful", "timer_callback_already_started", "requested_callback_started", "timer_claim_superseded", "retry_or_backoff", "serving_attempt_unavailable", "serving_attempt_binding_mismatch", "physical_evidence_incomplete": + return reason, true + default: + return "", false + } +} + +var requestedBackgroundLapNamesForProbe = []string{"scheduling_admission_setup", "initial_timer_arm", "canceled_quiet_wait", "timer_stop", "coalescing_control", "successor_timer_arm", "successor_quiet_wait"} diff --git a/bench/load/edit_probe_evidence_test.go b/bench/load/edit_probe_evidence_test.go new file mode 100644 index 000000000..448fc2507 --- /dev/null +++ b/bench/load/edit_probe_evidence_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestEditProbeOraclesRejectEchoAndRequireActualSourceAndEdges(t *testing.T) { + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + for oracle, collection := range map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"} { + facts := map[string]any{collection: []any{map[string]any{"id": "sample/" + corpus.File + "::" + step.Marker}}} + if !editProbeCorrect(oracle, facts, corpus, step, plan.Callee) { + t.Fatalf("%s rejected its producer collection", oracle) + } + if editProbeCorrect(oracle, map[string]any{"guidance": facts}, corpus, step, plan.Callee) { + t.Fatalf("%s accepted nested guidance", oracle) + } + } + echo := map[string]any{"query": step.Marker, "target": map[string]any{"id": "sample/" + corpus.File + "::" + step.Marker}} + for _, name := range []string{"summary", "editing_context", "search_visible", "text", "callers", "source"} { + if editProbeCorrect(name, echo, corpus, step, plan.Callee) { + t.Fatalf("%s accepted query echo", name) + } + } + source := strings.ReplaceAll(corpus.EditTemplate, "{{marker}}", step.Marker) + if !editProbeCorrect("source", map[string]any{"name": step.Marker, "source": source}, corpus, step, plan.Callee) { + t.Fatal("actual source/call rejected") + } + edge := map[string]any{"edges": []any{map[string]any{"from": "sample/" + corpus.File + "::" + step.Marker, "to": "sample/" + corpus.Symbol, "kind": "calls"}}} + if !editProbeCorrect("callers", edge, corpus, step, plan.Callee) { + t.Fatal("actual intrafile call rejected") + } + edge["edges"].([]any)[0].(map[string]any)["to"] = "ext::" + plan.Callee + if editProbeCorrect("callers", edge, corpus, step, plan.Callee) { + t.Fatal("external/name-only relation became source proof") + } +} + +func TestEditProbeWindowRejectsMissingFractionalNanosecondsAndOverlap(t *testing.T) { + check, err := checkEditProbeWindow(json.Number("0.000003"), map[string]any{"a": json.Number("0.000001"), "b": json.Number("0.000002")}) + if err != nil || check.WallNS != 3 || check.ResidualNS != 0 { + t.Fatalf("exact fractional-ms check%+v %v", check, err) + } + for _, value := range []any{nil, json.Number("0.0000001"), json.Number("-1"), json.Number("999999999999999999999")} { + if _, err := editProbeNS(value); err == nil { + t.Fatalf("invalid scalar accepted%v", value) + } + } + if _, err := checkEditProbeWindow(json.Number("1"), map[string]any{"a": json.Number("2")}); err == nil { + t.Fatal("overlapping/excess scalar sum accepted") + } +} + +func editProbeAccountingFixture() map[string]any { + ackPhases, ackStates := map[string]any{}, map[string]any{} + for _, name := range []string{"caller_result", "disk_write", "edit_control", "middleware", "parse", "publish", "reindex_wait", "reply_prepare"} { + ackPhases[name], ackStates[name] = json.Number("1"), "measured" + } + p, b, pStates := map[string]any{}, map[string]any{}, map[string]any{} + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "attempt_setup", "patch_inspection", "executor_control", "attempt_finalize", "cohort_admission", "lane_wait"} { + p[name], b[name] = json.Number("1"), json.Number("1") + pStates[name] = "measured" + } + for _, name := range requestedBackgroundLapNamesForProbe { + b[name] = json.Number("1") + } + return map[string]any{ + "receipt": "receipt-a", "reindex_generation": json.Number("1"), "applied_generation": json.Number("2"), + "acknowledgement": map[string]any{"rpc_id": "rpc-a", "disjoint_observed_intervals": true, "caller_response_ready_ms": json.Number("8"), "observed_execution_ms": ackPhases, "phase_states": ackStates}, + "publication_phases": map[string]any{"key": "original"}, + "coalesced_publication": map[string]any{"requested_generation": json.Number("1"), "applied_generation": json.Number("2"), "scope": "independent_physical_winner_serving_coalesced_receipt", "publication_phases": map[string]any{"ticket": json.Number("2"), "source": "canonical_coalesced_attempt", "key": "winner"}, "timings": map[string]any{"phase_timing": map[string]any{"patch_wall_ms": json.Number("13"), "patch_accounting": map[string]any{"observed_partition_complete": true, "attempt_ticket": json.Number("2"), "attempt_retry": json.Number("0"), "phases_ms": p, "phase_states": pStates}}}}, + "requested_background": map[string]any{"requested_generation": json.Number("1"), "applied_generation": json.Number("2"), "observed_partition_complete": true, "scope": "accepted_scheduling_call_entry_to_serving_attempt_return", "coverage": "one_canceled_timer_one_native_successor", "sum_state": "partial", "background_wall_ms": json.Number("20"), "phases_ms": b}, + } +} + +func TestEditProbeAccountingRejectsMissingApplicableACKAndWrongAttempt(t *testing.T) { + fixture := editProbeAccountingFixture + if got := checkEditProbeAccounting(fixture(), "receipt-a"); got.State != "observed_single_window_checks_pass" { + t.Fatalf("valid fixture: %+v", got) + } + for _, name := range []string{"missing_ack", "wrong_retry", "wrong_owner", "unmeasured_physical"} { + t.Run(name, func(t *testing.T) { + body := fixture() + if name == "missing_ack" { + delete(body["acknowledgement"].(map[string]any)["observed_execution_ms"].(map[string]any), "parse") + } else { + p := body["coalesced_publication"].(map[string]any)["timings"].(map[string]any)["phase_timing"].(map[string]any)["patch_accounting"].(map[string]any) + switch name { + case "unmeasured_physical": + p["phase_states"].(map[string]any)["apply"] = "not_observed" + case "wrong_retry": + p["attempt_retry"] = json.Number("1") + default: + p["attempt_ticket"] = json.Number("9") + } + } + if got := checkEditProbeAccounting(body, "receipt-a"); got.State == "observed_single_window_checks_pass" { + t.Fatalf("invalid fixture accepted: %+v", got) + } + }) + } +} + +func TestEditProbeFreshnessDoesNotReadGuidanceExamples(t *testing.T) { + fresh := map[string]any{"freshness": map[string]any{"graph_freshness": "fresh"}} + if !editProbeClaimsFresh(fresh) || editProbeClaimsFresh(map[string]any{"guidance": fresh}) { + t.Fatal("freshness rider confused with nested example") + } +} + +func TestEditProbeWaitsForLatePhysicalReturnButStopsImmutableMissingLaps(t *testing.T) { + fixture := func(partial bool) map[string]any { + body := editProbeAccountingFixture() + winner := body["coalesced_publication"].(map[string]any) + pub := winner["publication_phases"].(map[string]any) + pub["clock"], pub["terminal"] = "monotonic", true + physical := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + physical["clock"] = "monotonic" + accounting := physical["patch_accounting"].(map[string]any) + accounting["scope"] = "accepted_winning_physical_attempt" + if partial { + accounting["observed_partition_complete"], accounting["sum_state"] = false, "missing_laps" + accounting["phases_ms"].(map[string]any)["apply"] = nil + accounting["phase_states"].(map[string]any)["apply"] = "not_observed" + physical["patch_wall_ms"] = json.Number("12") + } + delete(body, "requested_background") + return body + } + early := fixture(true) + early["coalesced_publication"].(map[string]any)["timings"].(map[string]any)["phase_timing"].(map[string]any)["patch_wall_ms"] = nil + earlyView := checkEditProbeAccounting(early, "receipt-a") + if earlyView.State != "unavailable" || editProbeFinished(true, true, true, earlyView) { + t.Fatal("receipt terminality stopped waiting before late P") + } + lateView := checkEditProbeAccounting(fixture(true), "receipt-a") + if lateView.State != "terminal_partial_physical" || len(lateView.Reasons) == 0 || !editProbeFinished(true, true, true, lateView) { + t.Fatalf("immutable missing deep laps did not terminate failed coverage: %+v", lateView) + } + if editProbeFinished(false, true, true, lateView) || editProbeFinished(true, false, true, lateView) { + t.Fatal("missing graph/read evidence was discarded") + } + if earlyView.State != "unavailable" || checkEditProbeAccounting(early, "receipt-a").State != "unavailable" { + t.Fatal("later observation changed an earlier snapshot") + } + completeWithoutB := checkEditProbeAccounting(fixture(false), "receipt-a") + if completeWithoutB.State != "unavailable" || editProbeFinished(true, true, true, completeWithoutB) { + t.Fatal("complete P stopped before late B publication") + } + for _, bad := range []string{"owner", "clock", "terminal", "wall", "retry"} { + t.Run(bad, func(t *testing.T) { + body := fixture(true) + winner := body["coalesced_publication"].(map[string]any) + pub := winner["publication_phases"].(map[string]any) + physical := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + switch bad { + case "owner": + pub["ticket"] = json.Number("9") + case "clock": + pub["clock"] = "unavailable" + case "terminal": + pub["terminal"] = false + case "wall": + physical["patch_wall_ms"] = json.Number("-1") + case "retry": + delete(physical["patch_accounting"].(map[string]any), "attempt_retry") + } + if got := checkEditProbeAccounting(body, "receipt-a"); got.State == "terminal_partial_physical" { + t.Fatalf("unproven late-window identity stopped waiting: %+v", got) + } + }) + } +} + +func TestEditProbeObservedDuplicateWaitsForPublisherCoverage(t *testing.T) { + fixture := func() map[string]any { + body := editProbeAccountingFixture() + winner := body["coalesced_publication"].(map[string]any) + pub := winner["publication_phases"].(map[string]any) + pub["clock"], pub["terminal"] = "monotonic", true + physical := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + physical["clock"], physical["patch_wall_ms"] = "monotonic", json.Number("5") + a := physical["patch_accounting"].(map[string]any) + a["scope"], a["sum_state"], a["execution_disposition"] = "accepted_winning_physical_attempt", "partial", "native_mtime_duplicate" + phases, states := a["phases_ms"].(map[string]any), a["phase_states"].(map[string]any) + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", "executor_control"} { + phases[name], states[name] = nil, "not_applicable" + } + delete(body, "requested_background") + return body + } + body := fixture() + got := checkEditProbeAccounting(body, "receipt-a") + if got.State != "unavailable" || len(got.Reasons) == 0 || editProbeFinished(true, true, true, got) { + t.Fatalf("observed no-op cannot prove late publisher evidence absent: %+v", got) + } + for _, bad := range []string{"early", "disposition", "deep_value", "deep_state", "control", "gap", "retry", "owner", "receipt"} { + t.Run(bad, func(t *testing.T) { + body := fixture() + winner := body["coalesced_publication"].(map[string]any) + physical := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + a := physical["patch_accounting"].(map[string]any) + switch bad { + case "early": + physical["patch_wall_ms"] = nil + case "disposition": + delete(a, "execution_disposition") + case "deep_value": + a["phases_ms"].(map[string]any)["apply"] = json.Number("0") + case "deep_state": + a["phase_states"].(map[string]any)["apply"] = "not_observed" + case "control": + a["phase_states"].(map[string]any)["attempt_finalize"] = "not_observed" + case "gap": + physical["patch_wall_ms"] = json.Number("6") + case "retry": + a["attempt_retry"] = json.Number("1") + case "owner": + winner["applied_generation"] = json.Number("3") + case "receipt": + body["receipt"] = "foreign" + } + got := checkEditProbeAccounting(body, "receipt-a") + if got.State == "terminal_partial_physical" || got.State == "observed_single_window_checks_pass" || editProbeFinished(true, true, true, got) { + t.Fatalf("unproven no-op evidence terminated collection: %+v", got) + } + }) + } +} diff --git a/bench/load/edit_probe_leaf.go b/bench/load/edit_probe_leaf.go new file mode 100644 index 000000000..d2d56e960 --- /dev/null +++ b/bench/load/edit_probe_leaf.go @@ -0,0 +1,70 @@ +package main + +import ( + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "strings" +) + +// The append-leaf cell adds one numeric, zero-argument leaf declaration. A +// comment, string literal containing a call, or an actual outgoing call cannot +// stand in for a leaf. Other fixture declarations may coexist in a file read. +func editProbeLeafSource(file, source, name string) bool { + switch filepath.Ext(file) { + case ".ts": + valid, correct := editProbeTypeScriptFixture(source, name, "") + return valid && correct + case ".py": + valid, correct := editProbePythonFixture(source, name, "") + return valid && correct + case ".go": + parsed, err := parser.ParseFile(token.NewFileSet(), "leaf.go", source, 0) + if err != nil { + parsed, err = parser.ParseFile(token.NewFileSet(), "leaf.go", "package fixture\n"+source, 0) + } + if err != nil { + return false + } + matches := 0 + for _, decl := range parsed.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != name { + continue + } + if fn.Recv != nil || fn.Type.Params.NumFields() != 0 || fn.Body == nil || len(fn.Body.List) != 1 { + return false + } + ret, ok := fn.Body.List[0].(*ast.ReturnStmt) + if !ok || len(ret.Results) != 1 { + return false + } + value, ok := ret.Results[0].(*ast.BasicLit) + if !ok || value.Kind != token.INT || strings.Trim(value.Value, "0123456789") != "" { + return false + } + matches++ + } + return matches == 1 + } + return false +} + +func editProbeLeafRelations(body map[string]any, c Corpus, step editProbeStep) bool { + truncated, ok := body["truncated"].(bool) + if !ok || truncated { + return false + } + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + if !nok || !eok || len(nodes) != 1 || len(edges) != 0 { + return false + } + node, ok := nodes[0].(map[string]any) + if !ok { + return false + } + id, _ := node["id"].(string) + return editProbeSourceID(id, c.File, step.Marker) +} diff --git a/bench/load/edit_probe_leaf_test.go b/bench/load/edit_probe_leaf_test.go new file mode 100644 index 000000000..faaa94fe7 --- /dev/null +++ b/bench/load/edit_probe_leaf_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestEditProbeAppendLeafHasNoInventedCallEvidence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeSymbolCorpus(t, language) + c.EditKind = "append_leaf" + c.EditTemplate = strings.ReplaceAll(c.EditTemplate, "Example()", "7") + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + source := strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + if !editProbeLeafSource(c.File, source, step.Marker) { + t.Fatal("leaf declaration refused") + } + oracles := editProbeOracles(c, step) + if oracles[5].call.Arguments["target"].(map[string]any)["symbol"] != c.File+"::"+step.Marker { + t.Fatal("leaf callers still queried unrelated callee") + } + body := map[string]any{"nodes": []any{map[string]any{"id": "owned/" + c.File + "::" + step.Marker}}, "edges": []any{}, "total_nodes": json.Number("1"), "total_edges": json.Number("0"), "truncated": false} + if !editProbeCorrect("callers", body, c, step, plan.Callee) || !editProbeCorrect("call_chain", body, c, step, plan.Callee) { + t.Fatal("complete isolated leaf relation proof refused") + } + body["edges"] = []any{map[string]any{"from": "owned/" + c.File + "::" + step.Marker, "to": "owned/" + c.Symbol, "kind": "calls"}} + body["total_edges"] = json.Number("1") + if editProbeCorrect("call_chain", body, c, step, plan.Callee) { + t.Fatal("outgoing call accepted as leaf") + } + } + }) + } +} + +func TestEditProbeLeafRejectsCommentStringCallAndPartialRelationEvidence(t *testing.T) { + for _, test := range []struct{ file, source string }{ + {"file.go", "// func Leaf() int { return 1 }"}, + {"file.go", "func Leaf() int { return Example() }"}, + {"file.go", "func Leaf() string { return \"Example()\" }"}, + {"file.ts", "// export function Leaf(): number { return 1; }"}, + {"file.ts", "export function Leaf(): number { return Example(); }"}, + {"file.py", "# def Leaf():\n# return 1"}, + {"file.py", "def Leaf():\n return Example()"}, + } { + if editProbeLeafSource(test.file, test.source, "Leaf") { + t.Fatalf("non-leaf proof accepted: %s", test.source) + } + } + c := editProbeCorpus() + step := editProbeStep{Marker: "Leaf"} + for _, body := range []map[string]any{ + {}, + {"nodes": []any{}, "edges": []any{}, "total_nodes": json.Number("0"), "total_edges": json.Number("0"), "truncated": false}, + {"nodes": []any{map[string]any{"id": c.File + "::Leaf"}}, "edges": []any{}, "total_nodes": json.Number("1"), "total_edges": json.Number("0"), "truncated": true}, + } { + if editProbeLeafRelations(body, c, step) { + t.Fatalf("partial/empty relation page accepted:%v", body) + } + } +} diff --git a/bench/load/edit_probe_lifecycle.go b/bench/load/edit_probe_lifecycle.go new file mode 100644 index 000000000..0dc85f38c --- /dev/null +++ b/bench/load/edit_probe_lifecycle.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +func editProbeMarker(seed int64, ordinal int) string { + return fmt.Sprintf("GortexEditProbe%016xN%03d", uint64(seed), ordinal) +} + +// Each even ordinal natively creates its own Go file beside the fixture, in +// the fixture's package, whose marker calls the fixture's callee; the next +// ordinal deletes that file again. +func editProbeLifecycleStep(c Corpus, pkg string, seed int64, ordinal int) (r request, file, beforeSymbol string) { + marker := editProbeMarker(seed, ordinal-ordinal%2) + file = "gortex_edit_probe_" + marker + ".go" + if ordinal%2 == 1 { + return request{Tool: "native", Operation: "delete_file", Arguments: map[string]any{"path": file}}, file, file + "::" + marker + } + content := "package " + pkg + "\n" + strings.ReplaceAll(c.EditTemplate, "{{marker}}", marker) + return request{Tool: "native", Operation: "create_file", Arguments: map[string]any{"path": file, "content": content}}, file, "" +} + +// Hot-file pairs share one due time on both fixed editors. The pair stream +// is the single-editor stream, so each editor still offers that mean. +func editProbeHotFileArrivals(seed int64, count int) ([]editProbeArrival, error) { + pairs, err := editProbeArrivals(seed, count/2, 1) + if err != nil { + return nil, err + } + arrivals := make([]editProbeArrival, 0, count) + for _, pair := range pairs { + arrivals = append(arrivals, editProbeArrival{DueNS: pair.DueNS}, editProbeArrival{DueNS: pair.DueNS, Editor: 1}) + } + return arrivals, nil +} + +// A created or deleted file is the step's own fixture file: its marker calls +// the original fixture's callee from another file of the same package. +func editProbeStepCorpus(c Corpus, step editProbeStep) Corpus { + c.File, c.EditKind, c.EditCallee = step.File, "add_crossfile_call", c.Symbol + return c +} + +// A created file is checked like a crossfile call addition. A deleted file +// keeps only oracles whose answer for a missing file has a pinned contract: +// the exact scoped refusal and complete search, text and caller collections. +func editProbeLifecycleOracles(c Corpus, step editProbeStep) []editProbeOracle { + created, inner := editProbeStepCorpus(c, step), step + inner.File = "" + if !step.Deleted { + return editProbeOracles(created, inner) + } + var oracles []editProbeOracle + for _, oracle := range editProbeDeleteOracles(created, inner) { + switch oracle.name { + case "source", "search_visible", "text", "callers": + oracles = append(oracles, oracle) + } + } + return editProbeOutputOracles(c, oracles) +} + +func editProbeLifecycleCorrect(oracle string, body map[string]any, c Corpus, step editProbeStep, callee string) bool { + created, inner := editProbeStepCorpus(c, step), step + inner.File = "" + if !step.Deleted { + return editProbeCorrect(oracle, body, created, inner, callee) + } + if oracle != "callers" { + return editProbeDeleteCorrect(oracle, body, created, inner, callee) + } + // The stable callee keeps its own node in the fixture file, and no node + // or edge may still name the deleted file's marker. + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + truncated, tok := body["truncated"].(bool) + if !nok || !eok || !tok || truncated { + return false + } + old := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + survivor := false + for _, raw := range nodes { + row, ok := raw.(map[string]any) + id, _ := row["id"].(string) + if !ok || editProbeSourceID(id, step.File, old) { + return false + } + survivor = survivor || editProbeSourceID(id, c.File, callee) + } + for _, raw := range edges { + row, ok := raw.(map[string]any) + from, fok := row["from"].(string) + to, tok := row["to"].(string) + if !ok || !fok || !tok || editProbeSourceID(from, step.File, old) || editProbeSourceID(to, step.File, old) { + return false + } + } + return survivor +} + +// A create/delete cell never writes the fixture. Created files still on disk +// are removed by one timed public batch; none left means no batch is sent. +func (r *editProbeRuntime) restoreCreatedFiles(ctx context.Context) error { + var created []string + var changes []any + for _, step := range r.plan.Steps { + if step.Sender != "create_file" { + continue + } + created = append(created, step.File) + if _, err := os.Lstat(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(step.File))); err == nil { + changes = append(changes, map[string]any{"op": "delete_file", "path": r.p.Repos[0].Prefix + "/" + step.File}) + } + } + if len(changes) == 0 { + r.phase.EditProbe.RestoreMS = unavailable("ms", "every created file was deleted by its own pair") + } else if err := r.restoreBatch(ctx, changes); err != nil { + return err + } + for _, file := range created { + if _, err := os.Lstat(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(file))); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("created file %s remains after restore", file) + } + } + current, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) + if err != nil { + return err + } + if string(current) != string(r.original) { + return errors.New("create/delete cell changed the fixture bytes") + } + return nil +} + +func (r *editProbeRuntime) restoreBatch(ctx context.Context, changes []any) error { + req := request{Tool: "edit", Operation: "batch", Arguments: map[string]any{"operation": "batch", "changes": changes, "options": map[string]any{"transaction_id": fmt.Sprintf("load-restore-%016x", uint64(r.options.Seed))}, "output": map[string]any{"format": "json", "max_bytes": 0}}} + start := time.Now() + data, _, err := callEditProbeTool(ctx, r.editors[0], req, req.Arguments, "cleanup", 0, len(r.states), 0, r.options.Deadline, editProbeRetrySleep(r.options.Seed, len(r.states), 0)) + r.phase.EditProbe.RestoreMS = measured(float64(time.Since(start))/float64(time.Millisecond), "ms") + r.phase.EditProbe.Restore = data + return err +} diff --git a/bench/load/edit_probe_lifecycle_test.go b/bench/load/edit_probe_lifecycle_test.go new file mode 100644 index 000000000..3a45131e5 --- /dev/null +++ b/bench/load/edit_probe_lifecycle_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +func editProbeCreateDeleteCorpus(t *testing.T) Corpus { + t.Helper() + c := editProbeWriteCorpus(t, "go") + c.EditTool, c.EditSender = "edit.file", editProbeCreateDelete + return c +} + +func TestEditProbeCreateDeletePairsUseOwnedNewFilesAndExactAbsence(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + reference, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + for i, step := range plan.Steps { + creator := plan.Steps[i-i%2] + file := "gortex_edit_probe_" + creator.Marker + ".go" + if step.DueNS != reference.Steps[i].DueNS || step.File != file || step.Request.Tool != "native" || step.Request.Arguments["path"] != file || step.MutationID != "" { + t.Fatalf("lifecycle step does not own its new file: %+v", step) + } + names := map[string]bool{} + for _, oracle := range editProbeOracles(c, step) { + names[oracle.name] = true + } + if i%2 == 0 { + if step.Sender != "create_file" || step.After != nil || step.Deleted || step.BeforeSymbol != "" || step.Request.Arguments["content"] != "package sample\n"+strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) || len(names) != 7 { + t.Fatalf("create step: %+v %v", step, names) + } + continue + } + if step.Sender != "delete_file" || step.After == nil || *step.After != i-1 || step.AfterEvent != "first_correct" || !step.Deleted || step.BeforeSymbol != file+"::"+creator.Marker || len(names) != 4 || !names["source"] || !names["search_visible"] || !names["text"] || !names["callers"] { + t.Fatalf("delete step must prove exact absence of its own pair's file: %+v %v", step, names) + } + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 2) + created := filepath.Join(dir, plan.Steps[0].File) + for i, step := range plan.Steps[:2] { + run.activate(ctx, step) + awaitEditProbeFirstProbe(t, run.states[i]) + run.doEdit(ctx, step) + record := run.states[i].record + if record.Outcome != "complete" || len(record.Errors) != 0 || record.Ack.Value != nil || record.Accounting.State != "not_applicable_native_write" || record.NativeWrite == nil || record.NativeWrite.Window.Value == nil { + t.Fatalf("%s did not complete: %+v", step.Sender, record) + } + if _, err := os.Stat(created); (err == nil) != (i == 0) { + t.Fatalf("%s left the created file in the wrong state: %v", step.Sender, err) + } + } + create, remove := run.states[0].record, run.states[1].record + if create.NativeWrite.Before != nil || create.NativeWrite.Written == nil || create.Readback.Value == nil { + t.Fatalf("create window: %+v", create) + } + if remove.NativeWrite.Before == nil || remove.NativeWrite.Written != nil || remove.Readback.Value != nil || remove.Readback.Reason != "a deleted file has no readback to gate" || len(remove.Readbacks) != 1 || remove.Readbacks[0].ExpectedOutcome != "symbol_absent" || !remove.Readbacks[0].ObservationOnly || remove.ExpectedRefusals == 0 { + t.Fatalf("delete was not proved by exact absence: %+v", remove) + } + if err := run.restore(ctx); err != nil || run.phase.EditProbe.RestoreMS.Value != nil || run.phase.EditProbe.RestoreMS.Reason != "every created file was deleted by its own pair" { + t.Fatalf("restore after complete pairs: %v %+v", err, run.phase.EditProbe.RestoreMS) + } +} + +// A delete activates at its own due time, which can come before its create's +// file is visible. Its search, text and caller probes then answer "absent"; +// those answers precede the delete's dependency and must not count as the +// marker seen before its own edit. +func TestEditProbeDeleteActivatedBeforeItsCreateCompletes(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 2) + run.activate(ctx, plan.Steps[0]) + run.activate(ctx, plan.Steps[1]) + awaitEditProbeFirstProbe(t, run.states[0]) + // Two source answers mean every delete oracle has answered at least once + // while the created file did not exist yet. + for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(5 * time.Millisecond) { + run.states[1].mu.Lock() + answered := run.states[1].record.ProbeErrors >= 2 + run.states[1].mu.Unlock() + if answered { + break + } + if time.Now().After(deadline) { + t.Fatal("delete probes never answered before the create") + } + } + for i, step := range plan.Steps[:2] { + run.doEdit(ctx, step) + if record := run.states[i].record; record.Outcome != "complete" || len(record.Errors) != 0 { + t.Fatalf("%s did not complete: %+v", step.Sender, record) + } + } +} + +func TestEditProbeCreatedFilesRestoredByOneTimedBatch(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 2) + run.activate(ctx, plan.Steps[0]) + awaitEditProbeFirstProbe(t, run.states[0]) + run.doEdit(ctx, plan.Steps[0]) + // A create whose probes did not all become first-correct never admits + // its delete; the created file stays for the restore batch. + run.states[0].mu.Lock() + run.states[0].record.Outcome = "product_deadline" + run.states[0].mu.Unlock() + run.activate(ctx, plan.Steps[1]) + run.doEdit(ctx, plan.Steps[1]) + remove := run.states[1].record + created := filepath.Join(dir, plan.Steps[0].File) + if remove.Outcome != "diagnostic_failure" || !slices.Contains(remove.Errors, "native write dependency failed: ordinal 0 did not complete every first-correct probe") || remove.NativeWrite.StartNS != nil { + t.Fatalf("delete ran without its create: %+v", remove) + } + if _, err := os.Stat(created); err != nil { + t.Fatal("delete removed a file whose create was unproved") + } + if err := run.restore(ctx); err != nil || run.phase.EditProbe.RestoreMS.Value == nil { + t.Fatalf("restore did not remove the created file in one timed batch: %v", err) + } + if _, err := os.Stat(created); !os.IsNotExist(err) { + t.Fatal("created file remains after restore") + } + if data, _ := os.ReadFile(filepath.Join(dir, c.File)); string(data) != c.EditInitialContent { + t.Fatal("restore changed the fixture") + } + + for name, mutate := range map[string]func(*Corpus){ + "typescript": func(c *Corpus) { + *c = editProbeWriteCorpus(t, "typescript") + c.EditTool, c.EditSender = "edit.file", editProbeCreateDelete + }, + "nested_fixture": func(c *Corpus) { c.File, c.Symbol = "pkg/sample.go", "pkg/sample.go::Example" }, + "compact": func(c *Corpus) { c.EditOutput = "compact" }, + "no_fixture": func(c *Corpus) { c.EditInitialContent = "" }, + } { + bad := editProbeCreateDeleteCorpus(t) + mutate(&bad) + if _, err := makeEditProbePlan(bad, 42, 100); err == nil { + t.Errorf("%s: create/delete planned", name) + } else if name == "typescript" && !strings.Contains(err.Error(), "not implemented") { + t.Errorf("TypeScript create/delete is a gap, not inapplicable: %v", err) + } + } + if _, err := makeEditProbePlan(c, 42, 1); err == nil { + t.Error("create/delete one-edit diagnostic planned") + } +} + +func TestEditProbeHotFilePairsShareDueAndFixedEditors(t *testing.T) { + single, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + hot := editProbeCorpus() + hot.EditSender = editProbeHotFile + plan, err := makeEditProbeTrafficPlan(hot, 42, 100, 2) + if err != nil { + t.Fatal(err) + } + if plan.Editors != 2 || plan.Sender != editProbeHotFile || plan.SessionModel != editProbeFixedEditorModel || len(plan.Steps) != 100 { + t.Fatalf("hot-file plan identity: %+v", plan) + } + for k := range 50 { + a, b := plan.Steps[2*k], plan.Steps[2*k+1] + if a.DueNS != b.DueNS || a.DueNS != single.Steps[k].DueNS || a.Editor != 0 || b.Editor != 1 || a.Sender != "" || b.Sender != "" || a.MutationID == "" || b.MutationID == "" || a.Marker == b.Marker { + t.Fatalf("hot-file pair %d is not two MCP edits at one due time: %+v %+v", k, a, b) + } + } + write := editProbeWriteCorpus(t, "go") + write.EditSender = editProbeHotFile + symbol := editProbeSymbolCorpus(t, "go") + symbol.EditSender = editProbeHotFile + rename := editProbeRenameCorpus(t, "go", 100) + rename.EditTool, rename.EditSender = "edit.file", editProbeHotFile + for name, build := range map[string]func() (editProbePlan, error){ + "one_editor": func() (editProbePlan, error) { return makeEditProbePlan(hot, 42, 100) }, + "three_editors": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(hot, 42, 100, 3) }, + "symbol_tool": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(symbol, 42, 100, 2) }, + "rename": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(rename, 42, 100, 2) }, + "write_tool": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(write, 42, 100, 2) }, + } { + if _, err := build(); err == nil { + t.Errorf("%s: hot file planned", name) + } + } +} diff --git a/bench/load/edit_probe_move.go b/bench/load/edit_probe_move.go new file mode 100644 index 000000000..1100b5371 --- /dev/null +++ b/bench/load/edit_probe_move.go @@ -0,0 +1,398 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "errors" + "math" + "math/big" + "strconv" + "strings" +) + +type editProbeMovePath struct{ Relative, Qualified, Absolute string } +type editProbeMoveFileReference struct { + Path string `json:"path"` + Role string `json:"role"` + Receipt string `json:"receipt"` +} +type editProbeMoveReferences struct { + OperationID string `json:"operation_id"` + SourceID string `json:"source_id"` + TargetID string `json:"target_id"` + Files []editProbeMoveFileReference `json:"files"` + ReferenceError string `json:"reference_error,omitempty"` +} + +func movePathMatches(path string, want editProbeMovePath) bool { + return path != "" && (path == want.Relative || path == want.Qualified || path == want.Absolute) +} +func movePositiveSuffix(value, prefix string) bool { + if !strings.HasPrefix(value, prefix) { + return false + } + suffix := strings.TrimPrefix(value, prefix) + n, err := strconv.ParseUint(suffix, 10, 64) + return err == nil && n > 0 && strconv.FormatUint(n, 10) == suffix +} + +// Source acknowledgement and timing references are different facts. Missing +// observations cannot negate a successful correctly identified source move. +func decodeEditProbeMoveAcknowledgement(body map[string]any, sourceID, targetID string, paths map[string]editProbeMovePath) (editProbeMoveReferences, error) { + out := editProbeMoveReferences{SourceID: sourceID, TargetID: targetID} + count, countErr := strconvEditProbeUint(body["files_touched"]) + touched, ok := body["touched"].([]any) + if sourceID == "" || targetID == "" || sourceID == targetID || body["source_id"] != sourceID || body["target_id"] != targetID || body["same_package"] != true || body["dry_run"] != false || countErr != nil || count != 2 || !ok || len(touched) != 2 { + return out, errors.New("move acknowledgement is not the offered same-package two-file source result") + } + seen := map[string]bool{} + for _, raw := range touched { + row, ok := raw.(map[string]any) + if !ok { + return out, errors.New("move touched row malformed") + } + role, _ := row["role"].(string) + path, _ := row["path"].(string) + want, known := paths[role] + if !known || seen[role] || !movePathMatches(path, want) { + return out, errors.New("move touched identity mismatch") + } + seen[role] = true + } + if !seen["source"] || !seen["target"] { + return out, errors.New("move source/target authority incomplete") + } + out.OperationID, _ = body["move_observation_id"].(string) + refs, refsOK := body["file_observations"].([]any) + if !movePositiveSuffix(out.OperationID, "move-") || !refsOK || len(refs) != 2 || body["mutation_receipt"] != nil { + out.ReferenceError = "standalone move observation references unavailable" + return out, nil + } + seen = map[string]bool{} + receipts := map[string]bool{} + for _, raw := range refs { + row, ok := raw.(map[string]any) + if !ok { + out.ReferenceError = "malformed move file observation" + break + } + role, _ := row["role"].(string) + path, _ := row["path"].(string) + receipt, _ := row["receipt"].(string) + want, known := paths[role] + if !known || seen[role] || receipts[receipt] || !movePathMatches(path, want) || !movePositiveSuffix(receipt, "move-file-") { + out.ReferenceError = "foreign/duplicate move observation reference" + break + } + seen[role], receipts[receipt] = true, true + out.Files = append(out.Files, editProbeMoveFileReference{Path: path, Role: role, Receipt: receipt}) + } + if len(out.Files) != 2 || !seen["source"] || !seen["target"] { + out.ReferenceError = "move observation source/target references incomplete" + } + return out, nil +} + +type editProbeMoveFileAccounting struct { + Path string `json:"path"` + Role string `json:"role"` + Receipt string `json:"receipt"` + Blob string `json:"accepted_blob_sha"` + OwnerGeneration uint64 `json:"owner_generation"` + WriteStartedNS *int64 `json:"write_started_ns"` + WriteReturnedNS *int64 `json:"write_returned_ns"` + WriteWindowNS *int64 `json:"write_window_ns"` + D *editProbeWindow `json:"direct_reconcile,omitempty"` + R *editProbeWindow `json:"read_assisted_reconcile,omitempty"` + Route string `json:"publication_route,omitempty"` +} +type editProbeMoveAccounting struct { + State string `json:"state"` + RPCID string `json:"ack_rpc_id"` + A *editProbeWindow `json:"acknowledgement,omitempty"` + Files []editProbeMoveFileAccounting `json:"files"` + Reasons []string `json:"reasons,omitempty"` +} + +func moveBoundedNS(value any) (int64, error) { + n, err := strconvEditProbeUint(value) + if err != nil || n > math.MaxInt64 { + return 0, errors.New("missing/invalid monotonic nanoseconds") + } + return int64(n), nil +} + +// This discriminator is deliberately separate from physical A/P/B receipt +// accounting. No missing direct map can fall back to an unrelated physical P, +// watcher generation, batch transaction or synthetic zero background window. +func checkEditProbeMoveAccounting(statuses map[string]map[string]any, refs editProbeMoveReferences, wireID int64) editProbeMoveAccounting { + out := editProbeMoveAccounting{State: "unavailable"} + fail := func(reason string) { out.Reasons = append(out.Reasons, reason) } + if refs.validateReferences() != nil || wireID <= 0 { + fail("move references/wire identity unavailable") + return out + } + out.RPCID = "rpc-" + strings.TrimPrefix(refs.OperationID, "move-") + ackHash, session := "", "" + for _, ref := range refs.Files { + row := statuses[ref.Role] + file := editProbeMoveFileAccounting{Path: ref.Path, Role: ref.Role, Receipt: ref.Receipt} + if row["found"] != true || row["receipt_kind"] != "move_file_observation" || row["receipt"] != ref.Receipt || row["operation_id"] != refs.OperationID || row["source_id"] != refs.SourceID || row["target_id"] != refs.TargetID || row["path"] != ref.Path || row["role"] != ref.Role { + fail("move file observation identity mismatch") + out.Files = append(out.Files, file) + continue + } + fileReasonsBegin := len(out.Reasons) + if row["move_write_verdict"] != "all_writes_completed" || row["file_write_status"] != "committed" || row["writer_return_observed"] != true { + fail("move source writes not confirmed") + } + if row["requested_background_state"] != "not_applicable_direct_reconcile" || row["requested_background"] != nil { + fail("move direct route/background applicability mismatch") + } + blob, _ := row["new_sha"].(string) + _, blobErr := hex.DecodeString(blob) + file.Blob = blob + if len(blob) != 40 || blobErr != nil { + fail("move accepted blob unavailable") + } + ack, _ := row["acknowledgement"].(map[string]any) + if ack["rpc_id"] != out.RPCID || ack["clock"] != "monotonic" || ack["scope"] != "shared_rpc_mcp_bound_call" || ack["disjoint_observed_intervals"] != true || ack["parse_scope"] != "explicit_go_parser_calls_formatting_unattributed" { + fail("shared move acknowledgement identity unavailable") + } + hash, hashErr := hashJSON(ack) + if hashErr != nil || ackHash != "" && hash != ackHash { + fail("shared move acknowledgement differs across files") + } + if ackHash == "" { + ackHash = hash + } + phases, _ := ack["observed_execution_ms"].(map[string]any) + states, _ := ack["phase_states"].(map[string]any) + applicability, _ := row["acknowledgement_phase_applicability"].(map[string]any) + counts, _ := ack["counts"].(map[string]any) + for _, name := range []string{"caller_result", "disk_write", "edit_control", "middleware", "parse", "publish", "reindex_wait", "reply_prepare"} { + value, present := phases[name] + if name == "publish" || name == "reindex_wait" { + count, _ := counts[name].(map[string]any) + started, se := strconvEditProbeUint(count["started"]) + completed, ce := strconvEditProbeUint(count["completed"]) + na, ne := strconvEditProbeUint(count["not_applicable"]) + if !present || value != nil || states[name] != "unobserved" || applicability[name] != "not_applicable_direct_reconcile" || se != nil || ce != nil || ne != nil || started != 0 || completed != 0 || na != 0 { + fail("move scheduler A phase is not explicitly inapplicable: " + name) + } + } else if !present || value == nil || states[name] != "measured" { + fail("missing applicable move A phase " + name) + } + } + a, err := checkEditProbeWindow(ack["caller_response_ready_ms"], phases) + if err != nil { + fail("move A: " + err.Error()) + } else if out.A == nil { + out.A = &a + } + timing, _ := row["write_timing"].(map[string]any) + start, se := moveBoundedNS(timing["started_ns"]) + returned, re := moveBoundedNS(timing["returned_ns"]) + wall, we := moveBoundedNS(timing["window_ns"]) + if timing["scope"] != "move_file_write" || timing["clock"] != "monotonic" || timing["state"] != "measured" || timing["outcome"] != "committed" || se != nil || re != nil || we != nil || returned < start || wall != returned-start { + fail("move file write window unavailable") + } + if timing["scope"] == "move_file_write" && timing["clock"] == "monotonic" && timing["state"] == "measured" && timing["outcome"] == "committed" && se == nil && re == nil && we == nil && returned >= start && wall == returned-start { + file.WriteStartedNS, file.WriteReturnedNS, file.WriteWindowNS = &start, &returned, &wall + } + route, scope := "direct_reconcile", "move_direct_reconcile" + coalesced := row["direct_reconcile_state"] == "observed" && row["read_assisted_reconcile_state"] == "observed" + var directView map[string]any + if row["read_assisted_reconcile_state"] == "observed" { + route, scope = "read_assisted_reconcile", "move_read_assisted_reconcile" + } + if coalesced { + directView, _ = row["direct_reconcile"].(map[string]any) + repairView, _ := row["read_assisted_reconcile"].(map[string]any) + directPhases, _ := directView["phase_accounting"].(map[string]any) + if directView["scope"] != "move_direct_reconcile" || directPhases["scope"] != "move_direct_reconcile" || !moveSameExecution(directView, repairView) { + fail("conflicting observed move publication routes") + } + } + file.Route = route + if coalesced { + file.Route = "coalesced_direct_and_read_assisted" + } + direct, _ := row[route].(map[string]any) + owner, oe := strconvEditProbeUint(direct["owner_generation"]) + dwall, de := moveBoundedNS(direct["window_ns"]) + toAcceptance, ae := moveBoundedNS(direct["writer_return_to_acceptance_ns"]) + if row[route+"_state"] != "observed" || direct["scope"] != scope || direct["clock"] != "monotonic" || direct["accepted_blob_sha"] != blob || direct["additive_with_acknowledgement"] != false || oe != nil || owner == 0 || de != nil || dwall <= 0 || ae != nil { + fail("move actual accepted owner/window unavailable") + } + if out.A != nil && returned > out.A.WallNS { + fail("move write exceeds actual acknowledgement bounds") + } + if !movePublicationEndpoints(direct, route, dwall, returned, toAcceptance, out.A) { + fail("move actual acceptance/write/window endpoints disagree") + } + if coalesced && !movePublicationEndpoints(directView, "direct_reconcile", dwall, returned, toAcceptance, out.A) { + fail("coalesced direct work exceeds acknowledgement bounds") + } + file.OwnerGeneration = owner + original, _ := row["move_request_cause"].(map[string]any) + originalSession, _ := original["session"].(string) + originalKey, _ := original["request_id_key"].(string) + if originalSession == "" || originalKey != "n:"+strconv.FormatInt(wireID, 10) { + fail("original move wire cause unavailable") + } + if session == "" { + session = originalSession + } else if session != originalSession { + fail("move per-file original causes came from different sessions") + } + request, _ := direct["request_cause"].(map[string]any) + requestSession, _ := request["session"].(string) + requestKey, _ := request["request_id_key"].(string) + requestGeneration, rge := strconvEditProbeUint(request["admission_generation"]) + if requestSession == "" || requestKey == "" || rge != nil || requestGeneration == 0 { + fail("actual move publication request cause unavailable") + } + sameRequest := requestSession == originalSession && requestKey == originalKey + if route == "direct_reconcile" && !sameRequest || route == "read_assisted_reconcile" && sameRequest { + fail("move publication route/request cause mismatch") + } + causes, _ := direct["reconcile_causes"].([]any) + matchedCause := false + matchedDirect := !coalesced + directRequest, _ := directView["request_cause"].(map[string]any) + directGeneration, dge := strconvEditProbeUint(directRequest["admission_generation"]) + if coalesced && (directRequest["session"] != originalSession || directRequest["request_id_key"] != originalKey || dge != nil || directGeneration == 0) { + fail("coalesced direct cause does not match original move") + } + seen := map[uint64]bool{} + if len(causes) == 0 || len(causes) > 8 { + fail("move actual reconcile causes unavailable") + } + for _, raw := range causes { + cause, _ := raw.(map[string]any) + generation, ge := strconvEditProbeUint(cause["admission_generation"]) + s, _ := cause["session"].(string) + key, _ := cause["request_id_key"].(string) + if ge != nil || generation == 0 || generation > owner || seen[generation] || s == "" || key == "" { + fail("invalid move reconcile cause") + } + seen[generation] = true + if s == requestSession && key == requestKey && generation == requestGeneration { + matchedCause = true + } + if coalesced && s == originalSession && key == originalKey && generation == directGeneration { + matchedDirect = true + } + } + if !matchedCause { + fail("move actual request cause not retained") + } + if !matchedDirect { + fail("coalesced original direct cause not retained") + } + accounting, _ := direct["phase_accounting"].(map[string]any) + phaseOwner, poe := strconvEditProbeUint(accounting["owner_generation"]) + phaseWall, pwe := moveBoundedNS(accounting["window_ns"]) + dphases, _ := accounting["phases_ms"].(map[string]any) + dstates, _ := accounting["phase_states"].(map[string]any) + if accounting["scope"] != scope || accounting["clock"] != "monotonic" || accounting["additive_with_requested_background"] != false || accounting["observed_partition_complete"] != true || accounting["sum_state"] != "partial" || poe != nil || phaseOwner != owner || pwe != nil || phaseWall != dwall || len(dphases) != 9 || len(dstates) != 9 { + fail("move own publication phase binding unavailable") + } + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", "repair_control"} { + if dphases[name] == nil || dstates[name] != "measured" { + fail("missing applicable move publication phase " + name) + } + } + d, err := checkEditProbeWindowNS(dwall, dphases) + subtotal, sue := moveBoundedNS(accounting["observed_subtotal_ns"]) + residual, rse := moveBoundedNS(accounting["unattributed_ns"]) + if err != nil || d.ResidualNS != 0 || sue != nil || subtotal != d.SumNS || rse != nil || residual != 0 { + fail("move publication partition disagrees with actual window") + } else if len(out.Reasons) == fileReasonsBegin { + if route == "direct_reconcile" { + file.D = &d + } else { + file.R = &d + } + } + out.Files = append(out.Files, file) + } + if len(out.Reasons) == 0 { + out.State = "observed_move_windows_checks_pass" + } + return out +} + +func (r editProbeMoveReferences) validateReferences() error { + if r.ReferenceError != "" { + return errors.New(r.ReferenceError) + } + if !movePositiveSuffix(r.OperationID, "move-") || len(r.Files) != 2 || r.SourceID == "" || r.TargetID == "" || r.SourceID == r.TargetID { + return errors.New("move references unavailable") + } + roles, receipts, paths := map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, file := range r.Files { + if (file.Role != "source" && file.Role != "target") || roles[file.Role] || file.Path == "" || paths[file.Path] || !movePositiveSuffix(file.Receipt, "move-file-") || receipts[file.Receipt] { + return errors.New("move file references are foreign or repeated") + } + roles[file.Role], receipts[file.Receipt], paths[file.Path] = true, true, true + } + return nil +} + +func moveSignedNS(value any) (int64, error) { + switch n := value.(type) { + case json.Number: + return strconv.ParseInt(string(n), 10, 64) + case int64: + return n, nil + case int: + return int64(n), nil + default: + return 0, errors.New("missing/invalid signed monotonic nanoseconds") + } +} + +// The independent reader window need not begin at the move RPC. Compare exact +// signed offsets without overflow, retain its full window, and bind actual +// acceptance to the original file writer return. Direct work stays inside A. +func movePublicationEndpoints(observed map[string]any, route string, wall, returned, writerToAcceptance int64, ack *editProbeWindow) bool { + start, se := moveSignedNS(observed["started_from_ack_ns"]) + end, ee := moveSignedNS(observed["finished_from_ack_ns"]) + accepted, ae := moveBoundedNS(observed["accepted_from_start_ns"]) + if se != nil || ee != nil || ae != nil || wall <= 0 || accepted > wall { + return false + } + difference := new(big.Int).Sub(big.NewInt(end), big.NewInt(start)) + actual := new(big.Int).Add(big.NewInt(start), big.NewInt(accepted)) + written := new(big.Int).Add(big.NewInt(returned), big.NewInt(writerToAcceptance)) + if difference.Cmp(big.NewInt(wall)) != 0 || !actual.IsInt64() || actual.Cmp(written) != 0 { + return false + } + return route == "read_assisted_reconcile" || ack != nil && start >= 0 && end <= ack.WallNS +} + +// Two admitted requests can witness one coordinator execution. Compare the +// immutable execution facts (not route labels/request-specific admission) and +// count that window once; differing owners, clocks, bytes or phases refuse. +func moveSameExecution(direct, repair map[string]any) bool { + identity := func(row map[string]any) map[string]any { + out := make(map[string]any) + for _, key := range []string{"clock", "owner_generation", "window_ns", "accepted_blob_sha", "writer_return_to_acceptance_ns", "started_from_ack_ns", "finished_from_ack_ns", "accepted_from_start_ns", "additive_with_acknowledgement", "reconcile_causes"} { + out[key] = row[key] + } + phases, _ := row["phase_accounting"].(map[string]any) + phaseCopy := make(map[string]any, len(phases)) + for key, value := range phases { + if key != "scope" { + phaseCopy[key] = value + } + } + out["phase_accounting"] = phaseCopy + return out + } + a, ae := hashJSON(identity(direct)) + b, be := hashJSON(identity(repair)) + return ae == nil && be == nil && a == b +} diff --git a/bench/load/edit_probe_move_fixture.go b/bench/load/edit_probe_move_fixture.go new file mode 100644 index 000000000..023bd65a5 --- /dev/null +++ b/bench/load/edit_probe_move_fixture.go @@ -0,0 +1,152 @@ +package main + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "strings" +) + +func editProbeMoveFixture(c Corpus) error { + if filepath.Ext(c.File) != ".go" || filepath.Base(c.File) != c.File || filepath.Ext(c.EditDestination) != ".go" || filepath.Base(c.EditDestination) != c.EditDestination || c.File == c.EditDestination { + return errors.New("move fixture requires distinct root Go files in the same package") + } + if err := editProbeInitialFixture(c); err != nil { + return err + } + source, err := parser.ParseFile(token.NewFileSet(), c.File, c.EditInitialContent, 0) + if err != nil { + return err + } + target, err := parser.ParseFile(token.NewFileSet(), c.EditDestination, c.EditDestinationContent, 0) + if err != nil { + return err + } + if source.Name.Name != target.Name.Name { + return errors.New("move fixture requires exact same-package source and destination") + } + return nil +} +func editProbeMoveOracles(c Corpus, step editProbeStep) []editProbeOracle { + target := map[string]any{"symbol": step.TargetSymbol} + file := map[string]any{"file": step.DestinationFile} + name := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + return []editProbeOracle{ + {"source", request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": target, "options": map[string]any{"context_lines": 0}}}}, + {"summary", request{Tool: "read", Operation: "summary", Arguments: map[string]any{"operation": "summary", "target": file}}}, + {"editing_context", request{Tool: "read", Operation: "editing_context", Arguments: map[string]any{"operation": "editing_context", "target": file}}}, + {"search_visible", request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": name, "options": map[string]any{"limit": 1000, "max_per_file": 0, "query_class": "symbol", "expand": "off"}}}}, + {"text", request{Tool: "search", Operation: "text", Arguments: map[string]any{"operation": "text", "query": "func " + name, "options": map[string]any{"limit": 1000}}}}, + {"callers", request{Tool: "relations", Operation: "callers", Arguments: map[string]any{"operation": "callers", "target": map[string]any{"symbol": c.Symbol}, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + {"call_chain", request{Tool: "trace", Operation: "call_chain", Arguments: map[string]any{"operation": "call_chain", "target": target, "options": map[string]any{"depth": 1, "limit": 1000}}}}, + {"old_source_absent", request{Tool: "read", Operation: "source", ExpectedAbsentSymbol: step.BeforeSymbol, Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": step.BeforeSymbol}}}}, + } +} +func editProbeMoveCorrect(oracle string, body map[string]any, c Corpus, step editProbeStep, callee string) bool { + name := step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:] + if oracle == "source_file_after_move" { + path, _ := body["path"].(string) + source, _ := body["content"].(string) + if !editProbeFixturePath(path, c.File) { + return false + } + file, err := parser.ParseFile(token.NewFileSet(), "source.go", source, 0) + if err != nil { + return false + } + survivor := false + for _, decl := range file.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok { + if fn.Name.Name == name { + return false + } + if fn.Name.Name == callee { + survivor = true + } + } + } + return survivor + } + switch oracle { + case "source": + id, _ := body["id"].(string) + path, _ := body["file_path"].(string) + source, _ := body["source"].(string) + return body["name"] == name && editProbeSourceID(id, step.DestinationFile, name) && editProbeFixturePath(path, step.DestinationFile) && editProbeFixtureSource(step.DestinationFile, source, name, callee) + case "file": + path, _ := body["path"].(string) + source, _ := body["content"].(string) + return editProbeFixturePath(path, step.DestinationFile) && editProbeFixtureSource(step.DestinationFile, source, name, callee) + case "summary", "editing_context", "search_visible": + key := map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"}[oracle] + rows, ok := body[key].([]any) + if !ok { + return false + } + found := false + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if editProbeSourceID(id, c.File, name) { + return false + } + if editProbeSourceID(id, step.DestinationFile, name) { + found = true + } + } + return found + case "text": + rows, _ := body["matches"].([]any) + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + continue + } + path, _ := row["path"].(string) + id, _ := row["symbol_id"].(string) + text, _ := row["text"].(string) + if editProbeFixturePath(path, step.DestinationFile) && editProbeSourceID(id, step.DestinationFile, name) && editProbeSourceCall(text, name, callee) { + return true + } + } + case "callers", "call_chain": + truncated, ok := body["truncated"].(bool) + nodes, nok := editProbeCompleteCount(body, "nodes", "total_nodes") + edges, eok := editProbeCompleteCount(body, "edges", "total_edges") + if !ok || truncated || !nok || !eok { + return false + } + for _, raw := range nodes { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if editProbeSourceID(id, c.File, name) { + return false + } + } + found := false + for _, raw := range edges { + row, ok := raw.(map[string]any) + if !ok { + return false + } + from, _ := row["from"].(string) + to, _ := row["to"].(string) + if editProbeSourceID(from, c.File, name) || editProbeSourceID(to, c.File, name) { + return false + } + if row["kind"] == "calls" && editProbeSourceID(from, step.DestinationFile, name) && editProbeSourceID(to, c.File, callee) { + found = true + } + } + return found + } + return false +} diff --git a/bench/load/edit_probe_move_fixture_test.go b/bench/load/edit_probe_move_fixture_test.go new file mode 100644 index 000000000..b27636c54 --- /dev/null +++ b/bench/load/edit_probe_move_fixture_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + "strings" + "testing" +) + +func moveProbeCorpus(seed int64) Corpus { + c := editProbeCorpus() + c.EditTool, c.EditKind = "refactor.move", "relocate" + c.EditDestination = "destination.go" + c.EditDestinationContent = "package main\n\nfunc DestinationAnchor() int { return 7 }\n" + c.EditInitialContent = "package main\n\nfunc Example() int { return 1 }\n\n" + c.EditAnchor + "\n" + for i := range 100 { + name := fmt.Sprintf("GortexEditProbe%016xN%03dBefore", uint64(seed), i) + c.EditInitialContent += strings.ReplaceAll(c.EditTemplate, "{{marker}}", name) + } + return c +} + +func TestMovePlanPreservesFullScheduleAndDistinctSourceDestination(t *testing.T) { + c := moveProbeCorpus(42) + p, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if len(p.Steps) != 100 { + t.Fatal("lost offered moves") + } + for _, step := range p.Steps { + name := step.Marker + "Before" + if !step.Relocated || step.BeforeSymbol != c.File+"::"+name || step.TargetSymbol != c.EditDestination+"::"+name || step.Request.Tool != "refactor" || step.Request.Operation != "move" || step.Request.Arguments["destination"] != c.EditDestination || step.Request.Arguments["options"] != nil { + t.Fatalf("move changed source authority: %+v", step) + } + if len(editProbeOracles(c, step)) != 8 { + t.Fatal("missing positive or explicit absence oracle") + } + } +} + +func TestMovePlanRefusesUnownedOrInapplicableFixture(t *testing.T) { + for _, name := range []string{"nested_source", "nested_target", "same_file", "foreign_package", "preexisting_destination", "missing_source", "wrong_kind"} { + t.Run(name, func(t *testing.T) { + c := moveProbeCorpus(42) + switch name { + case "nested_source": + c.File = "nested/sample.go" + case "nested_target": + c.EditDestination = "nested/destination.go" + case "same_file": + c.EditDestination = c.File + case "foreign_package": + c.EditDestinationContent = "package other\n" + case "preexisting_destination": + c.EditDestinationContent += "func GortexEditProbe000000000000002aN000Before() int { return 1 }\n" + case "missing_source": + c.EditInitialContent = strings.ReplaceAll(c.EditInitialContent, "GortexEditProbe000000000000002aN000Before", "Other") + case "wrong_kind": + c.EditKind = "append_leaf" + } + if _, err := makeEditProbePlan(c, 42, 100); err == nil { + t.Fatal("unsupported move fixture admitted") + } + }) + } +} diff --git a/bench/load/edit_probe_move_run.go b/bench/load/edit_probe_move_run.go new file mode 100644 index 000000000..a4ddf751b --- /dev/null +++ b/bench/load/edit_probe_move_run.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "strings" +) + +func (r *editProbeRuntime) movePaths() map[string]editProbeMovePath { + out := make(map[string]editProbeMovePath, 2) + for role, path := range map[string]string{"source": r.plan.File, "target": r.p.Repos[0].Corpus.EditDestination} { + out[role] = editProbeMovePath{Relative: path, Qualified: r.p.Repos[0].Prefix + "/" + path, Absolute: filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(path))} + } + return out +} + +func movePublicFactsComplete(record editProbeRecord) bool { + if len(record.Readbacks) != 4 || len(record.FirstCorrect) != 8 { + return false + } + readbacks := map[string]bool{"file": false, "source": false, "old_source_absent": false, "source_file_after_move": false} + for _, readback := range record.Readbacks { + seen, known := readbacks[readback.Kind] + if !known || seen || !readback.Correct { + return false + } + readbacks[readback.Kind] = true + } + return moveProbesComplete(record) +} + +func moveProbesComplete(record editProbeRecord) bool { + if len(record.FirstCorrect) != 8 { + return false + } + for _, name := range []string{"source", "summary", "editing_context", "search_visible", "text", "callers", "call_chain", "old_source_absent"} { + first, present := record.FirstCorrect[name] + if !present || first.Value == nil { + return false + } + } + return true +} + +// Only the original, exactly identified per-file owner can freeze a terminal +// refusal. Missing/foreign rows and late pending callbacks remain collecting. +func moveOwnedRefusal(row map[string]any, refs editProbeMoveReferences, file editProbeMoveFileReference) bool { + direct, _ := row["direct_reconcile_state"].(string) + repair, _ := row["read_assisted_reconcile_state"].(string) + immutable := direct == "observed" || repair == "observed" || strings.HasPrefix(direct, "unavailable_") && strings.HasPrefix(repair, "unavailable_") + return row["found"] == true && row["receipt_kind"] == "move_file_observation" && row["receipt"] == file.Receipt && row["operation_id"] == refs.OperationID && row["source_id"] == refs.SourceID && row["target_id"] == refs.TargetID && row["path"] == file.Path && row["role"] == file.Role && immutable +} + +// Own-ACK readbacks precede these separate observation queries. Shared A is +// validated once across the two original files; no physical P/B fallback exists. +func (r *editProbeRuntime) collectMoveEvidence(ctx context.Context, s *editProbeState, refs editProbeMoveReferences, wireID int64) { + for ctx.Err() == nil { + statuses := make(map[string]map[string]any, 2) + refused := false + if refs.validateReferences() == nil { + for i, file := range refs.Files { + req := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": file.Receipt}}} + raw, _, err := r.perform(ctx, s, s.client, req, "move_file_status:"+file.Role, "probe", 200+i) + if errors.Is(err, errEditProbeEvidenceIncomplete) { + return + } + if err == nil { + row, decodeErr := decodeEditProbeResult(raw) + if decodeErr == nil { + statuses[file.Role] = row + refused = refused || moveOwnedRefusal(row, refs, file) + } + } + } + } + accounting := checkEditProbeMoveAccounting(statuses, refs, wireID) + s.mu.Lock() + s.record.MoveAccounting = &accounting + all := movePublicFactsComplete(s.record) + if accounting.State == "observed_move_windows_checks_pass" { + s.record.GraphTerminal = true + r.settlePendingLocked(s) + s.record.GraphTerminalBasis = "original_move_files_actual_accepted_reconcile" + } else if !r.requireAccounting && all { + // Old controls may have no instrumentation. Public facts establish only + // functional completion, never an observed graph receipt or zero phases. + s.record.GraphTerminalBasis = "baseline_complete_public_move_facts_no_receipt_proof" + } + done := moveProbesComplete(s.record) && (!r.requireAccounting || accounting.State == "observed_move_windows_checks_pass" || refused || refs.ReferenceError != "") + s.mu.Unlock() + if done { + return + } + if waitContext(ctx, editProbePeriod) != nil { + return + } + } +} diff --git a/bench/load/edit_probe_move_run_test.go b/bench/load/edit_probe_move_run_test.go new file mode 100644 index 000000000..1481f6460 --- /dev/null +++ b/bench/load/edit_probe_move_run_test.go @@ -0,0 +1,282 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "sync" + "testing" + "time" +) + +func completeMovePublicRecord() editProbeRecord { + record := editProbeRecord{FirstCorrect: make(map[string]Metric)} + for _, name := range []string{"source", "summary", "editing_context", "search_visible", "text", "callers", "call_chain", "old_source_absent"} { + record.FirstCorrect[name] = measured(1, "ms") + } + for _, name := range []string{"file", "source", "old_source_absent", "source_file_after_move"} { + record.Readbacks = append(record.Readbacks, editProbeReadback{Kind: name, Correct: true}) + } + return record +} + +func TestMoveCollectorQueriesOriginalFilesAndKeepsASeparate(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + var mu sync.Mutex + var queried []string + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + options, _ := args["options"].(map[string]any) + receipt, _ := options["receipt"].(string) + mu.Lock() + queried = append(queried, receipt) + mu.Unlock() + for _, file := range refs.Files { + if file.Receipt == receipt { + return stubContent(statuses[file.Role]), nil + } + } + return stubContent(map[string]any{"found": false}), nil + }) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + sink, err := os.CreateTemp(t.TempDir(), "move-evidence-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer sink.Close() + state := &editProbeState{client: client, sink: sink, record: completeMovePublicRecord()} + state.startRaw() + defer state.finishRaw() + run := &editProbeRuntime{options: runOptions{Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Prefix: "fixture"}}}, phase: &Phase{Metrics: map[string]Metric{}}, origin: time.Now(), requireAccounting: true} + run.collectMoveEvidence(ctx, state, refs, 17) + if ctx.Err() != nil || !state.record.GraphTerminal || state.record.MoveAccounting == nil || state.record.MoveAccounting.State != "observed_move_windows_checks_pass" || state.record.Receipt != "" || state.record.Accounting.A != nil { + t.Fatalf("standalone collector borrowed physical timing or failed: %+v %v", state.record, ctx.Err()) + } + mu.Lock() + defer mu.Unlock() + if len(queried) != 2 || queried[0] != refs.Files[0].Receipt || queried[1] != refs.Files[1].Receipt { + t.Fatal(queried) + } +} + +func TestMoveCollectorCannotBorrowForeignOrPendingRefusal(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + file := refs.Files[0] + row := statuses[file.Role] + row["direct_reconcile_state"] = "pending_observer" + if moveOwnedRefusal(row, refs, file) { + t.Fatal("pending callback treated as terminal") + } + row["direct_reconcile_state"] = "unavailable_competing_commit" + if moveOwnedRefusal(row, refs, file) { + t.Fatal("missing late read-assisted route treated as irrevocable refusal") + } + row["read_assisted_reconcile_state"] = "unavailable_owner_bytes_or_acceptance" + if !moveOwnedRefusal(row, refs, file) { + t.Fatal("owned immutable refusal not retained") + } + row["operation_id"] = "move-999" + if moveOwnedRefusal(row, refs, file) { + t.Fatal("foreign refusal ended collection") + } +} + +func TestMoveBaselinePublicCompletionNeverInventsReceiptOrMeasuredPhases(t *testing.T) { + refs, _ := moveAccountingFixture(t) + refs.Files, refs.ReferenceError = nil, "standalone move observation references unavailable" + state := &editProbeState{record: completeMovePublicRecord()} + run := &editProbeRuntime{} + run.collectMoveEvidence(t.Context(), state, refs, 17) + if state.record.GraphTerminal || state.record.GraphTerminalBasis != "baseline_complete_public_move_facts_no_receipt_proof" || state.record.MoveAccounting == nil || state.record.MoveAccounting.State != "unavailable" || state.record.MoveAccounting.A != nil { + t.Fatal(state.record) + } + state.record.Readbacks[0].Correct = false + if movePublicFactsComplete(state.record) { + t.Fatal("incorrect public readback qualified baseline") + } + if !moveProbesComplete(state.record) { + t.Fatal("failed immutable readback erased later probe evidence") + } +} + +func TestMoveRestoreUsesOnePublicBatchForBothOwnedFiles(t *testing.T) { + root := t.TempDir() + source, target := filepath.Join(root, "source.go"), filepath.Join(root, "target.go") + sourceNow, targetNow := "package fixture\nfunc Survives() {}\n", "package fixture\nfunc Moved() {}\n" + sourceBefore, targetBefore := "package fixture\nfunc Moved() {}\nfunc Survives() {}\n", "package fixture\n" + if err := os.WriteFile(source, []byte(sourceNow), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte(targetNow), 0600); err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var received []any + var handlerErr error + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + changes, _ := args["changes"].([]any) + mu.Lock() + defer mu.Unlock() + received = changes + for _, raw := range changes { + change, _ := raw.(map[string]any) + path, _ := change["path"].(string) + before, _ := change["old_string"].(string) + after, _ := change["new_string"].(string) + var absolute, want string + switch path { + case "fixture/source.go": + absolute, want = source, sourceNow + case "fixture/target.go": + absolute, want = target, targetNow + default: + handlerErr = errors.New("unowned restore path") + continue + } + if change["op"] != "edit_file" || before != want { + handlerErr = errors.New("restore lost exact current bytes") + continue + } + if err := os.WriteFile(absolute, []byte(after), 0600); err != nil { + handlerErr = err + } + } + return stubContent(map[string]any{"status": "committed"}), nil + }) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + refs, _ := moveAccountingFixture(t) + run := &editProbeRuntime{options: runOptions{Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Path: root, Prefix: "fixture", Corpus: Corpus{EditDestination: "target.go"}}}}, editors: []*socketClient{client}, plan: editProbePlan{Tool: "refactor.move", File: "source.go"}, original: []byte(sourceBefore), moveOriginal: []byte(targetBefore), requireAccounting: true, phase: &Phase{EditProbe: &editProbeReport{Records: []editProbeRecord{{Move: &refs}}}}} + if err := run.restore(ctx); err == nil { + t.Fatal("unknown publication allowed cleanup mutation") + } + mu.Lock() + called := len(received) + mu.Unlock() + if called != 0 { + t.Fatal("pending outcome reached public restore") + } + run.phase.EditProbe.Records[0].GraphTerminal = true + if err := run.restore(ctx); err != nil { + t.Fatal(err) + } + mu.Lock() + defer mu.Unlock() + if handlerErr != nil || len(received) != 2 { + t.Fatal(received, handlerErr) + } + actualSource, err := os.ReadFile(source) + if err != nil { + t.Fatal(err) + } + actualTarget, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if string(actualSource) != sourceBefore || string(actualTarget) != targetBefore { + t.Fatal("two-file restore lost bytes") + } +} + +// Exercise the public continuation, not just the status helper: no observation +// request may precede the four immediate own-ACK readbacks. +func TestMoveOwnACKReadbacksPrecedeBothOriginalOwnerQueries(t *testing.T) { + t.Run("correct", func(t *testing.T) { checkMoveOwnACKContinuation(t, false) }) + t.Run("immutable_bad_readback", func(t *testing.T) { checkMoveOwnACKContinuation(t, true) }) +} + +func checkMoveOwnACKContinuation(t *testing.T, badReadback bool) { + t.Helper() + refs, statuses := moveAccountingFixture(t) + ackBody, _ := moveAckFixture() + var mu sync.Mutex + var order []string + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + operation, _ := args["operation"].(string) + target, _ := args["target"].(map[string]any) + options, _ := args["options"].(map[string]any) + var body any + key := operation + if receipt, ok := options["receipt"].(string); ok { + key = receipt + for _, file := range refs.Files { + if file.Receipt == receipt { + body = stubContent(statuses[file.Role]) + } + } + } else if operation == "file" && target["file"] == "fixture/target.go" { + key = "destination_file" + body = stubContent(map[string]any{"path": "fixture/target.go", "content": "package main\nfunc Relocated() int { return Callee() }\n"}) + if badReadback { + body = stubContent(map[string]any{"path": "fixture/target.go", "content": "package main\n// stale destination\n"}) + } + } else if operation == "source" && target["symbol"] == "fixture/target.go::Relocated" { + key = "new_source" + body = stubContent(map[string]any{"id": "fixture/target.go::Relocated", "name": "Relocated", "file_path": "fixture/target.go", "source": "func Relocated() int { return Callee() }"}) + } else if operation == "source" && target["symbol"] == "fixture/source.go::Relocated" { + key = "old_source_absent" + body = map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": "symbol not found: fixture/source.go::Relocated — did you mean: fixture/target.go::Relocated, fixture/source.go::Callee"}}} + } else if operation == "file" && target["file"] == "fixture/source.go" { + key = "source_file" + body = stubContent(map[string]any{"path": "fixture/source.go", "content": "package main\nfunc Callee() int { return 1 }\n"}) + } + mu.Lock() + order = append(order, key) + mu.Unlock() + return body, nil + }) + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + sink, err := os.CreateTemp(t.TempDir(), "move-continuation-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer sink.Close() + record := completeMovePublicRecord() + record.Readbacks = nil + started := time.Now().Add(-time.Millisecond) + step := editProbeStep{Relocated: true, BeforeSymbol: "source.go::Relocated", TargetSymbol: "target.go::Relocated", DestinationFile: "target.go"} + state := &editProbeState{step: step, client: client, sink: sink, record: record, firstProbeWrite: started.Add(-time.Millisecond), editWrite: started, absenceBefore: map[string]bool{step.BeforeSymbol: true}} + state.startRaw() + defer state.finishRaw() + run := &editProbeRuntime{options: runOptions{Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Path: "/owned", Prefix: "fixture", Corpus: Corpus{File: "source.go", EditDestination: "target.go"}}}}, editors: []*socketClient{client}, plan: editProbePlan{Tool: "refactor.move", File: "source.go", Callee: "Callee"}, phase: &Phase{Metrics: map[string]Metric{}}, origin: started, requireAccounting: true} + ack, err := json.Marshal(stubContent(ackBody)) + if err != nil { + t.Fatal(err) + } + run.finishEditAfterACK(ctx, ctx, state, step, started, ack, wireCallIdentity{ID: 17, Finished: time.Now()}, nil) + if badReadback { + if ctx.Err() != nil || state.record.Outcome != "diagnostic_failure" || len(state.record.Errors) != 1 || state.record.Errors[0] != "immediate file readback incorrect" || state.record.Readbacks[0].Correct || !state.record.GraphTerminal || state.record.MoveAccounting == nil || state.record.MoveAccounting.State != "observed_move_windows_checks_pass" { + t.Fatalf("immutable failed readback was lost or polled forever: %+v %v", state.record, ctx.Err()) + } + } else if state.record.Outcome != "complete" || !state.record.GraphTerminal || len(state.record.Errors) != 0 { + t.Fatalf("continuation: %+v", state.record) + } + mu.Lock() + defer mu.Unlock() + want := []string{"destination_file", "new_source", "old_source_absent", "source_file", refs.Files[0].Receipt, refs.Files[1].Receipt} + if !reflect.DeepEqual(order, want) { + t.Fatalf("own-ACK continuation order=%v want=%v", order, want) + } +} diff --git a/bench/load/edit_probe_move_test.go b/bench/load/edit_probe_move_test.go new file mode 100644 index 000000000..d3eded4be --- /dev/null +++ b/bench/load/edit_probe_move_test.go @@ -0,0 +1,253 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func moveAckFixture() (map[string]any, map[string]editProbeMovePath) { + paths := map[string]editProbeMovePath{"source": {"source.go", "fixture/source.go", "/owned/source.go"}, "target": {"target.go", "fixture/target.go", "/owned/target.go"}} + body := map[string]any{"source_id": "fixture/source.go::Relocated", "target_id": "fixture/target.go::Relocated", "same_package": true, "dry_run": false, "files_touched": json.Number("2"), "touched": []any{map[string]any{"path": "fixture/source.go", "role": "source"}, map[string]any{"path": "fixture/target.go", "role": "target"}}, "move_observation_id": "move-31", "file_observations": []any{map[string]any{"path": "fixture/source.go", "role": "source", "receipt": "move-file-52"}, map[string]any{"path": "fixture/target.go", "role": "target", "receipt": "move-file-53"}}} + return body, paths +} +func moveAccountingFixture(t *testing.T) (editProbeMoveReferences, map[string]map[string]any) { + t.Helper() + body, paths := moveAckFixture() + refs, err := decodeEditProbeMoveAcknowledgement(body, "fixture/source.go::Relocated", "fixture/target.go::Relocated", paths) + if err != nil { + t.Fatal(err) + } + statuses := map[string]map[string]any{} + for i, ref := range refs.Files { + phases, states, counts := map[string]any{}, map[string]any{}, map[string]any{} + for _, name := range []string{"caller_result", "disk_write", "edit_control", "middleware", "parse", "publish", "reindex_wait", "reply_prepare"} { + phases[name], states[name] = json.Number("10"), "measured" + counts[name] = map[string]any{"started": json.Number("1"), "completed": json.Number("1"), "not_applicable": json.Number("0")} + } + for _, name := range []string{"publish", "reindex_wait"} { + phases[name], states[name] = nil, "unobserved" + counts[name] = map[string]any{"started": json.Number("0"), "completed": json.Number("0"), "not_applicable": json.Number("0")} + } + ack := map[string]any{"rpc_id": "rpc-31", "scope": "shared_rpc_mcp_bound_call", "clock": "monotonic", "parse_scope": "explicit_go_parser_calls_formatting_unattributed", "disjoint_observed_intervals": true, "caller_response_ready_ms": json.Number("60"), "observed_execution_ms": phases, "phase_states": states, "counts": counts} + owner := json.Number("42") + blob := strings.Repeat("a", 40) + if i == 1 { + owner = json.Number("43") + blob = strings.Repeat("b", 40) + } + dp, ds := map[string]any{}, map[string]any{} + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", "repair_control"} { + dp[name], ds[name] = json.Number("1"), "measured" + } + direct := map[string]any{"scope": "move_direct_reconcile", "clock": "monotonic", "owner_generation": owner, "window_ns": json.Number("9000000"), "writer_return_to_acceptance_ns": json.Number("2000000"), "accepted_blob_sha": blob, "additive_with_acknowledgement": false, "reconcile_causes": []any{map[string]any{"session": "actual-session", "request_id_key": "n:17", "admission_generation": json.Number("7")}}, "phase_accounting": map[string]any{"scope": "move_direct_reconcile", "clock": "monotonic", "owner_generation": owner, "window_ns": json.Number("9000000"), "observed_subtotal_ns": json.Number("9000000"), "unattributed_ns": json.Number("0"), "sum_state": "partial", "observed_partition_complete": true, "additive_with_requested_background": false, "phases_ms": dp, "phase_states": ds}} + direct["request_cause"] = map[string]any{"session": "actual-session", "request_id_key": "n:17", "admission_generation": json.Number("7")} + direct["started_from_ack_ns"], direct["finished_from_ack_ns"], direct["accepted_from_start_ns"] = json.Number("4000000"), json.Number("13000000"), json.Number("1000000") + statuses[ref.Role] = map[string]any{"found": true, "receipt_kind": "move_file_observation", "receipt": ref.Receipt, "operation_id": refs.OperationID, "source_id": refs.SourceID, "target_id": refs.TargetID, "path": ref.Path, "role": ref.Role, "move_write_verdict": "all_writes_completed", "file_write_status": "committed", "writer_return_observed": true, "new_sha": blob, "requested_background_state": "not_applicable_direct_reconcile", "acknowledgement_phase_applicability": map[string]any{"publish": "not_applicable_direct_reconcile", "reindex_wait": "not_applicable_direct_reconcile"}, "acknowledgement": ack, "write_timing": map[string]any{"scope": "move_file_write", "clock": "monotonic", "state": "measured", "outcome": "committed", "started_ns": json.Number("1000000"), "returned_ns": json.Number("3000000"), "window_ns": json.Number("2000000")}, "direct_reconcile_state": "observed", "direct_reconcile": direct} + statuses[ref.Role]["move_request_cause"] = map[string]any{"session": "actual-session", "request_id_key": "n:17"} + } + return refs, statuses +} + +func TestMoveConsumerKeepsSourceAcknowledgementSeparateFromTimingReferences(t *testing.T) { + for _, variant := range []string{"valid", "missing_refs", "legacy_receipt", "foreign_ref", "duplicate_role", "wrong_source"} { + t.Run(variant, func(t *testing.T) { + body, paths := moveAckFixture() + switch variant { + case "missing_refs": + delete(body, "file_observations") + case "legacy_receipt": + body["mutation_receipt"] = "physical-foreign" + case "foreign_ref": + body["file_observations"].([]any)[0].(map[string]any)["path"] = "other/source.go" + case "duplicate_role": + body["file_observations"].([]any)[1].(map[string]any)["role"] = "source" + case "wrong_source": + body["source_id"] = "other/source.go::Relocated" + } + refs, err := decodeEditProbeMoveAcknowledgement(body, "fixture/source.go::Relocated", "fixture/target.go::Relocated", paths) + if variant == "wrong_source" { + if err == nil { + t.Fatal("foreign source authority accepted") + } + return + } + if err != nil { + t.Fatalf("timing loss negated genuine source acknowledgement: %v", err) + } + if (refs.validateReferences() == nil) != (variant == "valid") { + t.Fatalf("reference validation wrong: %+v", refs) + } + }) + } +} +func TestMoveConsumerUsesSeparateOwnedAWriteAndDirectWindows(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + got := checkEditProbeMoveAccounting(statuses, refs, 17) + if got.State != "observed_move_windows_checks_pass" || got.A == nil || got.A.WallNS != 60000000 || len(got.Files) != 2 { + t.Fatalf("owned move accounting refused/doubled shared A: %+v", got) + } + if got.Files[0].OwnerGeneration == 31 || got.Files[0].OwnerGeneration == 17 { + t.Fatal("independent executor owner was replaced by ACK/wire domain") + } + for _, file := range got.Files { + if file.D == nil || file.D.WallNS != 9000000 || file.D.ResidualNS != 0 || (file.WriteWindowNS == nil || *file.WriteWindowNS != 2000000) { + t.Fatalf("bad own-file windows: %+v", file) + } + } +} +func TestMoveConsumerRefusesMissingStandaloneMapsForeignOwnersAndInventedZero(t *testing.T) { + for _, variant := range []string{"missing_kind", "physical_status", "missing_A", "wrong_A_owner", "missing_scheduler_key", "measured_scheduler_zero", "missing_applicability", "partial_source", "missing_write_map", "wrong_write_wall", "missing_direct_map", "foreign_blob", "foreign_phase_owner", "foreign_wire", "split_session", "missing_deep_map", "missing_deep_phase", "unmeasured_deep_phase", "nonzero_residual", "outside_A", "legacy_maps_only", "duplicate_file_role"} { + t.Run(variant, func(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + row := statuses["source"] + ack := row["acknowledgement"].(map[string]any) + direct := row["direct_reconcile"].(map[string]any) + accounting := direct["phase_accounting"].(map[string]any) + switch variant { + case "missing_kind": + delete(row, "receipt_kind") + case "physical_status": + row["receipt_kind"] = "physical_mutation" + case "missing_A": + delete(row, "acknowledgement") + case "wrong_A_owner": + ack["rpc_id"] = "rpc-foreign" + case "missing_scheduler_key": + delete(ack["observed_execution_ms"].(map[string]any), "publish") + case "measured_scheduler_zero": + ack["observed_execution_ms"].(map[string]any)["publish"] = json.Number("0") + ack["phase_states"].(map[string]any)["publish"] = "measured" + case "missing_applicability": + delete(row, "acknowledgement_phase_applicability") + case "partial_source": + row["move_write_verdict"] = "partial_or_failed" + case "missing_write_map": + delete(row, "write_timing") + case "wrong_write_wall": + row["write_timing"].(map[string]any)["window_ns"] = json.Number("1") + case "missing_direct_map": + delete(row, "direct_reconcile") + case "foreign_blob": + direct["accepted_blob_sha"] = strings.Repeat("c", 40) + case "foreign_phase_owner": + accounting["owner_generation"] = json.Number("9") + case "foreign_wire": + direct["reconcile_causes"].([]any)[0].(map[string]any)["request_id_key"] = "n:18" + case "split_session": + statuses["target"]["direct_reconcile"].(map[string]any)["reconcile_causes"].([]any)[0].(map[string]any)["session"] = "foreign-session" + case "missing_deep_map": + delete(accounting, "phases_ms") + case "missing_deep_phase": + delete(accounting["phases_ms"].(map[string]any), "resolve") + case "unmeasured_deep_phase": + accounting["phase_states"].(map[string]any)["resolve"] = "unobserved" + case "nonzero_residual": + accounting["unattributed_ns"] = json.Number("1") + case "outside_A": + direct["writer_return_to_acceptance_ns"] = json.Number("60000000") + case "legacy_maps_only": + delete(row, "direct_reconcile") + row["phase_timing"] = editProbeRepairAccountingFixture()["phase_timing"] + case "duplicate_file_role": + refs.Files[1] = refs.Files[0] + } + got := checkEditProbeMoveAccounting(statuses, refs, 17) + if got.State == "observed_move_windows_checks_pass" || len(got.Reasons) == 0 { + t.Fatalf("%s falsely supplied move coverage: %+v", variant, got) + } + }) + } +} + +func moveReadAssistedFixture(row map[string]any) map[string]any { + repair := row["direct_reconcile"].(map[string]any) + delete(row, "direct_reconcile") + row["direct_reconcile_state"] = "unavailable_owner_bytes_or_acceptance" + row["read_assisted_reconcile_state"], row["read_assisted_reconcile"] = "observed", repair + repair["scope"] = "move_read_assisted_reconcile" + repair["phase_accounting"].(map[string]any)["scope"] = "move_read_assisted_reconcile" + cause := map[string]any{"session": "independent-reader", "request_id_key": "n:23", "admission_generation": json.Number("7")} + repair["request_cause"], repair["reconcile_causes"] = cause, []any{cause} + return repair +} + +func TestMoveConsumerNamesActualReadAssistedRouteWithoutInventingDirectWork(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + repair := moveReadAssistedFixture(statuses["target"]) + // Independent R can start before shared A. Retain its full non-additive90ms + // window; actual acceptance5ms after A entry still follows writer return3ms. + repair["started_from_ack_ns"], repair["finished_from_ack_ns"], repair["accepted_from_start_ns"] = json.Number("-40000000"), json.Number("50000000"), json.Number("45000000") + repair["window_ns"] = json.Number("90000000") + phase := repair["phase_accounting"].(map[string]any) + phase["window_ns"], phase["observed_subtotal_ns"] = json.Number("90000000"), json.Number("90000000") + for name := range phase["phases_ms"].(map[string]any) { + phase["phases_ms"].(map[string]any)[name] = json.Number("10") + } + got := checkEditProbeMoveAccounting(statuses, refs, 17) + if got.State != "observed_move_windows_checks_pass" || len(got.Files) != 2 || got.Files[0].Route != "direct_reconcile" || got.Files[1].Route != "read_assisted_reconcile" || got.Files[1].D != nil || got.Files[1].R == nil || got.Files[1].R.WallNS != 90000000 || got.A == nil || got.A.WallNS != 60000000 { + t.Fatalf("borrowed or truncated publication window: %+v", got) + } +} + +func TestMoveReadAssistedConsumerRejectsForeignCauseAndInvalidActualAcceptance(t *testing.T) { + for _, name := range []string{"missing_move_cause", "foreign_move_wire", "missing_actual_cause", "ack_cause_as_reader", "foreign_admission", "missing_window_start", "missing_acceptance", "acceptance_outside_window", "acceptance_before_write", "acceptance_disagrees", "wrong_route_scope", "both_routes"} { + t.Run(name, func(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + row := statuses["target"] + repair := moveReadAssistedFixture(row) + switch name { + case "missing_move_cause": + delete(row, "move_request_cause") + case "foreign_move_wire": + row["move_request_cause"].(map[string]any)["request_id_key"] = "n:99" + case "missing_actual_cause": + delete(repair, "request_cause") + case "ack_cause_as_reader": + repair["request_cause"].(map[string]any)["session"] = "actual-session" + repair["request_cause"].(map[string]any)["request_id_key"] = "n:17" + case "foreign_admission": + repair["request_cause"] = map[string]any{"session": "other-reader", "request_id_key": "n:999", "admission_generation": json.Number("7")} + case "missing_window_start": + delete(repair, "started_from_ack_ns") + case "missing_acceptance": + delete(repair, "accepted_from_start_ns") + case "acceptance_outside_window": + repair["accepted_from_start_ns"] = json.Number("10000000") + case "acceptance_before_write": + repair["started_from_ack_ns"], repair["finished_from_ack_ns"] = json.Number("0"), json.Number("9000000") + repair["writer_return_to_acceptance_ns"] = json.Number("0") + case "acceptance_disagrees": + repair["writer_return_to_acceptance_ns"] = json.Number("3") + case "wrong_route_scope": + repair["scope"] = "move_direct_reconcile" + case "both_routes": + row["direct_reconcile_state"], row["direct_reconcile"] = "observed", repair + } + if got := checkEditProbeMoveAccounting(statuses, refs, 17); got.State == "observed_move_windows_checks_pass" { + t.Fatal("invalid read route accepted", name, got) + } + }) + } +} + +func TestMoveConsumerCountsCoalescedDirectAndReaderExecutionOnce(t *testing.T) { + refs, statuses := moveAccountingFixture(t) + _, other := moveAccountingFixture(t) + row := statuses["target"] + direct := other["target"]["direct_reconcile"].(map[string]any) + repair := moveReadAssistedFixture(row) + readerCause := repair["request_cause"].(map[string]any) + readerCause["admission_generation"] = json.Number("8") + causes := []any{direct["request_cause"], readerCause} + direct["reconcile_causes"], repair["reconcile_causes"] = causes, causes + row["direct_reconcile_state"], row["direct_reconcile"] = "observed", direct + got := checkEditProbeMoveAccounting(statuses, refs, 17) + if got.State != "observed_move_windows_checks_pass" || got.Files[1].Route != "coalesced_direct_and_read_assisted" || got.Files[1].D != nil || got.Files[1].R == nil { + t.Fatalf("same executed window double-counted or lost: %+v", got) + } + direct["window_ns"] = json.Number("9000001") + if changed := checkEditProbeMoveAccounting(statuses, refs, 17); changed.State == "observed_move_windows_checks_pass" { + t.Fatal("different actual executions deduplicated", changed) + } +} diff --git a/bench/load/edit_probe_native.go b/bench/load/edit_probe_native.go new file mode 100644 index 000000000..4d455c4b5 --- /dev/null +++ b/bench/load/edit_probe_native.go @@ -0,0 +1,435 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Native senders write the owned fixture outside MCP, as a shell redirection +// or an editor save would. They have no MCP acknowledgement, receipt, phases +// or derived time; each record keeps its own write window instead. +const ( + editProbeShellAfterTool = "shell_after_tool" + editProbeEditorSave = "editor_save" + editProbeCreateDelete = "create_delete" + editProbeHotFile = "hot_file" // MCP edits; only the arrivals are shaped + editProbeRestart = "restart" // one MCP edit; the daemon is killed while its ACK and receipt are pending +) + +type editProbeFileIdentity struct { + Size int64 `json:"size"` + MtimeNS int64 `json:"mtime_ns"` + Inode *uint64 `json:"inode"` +} + +type editProbeNativeWrite struct { + Sender string `json:"sender"` + After *int `json:"after_ordinal,omitempty"` + DependencyReadyNS *int64 `json:"dependency_ready_ns,omitempty"` + StartNS *int64 `json:"write_start_ns"` + EndNS *int64 `json:"write_end_ns"` + Window Metric `json:"write_window_ms"` + Before *editProbeFileIdentity `json:"before"` + Written *editProbeFileIdentity `json:"after"` + Replaced bool `json:"replaced"` +} + +// Native operations have no MCP workload attempt. A completed write with all +// of its correctness probes observed still proves that this phase did work. +func editProbeNativeWorkObserved(report *editProbeReport) bool { + if report == nil { + return false + } + for _, record := range report.Records { + write := record.NativeWrite + if record.Outcome != "complete" || len(record.Errors) != 0 || write == nil || write.Sender == "" || write.StartNS == nil || write.EndNS == nil || *write.EndNS < *write.StartNS || write.Window.Value == nil || *write.Window.Value < 0 || len(record.FirstCorrect) == 0 { + continue + } + correct := true + for _, observed := range record.FirstCorrect { + if observed.Value == nil { + correct = false + break + } + } + if correct { + return true + } + } + return false +} + +// editProbeNativeSender admits a native sender only with the fixture shape it +// writes. An empty edit_sender leaves every MCP cell unchanged. +func editProbeNativeSender(c Corpus, kind, tool string, count, editors int) error { + switch c.EditSender { + case "": + return nil + case editProbeShellAfterTool: + if kind != "add_intrafile_call" || tool != "edit.file" { + return errors.New("shell_after_tool pairs edit.file call additions with shell writes of the next marker") + } + if count != 100 || editors != 1 { + return errors.New("shell_after_tool runs 50 tool/shell pairs: 100 edits on one editor") + } + case editProbeEditorSave: + if kind != "rename" || tool != "edit.file" { + return errors.New("editor_save renames precreated same-length placeholders: it needs the rename fixture through edit.file") + } + if editors != 1 { + return errors.New("editor_save has one editor") + } + case editProbeCreateDelete: + if filepath.Ext(c.File) != ".go" { + return errors.New("create_delete is implemented for Go fixtures only; TypeScript/Python native create/delete is not implemented") + } + if kind != "add_intrafile_call" || tool != "edit.file" || filepath.Base(c.File) != c.File || c.EditOutput == "compact" || c.EditOutput == "cli" { + return errors.New("create_delete needs a root-level Go call-addition fixture with JSON or TOON probes: a deleted file's absence is proved by the exact refusal and complete collections") + } + if count != 100 || editors != 1 { + return errors.New("create_delete runs 50 create/delete pairs: 100 edits on one editor") + } + case editProbeRestart: + if kind != "add_intrafile_call" || tool != "edit.file" || c.EditOutput != "" && c.EditOutput != "json" { + return errors.New("restart kills the daemon under one JSON edit.file call addition") + } + if count != 1 || editors != 1 { + return errors.New("restart is a one-edit, one-editor diagnostic") + } + case editProbeHotFile: + if kind != "add_intrafile_call" || tool != "edit.file" { + return errors.New("hot_file pairs edit.file call additions on one file") + } + if count != 100 || editors != 2 { + return errors.New("hot_file runs 50 simultaneous pairs: 100 edits on exactly two editors") + } + default: + return fmt.Errorf("unsupported edit_sender %q", c.EditSender) + } + return nil +} + +// Odd shell_after_tool ordinals are shell writes that follow the preceding +// tool edit's own ACK; every editor_save ordinal is a native save. +func editProbeNativeStep(sender string, ordinal int) (string, *int, string) { + switch { + case sender == editProbeShellAfterTool && ordinal%2 == 1: + after := ordinal - 1 + return "shell_write", &after, "ack" + case sender == editProbeEditorSave: + return "editor_save", nil, "" + case sender == editProbeCreateDelete && ordinal%2 == 0: + return "create_file", nil, "" + case sender == editProbeCreateDelete: + after := ordinal - 1 + return "delete_file", &after, "first_correct" + } + return "", nil, "" +} + +// The native request describes the splice the sender writes; it is offered +// load in the plan but never dispatched to MCP. +func editProbeNativeRequest(file, sender string, splice request) request { + return request{Tool: "native", Operation: sender, Arguments: map[string]any{"path": file, "match": splice.Arguments["match"], "replacement": splice.Arguments["replacement"]}} +} + +// A placeholder has exactly the marker's length, so saving the marker over it +// keeps the file size. +func editProbePlaceholder(seed int64, ordinal int) string { + return fmt.Sprintf("GortexEditProbe%016xP%03d", uint64(seed), ordinal) +} + +func editProbeNativeLimitations(sender string) []string { + if sender == editProbeRestart { + return []string{"the restart cell polls its edit's status by mutation_id while the ACK is outstanding and kills the owned daemon once the bytes are committed and the graph receipt is not yet terminal; the harness stops waiting for that ACK, so t_ack is unavailable and the abandoned call is counted censored, not as an error", "the daemon is restarted on the same store and the editor and prober sessions are reopened; probes pause from the kill until the restarted daemon is ready, so no probe error is attributed to the downtime", "first-correct times are post-restart probe answers, from send and from ready; answers before the kill are kept separately and never count as first-correct", "a receipt already terminal at the last status read before the kill (as when the ACK returns first) qualifies the record instead of passing; the interrupted receipt has no phases, accounting or derived time, and process counters and peaks cover the pre-kill daemon only"} + } + if sender == editProbeHotFile { + return []string{"two fixed editors send each pair of edits to one file at the same due time; every edit keeps its own ACK, receipt and accounting, and a lost write is a measured outcome"} + } + out := []string{"native writes have no MCP acknowledgement, receipt, phases or derived time: each record keeps its own write window, its immediate file readback gates t_readback, graph-backed readbacks are observation-only, and it completes once every probe is first-correct inside the deadline", "the daemon learns of a native write only through its watcher, so a fresh claim after the write is counted unassessed, never stale-claimed-fresh"} + switch sender { + case editProbeShellAfterTool: + out = append(out, "odd ordinals are in-place shell writes (truncate, then write) dispatched at their due time or the preceding tool edit's own ACK, whichever is later; a tool edit that overlaps a shell write can lose either change, which is a measured outcome") + case editProbeEditorSave: + out = append(out, "every ordinal is an editor save: same-size bytes are written to a temporary file beside the fixture and renamed over it") + case editProbeCreateDelete: + out = append(out, "even ordinals create their own Go file beside the fixture and odd ordinals delete it once every probe of that create was first-correct; a deleted file has no gating readback, and its file-scoped summary, editing context and call chain are not probed because a missing file's answer has no pinned contract", "TypeScript/Python native create/delete is not implemented: a gap, not an inapplicable cell") + } + return out +} + +func (s *editProbeState) ackSignal() chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + if s.ackReady == nil { + s.ackReady = make(chan struct{}) + } + return s.ackReady +} + +func (s *editProbeState) signalAck() { + ready := s.ackSignal() + s.ackOnce.Do(func() { close(ready) }) +} + +func (s *editProbeState) firstCorrectComplete() bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, value := range s.record.FirstCorrect { + if value.Value == nil { + return false + } + } + return true +} + +// A native write runs at its due time, or once its dependency is met: the +// preceding tool edit's own ACK, or every first-correct probe of the paired +// create. It is never retried; an unmet dependency is an explicit failure and +// nothing is written. +func (r *editProbeRuntime) doNativeWrite(ctx context.Context, step editProbeStep) { + native := &editProbeNativeWrite{Sender: step.Sender, After: step.After, Window: unavailable("ms", "native write not performed")} + var written error + if step.After != nil { + predecessor := r.states[*step.After] + select { + case <-predecessor.ackSignal(): + case <-ctx.Done(): + } + predecessor.mu.Lock() + met := !predecessor.acknowledged.IsZero() + if step.AfterEvent == "first_correct" { + met = predecessor.record.Outcome == "complete" + } + predecessor.mu.Unlock() + now := time.Now() + ready := now.Sub(r.origin).Nanoseconds() + native.DependencyReadyNS = &ready + if met { + own := r.states[step.Ordinal] + own.mu.Lock() + own.dependencyMet = now + own.mu.Unlock() + } + switch { + case ctx.Err() != nil: + case !met && step.AfterEvent == "first_correct": + written = fmt.Errorf("native write dependency failed: ordinal %d did not complete every first-correct probe", *step.After) + case !met: + written = fmt.Errorf("native write dependency failed: ordinal %d has no own acknowledgement", *step.After) + } + } + s, callCtx, cancel, started := r.beginEdit(ctx, step) + r.markPending(s) + defer r.settlePending(s) + defer s.signalAck() + defer s.stopProbes() + defer cancel() + if written == nil && ctx.Err() == nil { + if step.Deleted { + // The paired create's own completed source probe proved presence. + s.mu.Lock() + s.absenceBefore = map[string]bool{step.BeforeSymbol: true} + s.mu.Unlock() + } + written = r.writeNative(s, step, native) + } + s.mu.Lock() + s.record.NativeWrite = native + s.record.Receipt, s.record.MutationID = "", "" + s.record.Ack = unavailable("ms", "native write has no MCP acknowledgement") + s.record.Accounting = editProbeAccounting{State: "not_applicable_native_write"} + s.record.PhasesMS = editProbePhases(nil, "native write has no mutation receipt") + s.record.Derived = unavailable("ms", "native write has no mutation receipt") + s.record.Readback = unavailable("ms", "native write or its immediate file readback failed") + if step.Deleted { + s.record.Readback = unavailable("ms", "a deleted file has no readback to gate") + } + if written != nil { + s.record.Errors = append(s.record.Errors, written.Error()) + } + s.mu.Unlock() + if written == nil { + kinds := []string{"file", "source"} + if step.BeforeSymbol != "" { + kinds = append(kinds, "old_source_absent") + } + if step.Deleted { + kinds = []string{"source"} + } + // The file is on disk once the write returns; graph answers may lag + // the watcher, so they are recorded without gating anything. + end, correct := r.immediateReadbacks(callCtx, s, step, kinds, map[string]bool{"source": true, "old_source_absent": true}) + if correct && !end.IsZero() { + s.mu.Lock() + s.record.Readback = measured(float64(end.Sub(started))/float64(time.Millisecond), "ms") + s.mu.Unlock() + } + for callCtx.Err() == nil && !s.firstCorrectComplete() { + if waitContext(callCtx, editProbePeriod) != nil { + break + } + } + } + s.mu.Lock() + defer s.mu.Unlock() + if written == nil && (s.firstProbeWrite.IsZero() || s.editWrite.IsZero() || !s.firstProbeWrite.Before(s.editWrite)) { + s.record.Errors = append(s.record.Errors, "no actual prober wire start before native write") + } + for name, value := range s.record.FirstCorrect { + if value.Value == nil { + s.record.Errors = append(s.record.Errors, "first-correct oracle unavailable: "+name) + } + } + switch { + case ctx.Err() != nil: + s.record.Outcome = "phase_censored" + case callCtx.Err() != nil: + s.record.Outcome = "product_deadline" + case len(s.record.Errors) == 0: + s.record.Outcome = "complete" + default: + s.record.Outcome = "diagnostic_failure" + } +} + +// writeNative writes only its registered owned file: the fixture, or the +// step's own created file. A shell write truncates and rewrites the file in +// place; an editor save writes same-size bytes to a temporary file beside it +// and renames that over the fixture; create and delete add or remove the +// step's own file. +func (r *editProbeRuntime) writeNative(s *editProbeState, step editProbeStep, native *editProbeNativeWrite) error { + path, _ := step.Request.Arguments["path"].(string) + owned := r.plan.File + if step.File != "" { + owned = step.File + } + repo := r.p.Repos[0].Path + target := filepath.Join(repo, filepath.FromSlash(path)) + if step.Request.Tool != "native" || step.Request.Operation != step.Sender || path != owned || !inside(repo, target) { + return errors.New("native sender writes only the registered owned fixture file") + } + var write func() error + before, err := os.Lstat(target) + if step.Sender == "create_file" { + content, _ := step.Request.Arguments["content"].(string) + if !errors.Is(err, os.ErrNotExist) { + return errors.New("native create target already exists") + } + write = func() error { return editProbeCreateFile(target, []byte(content)) } + } else { + if err != nil { + return err + } + before, err = editProbeNativeSnapshot(before, func() (os.FileInfo, error) { + // Windows Lstat resolves file IDs lazily by pathname. A handle + // Stat pins this file's ID before a save can replace the path. + file, err := os.Open(target) + if err != nil { + return nil, err + } + info, err := file.Stat() + // Release the handle before mutation so Windows sharing and + // delete permissions are unchanged during the measured write. + return info, errors.Join(err, file.Close()) + }) + if err != nil { + return err + } + native.Before = editProbeIdentity(before) + write = func() error { return os.Remove(target) } + } + if step.Sender == "shell_write" || step.Sender == "editor_save" { + match, _ := step.Request.Arguments["match"].(string) + replacement, _ := step.Request.Arguments["replacement"].(string) + current, err := os.ReadFile(target) + if err != nil { + return err + } + if match == "" || strings.Count(string(current), match) != 1 { + return errors.New("native write splice does not occur exactly once in the current fixture bytes") + } + next := []byte(strings.Replace(string(current), match, replacement, 1)) + if step.Sender == "editor_save" && len(next) != len(current) { + return errors.New("editor save must keep the fixture size") + } + write = func() error { return editProbeShellWrite(target, next) } + if step.Sender == "editor_save" { + write = func() error { return editProbeSaveReplace(target, step.Ordinal, next, before.Mode().Perm()) } + } + } + start := time.Now() + s.mu.Lock() + s.editWrite = start + s.mu.Unlock() + err = write() + end := time.Now() + startNS, endNS := start.Sub(r.origin).Nanoseconds(), end.Sub(r.origin).Nanoseconds() + native.StartNS, native.EndNS = &startNS, &endNS + native.Window = measured(float64(end.Sub(start))/float64(time.Millisecond), "ms") + if after, statErr := os.Lstat(target); statErr == nil { + native.Written = editProbeIdentity(after) + native.Replaced = before != nil && !os.SameFile(before, after) + } + row := map[string]any{"event": "native_write", "sender": step.Sender, "path": path, "write_start_ns": startNS, "write_end_ns": endNS, "before": native.Before, "after": native.Written, "replaced": native.Replaced} + if err != nil { + row["error"] = err.Error() + } + return errors.Join(err, s.emit(row)) +} + +func editProbeCreateFile(target string, content []byte) error { + file, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return err + } + _, err = file.Write(content) + return errors.Join(err, file.Close()) +} + +// Shell redirection semantics: the existing file is truncated, then written. +func editProbeShellWrite(target string, content []byte) error { + file, err := os.OpenFile(target, os.O_WRONLY|os.O_TRUNC, 0) + if err != nil { + return err + } + _, err = file.Write(content) + return errors.Join(err, file.Close()) +} + +// Editor save semantics: a temporary file with no source extension is written +// beside the fixture, then renamed over it. +func editProbeSaveReplace(target string, ordinal int, content []byte, mode os.FileMode) error { + temp := filepath.Join(filepath.Dir(target), fmt.Sprintf(".%s.%03d.gortex-save.tmp", filepath.Base(target), ordinal)) + file, err := os.OpenFile(temp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + _, err = file.Write(content) + err = errors.Join(err, file.Close()) + if err == nil { + err = os.Rename(temp, target) + } + if err != nil { + _ = os.Remove(temp) + } + return err +} + +func editProbeIdentity(info os.FileInfo) *editProbeFileIdentity { + return &editProbeFileIdentity{Size: info.Size(), MtimeNS: info.ModTime().UnixNano(), Inode: editProbeInode(info)} +} + +// editProbeNativeSnapshot keeps the owned regular-file guard before taking +// the identity snapshot. The capture callback is local to this native writer. +func editProbeNativeSnapshot(before os.FileInfo, capture func() (os.FileInfo, error)) (os.FileInfo, error) { + if !before.Mode().IsRegular() { + return nil, errors.New("native sender writes only a regular owned fixture file") + } + return capture() +} diff --git a/bench/load/edit_probe_native_identity_test.go b/bench/load/edit_probe_native_identity_test.go new file mode 100644 index 000000000..b24e67b7e --- /dev/null +++ b/bench/load/edit_probe_native_identity_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +// Windows path-based FileInfo loads its file ID on the first SameFile call. +// This model makes that delayed path lookup explicit on every test host. +// Capturing the handle's Stat before a replacement must freeze the old ID. +type editProbeLazyFileInfo struct { + os.FileInfo + identity func() uint64 +} + +func (i editProbeLazyFileInfo) Sys() any { return i.identity() } + +func TestEditProbeNativeSnapshotPinsIdentityBeforePathReplacement(t *testing.T) { + path := filepath.Join(t.TempDir(), "fixture.go") + if err := os.WriteFile(path, []byte("before"), 0o600); err != nil { + t.Fatal(err) + } + info, err := os.Lstat(path) + if err != nil { + t.Fatal(err) + } + boundID := uint64(1) + lazy := editProbeLazyFileInfo{FileInfo: info, identity: func() uint64 { return boundID }} + captures := 0 + pinned, err := editProbeNativeSnapshot(lazy, func() (os.FileInfo, error) { + captures++ + captured := boundID + return editProbeLazyFileInfo{FileInfo: info, identity: func() uint64 { return captured }}, nil + }) + if err != nil { + t.Fatal(err) + } + // The pathname now belongs to the replacement, before either retained + // FileInfo is asked for its ID. A lazy path lookup would see the new file. + boundID = 2 + if pinned.Sys() != uint64(1) || captures != 1 { + t.Fatalf("before identity followed the replacement path: id=%v captures=%d", pinned.Sys(), captures) + } + if lazy.Sys() != uint64(2) { + t.Fatal("control did not model the rebound pathname") + } +} + +func TestEditProbeNativeSnapshotRejectsUnobservedIdentity(t *testing.T) { + info, err := os.Stat(t.TempDir()) + if err != nil { + t.Fatal(err) + } + called := false + if _, err := editProbeNativeSnapshot(info, func() (os.FileInfo, error) { + called = true + return info, nil + }); err == nil || called { + t.Fatalf("nonregular target reached capture: called=%v err=%v", called, err) + } + path := filepath.Join(t.TempDir(), "fixture.go") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + info, err = os.Lstat(path) + if err != nil { + t.Fatal(err) + } + snapshotErr := errors.New("native identity snapshot or handle close failed") + if _, err := editProbeNativeSnapshot(info, func() (os.FileInfo, error) { return nil, snapshotErr }); !errors.Is(err, snapshotErr) { + t.Fatalf("failed identity snapshot was silently accepted: %v", err) + } +} diff --git a/bench/load/edit_probe_native_other.go b/bench/load/edit_probe_native_other.go new file mode 100644 index 000000000..affbd41f1 --- /dev/null +++ b/bench/load/edit_probe_native_other.go @@ -0,0 +1,8 @@ +//go:build !linux && !darwin + +package main + +import "os" + +// Other hosts report no inode; os.SameFile still decides replacement. +func editProbeInode(os.FileInfo) *uint64 { return nil } diff --git a/bench/load/edit_probe_native_test.go b/bench/load/edit_probe_native_test.go new file mode 100644 index 000000000..cac71e48d --- /dev/null +++ b/bench/load/edit_probe_native_test.go @@ -0,0 +1,416 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" + "time" +) + +func editProbeEditorSaveCorpus(t *testing.T, count int) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, "go") + c.EditKind, c.EditTool, c.EditSender = "rename", "edit.file", editProbeEditorSave + for i := range count { + c.EditInitialContent += strings.ReplaceAll(c.EditTemplate, "{{marker}}", editProbePlaceholder(42, i)) + } + return c +} + +// editProbeDiskStub answers every default oracle, readback and the restore +// batch from the current bytes of the Go files on disk, as a daemon that has +// indexed them would. A name no longer declared answers the exact not-found +// refusal; caller subgraphs keep the stable callee's own node. +func editProbeDiskStub(t *testing.T, c Corpus, dir string) func(stubFrame) (any, *rpcError) { + t.Helper() + return editProbeDiskReplies(c, dir, func(err error) { t.Error(err) }) +} + +func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFrame) (any, *rpcError) { + declared := regexp.MustCompile(`func (GortexEditProbe\w+)\(`) + callee := c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] + return func(frame stubFrame) (any, *rpcError) { + if frame.Method != "tools/call" { + return stubContent(map[string]any{}), nil + } + args, _ := frame.Params["arguments"].(map[string]any) + if frame.Params["name"] == "edit" && args["operation"] == "batch" { + for _, raw := range args["changes"].([]any) { + change := raw.(map[string]any) + path := filepath.Join(dir, strings.TrimPrefix(change["path"].(string), "fixture/")) + var err error + if change["op"] == "delete_file" { + err = os.Remove(path) + } else { + err = os.WriteFile(path, []byte(change["new_string"].(string)), 0o600) + } + if err != nil { + report(err) + } + } + return stubContent(map[string]any{"committed": true}), nil + } + owner := map[string]string{} + byFile := map[string][]any{} + graph, edges := []any{map[string]any{"id": "fixture/" + c.Symbol}}, []any{} + paths, _ := filepath.Glob(filepath.Join(dir, "*.go")) + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + report(err) + } + file := filepath.Base(path) + for _, match := range declared.FindAllStringSubmatch(string(data), -1) { + id := "fixture/" + file + "::" + match[1] + owner[match[1]] = file + byFile[file] = append(byFile[file], map[string]any{"id": id}) + graph = append(graph, map[string]any{"id": id}) + edges = append(edges, map[string]any{"from": id, "to": "fixture/" + c.Symbol, "kind": "calls"}) + } + } + target, _ := args["target"].(map[string]any) + symbol, _ := target["symbol"].(string) + file, _ := target["file"].(string) + symbol, file = strings.TrimPrefix(symbol, "fixture/"), strings.TrimPrefix(file, "fixture/") + _, name, _ := strings.Cut(symbol, "::") + query, _ := args["query"].(string) + rows := []any{} + switch frame.Params["name"].(string) + "." + args["operation"].(string) { + case "read.file": + data, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + report(err) + } + return stubContent(map[string]any{"path": "fixture/" + file, "content": string(data)}), nil + case "read.source": + if owner[name] == "" || symbol != owner[name]+"::"+name { + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": "symbol not found: fixture/" + symbol}}}, nil + } + return stubContent(map[string]any{"id": "fixture/" + symbol, "name": name, "file_path": "fixture/" + owner[name], "source": "func " + name + "() int { return " + callee + "() }"}), nil + case "read.summary": + return stubContent(map[string]any{"nodes": byFile[file]}), nil + case "read.editing_context": + return stubContent(map[string]any{"defines": byFile[file]}), nil + case "search.symbols": + if owner[query] != "" { + rows = append(rows, map[string]any{"id": "fixture/" + owner[query] + "::" + query}) + } + return stubContent(map[string]any{"results": rows, "total": len(rows), "truncated": false, "query_class": "symbol"}), nil + case "search.text": + if owner[query] != "" { + rows = append(rows, map[string]any{"path": "fixture/" + owner[query], "line": 3, "text": "func " + query + "() int { return " + callee + "() }"}) + } + return stubContent(map[string]any{"matches": rows, "count": len(rows), "query": query, "truncated": false}), nil + case "relations.callers", "trace.call_chain": + return stubContent(map[string]any{"nodes": graph, "edges": edges, "total_nodes": len(graph), "total_edges": len(edges), "truncated": false}), nil + } + report(fmt.Errorf("unexpected tool call %v", frame.Params)) + return stubContent(map[string]any{}), nil + } +} + +// The runtime holds prepared states for the first n planned ordinals; each +// has its own prober session beside the one fixed editor. +func newEditProbeNativeRun(t *testing.T, ctx context.Context, c Corpus, plan editProbePlan, dir string, n int) *editProbeRuntime { + t.Helper() + return newEditProbeStubRun(t, ctx, c, plan, dir, n, editProbeDiskStub(t, c, dir)) +} + +func newEditProbeStubRun(t *testing.T, ctx context.Context, c Corpus, plan editProbePlan, dir string, n int, handle func(stubFrame) (any, *rpcError)) *editProbeRuntime { + t.Helper() + socket := startStub(t, handle) + open := func() *socketClient { + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + return client + } + original, err := os.ReadFile(filepath.Join(dir, c.File)) + if err != nil { + t.Fatal(err) + } + phase := &Phase{Metrics: map[string]Metric{}, EditProbe: newEditProbeReport(plan, c, true)} + run := &editProbeRuntime{options: runOptions{Deadline: 3 * time.Second, Seed: 42}, p: &privateRun{Repos: []privateRepo{{Prefix: "fixture", Path: dir, Corpus: c}}}, phase: phase, plan: plan, editors: []*socketClient{open()}, original: original, origin: time.Now(), requireAccounting: true} + run.p.Socket = socket + for _, step := range plan.Steps[:n] { + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step, state.record, state.client = step, newEditProbeRecord(c, step), open() + state.record.Outcome, state.record.Errors = "not_started", nil // as prepared + if step.BeforeSymbol != "" { + state.absenceBefore = map[string]bool{step.BeforeSymbol: true} + } + run.states = append(run.states, state) + } + return run +} + +func awaitEditProbeFirstProbe(t *testing.T, s *editProbeState) { + t.Helper() + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + s.mu.Lock() + started := !s.firstProbeWrite.IsZero() + s.mu.Unlock() + if started { + return + } + } + t.Fatal("no probe wire start") +} + +func checkEditProbeNativeRecord(t *testing.T, record editProbeRecord, sender string) { + t.Helper() + if record.Outcome != "complete" || len(record.Errors) != 0 || record.Readback.Value == nil { + t.Fatalf("native record did not complete: %+v", record) + } + if record.Ack.Value != nil || record.Ack.Reason != "native write has no MCP acknowledgement" || record.Receipt != "" || record.MutationID != "" || record.Accounting.State != "not_applicable_native_write" || record.Derived.Value != nil || record.GraphTerminal { + t.Fatalf("native record invented an ACK, receipt or accounting: %+v", record) + } + for name, lap := range record.PhasesMS { + if lap.Value != nil { + t.Fatal("native record invented a phase", name) + } + } + native := record.NativeWrite + if native == nil || native.Sender != sender || native.StartNS == nil || native.EndNS == nil || *native.EndNS < *native.StartNS || native.Window.Value == nil || native.Before == nil || native.Written == nil { + t.Fatalf("native write window missing: %+v", native) + } + for _, back := range record.Readbacks { + if back.ObservationOnly != (back.Kind != "file") || back.Kind == "file" && !back.Correct { + t.Fatalf("only the file readback gates a native write: %+v", record.Readbacks) + } + } +} + +func TestEditProbeShellWriteFollowsOwnToolACKInPlace(t *testing.T) { + c := editProbeCorpus() + c.EditSender = editProbeShellAfterTool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + reference, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Sender != editProbeShellAfterTool || plan.Tool != "edit.file" || !slices.Contains(plan.NotSelected, "other native senders run as their own cells") { + t.Fatalf("shell plan identity: %+v", plan) + } + for i, step := range plan.Steps { + want := reference.Steps[i] + if step.DueNS != want.DueNS || step.ActivateNS != want.ActivateNS || step.Marker != want.Marker { + t.Fatal("native sender changed the offered schedule") + } + if i%2 == 0 { + if step.Sender != "" || step.After != nil || step.RequestHash != want.RequestHash || step.MutationID != want.MutationID { + t.Fatalf("tool ordinal changed: %+v", step) + } + continue + } + if step.Sender != "shell_write" || step.After == nil || *step.After != i-1 || step.AfterEvent != "ack" || step.MutationID != "" || step.Request.Tool != "native" || step.Request.Arguments["path"] != c.File || step.Request.Arguments["match"] != want.Request.Arguments["match"] || step.Request.Arguments["replacement"] != want.Request.Arguments["replacement"] || step.Request.Arguments["options"] != nil || step.Request.Arguments["output"] != nil { + t.Fatalf("shell ordinal does not follow its tool edit: %+v", step) + } + } + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, c.File) + original := "package sample\n\nfunc Example() int { return 1 }\n\n" + c.EditAnchor + "\n" + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 2) + tool, shell := run.states[0], run.states[1] + run.activate(ctx, plan.Steps[1]) + done := make(chan struct{}) + go func() { + run.doEdit(ctx, plan.Steps[1]) + close(done) + }() + time.Sleep(100 * time.Millisecond) + if data, _ := os.ReadFile(path); string(data) != original { + t.Fatal("shell write did not wait for the preceding tool edit's own ACK") + } + tool.mu.Lock() + tool.acknowledged = time.Now() + tool.mu.Unlock() + tool.signalAck() + <-done + record := shell.record + checkEditProbeNativeRecord(t, record, "shell_write") + native := record.NativeWrite + if native.After == nil || *native.After != 0 || native.DependencyReadyNS == nil || *native.StartNS < *native.DependencyReadyNS || native.Replaced || native.Written.Size <= native.Before.Size { + t.Fatalf("shell write was not an in-place write after the ACK: %+v", native) + } + if native.Before.Inode != nil && (native.Written.Inode == nil || *native.Before.Inode != *native.Written.Inode) { + t.Fatal("shell write replaced the file") + } + if data, _ := os.ReadFile(path); !strings.Contains(string(data), "func "+plan.Steps[1].Marker+"()") { + t.Fatal("shell write missing from disk") + } +} + +func TestEditProbeEditorSaveKeepsSizeAndReplacesAtomically(t *testing.T) { + c := editProbeEditorSaveCorpus(t, 100) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + reference, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + for i, step := range plan.Steps { + placeholder := editProbePlaceholder(42, i) + match, _ := step.Request.Arguments["match"].(string) + replacement, _ := step.Request.Arguments["replacement"].(string) + if step.DueNS != reference.Steps[i].DueNS || step.Sender != "editor_save" || step.After != nil || step.MutationID != "" || step.BeforeSymbol != c.File+"::"+placeholder || len(placeholder) != len(step.Marker) || step.Request.Tool != "native" || len(match) != len(replacement) || !strings.Contains(match, placeholder) || !strings.Contains(replacement, step.Marker) { + t.Fatalf("editor save step is not a same-size placeholder rename: %+v", step) + } + } + + c = editProbeEditorSaveCorpus(t, 1) + plan, err = makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, c.File) + if err := os.WriteFile(path, []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 1) + run.activate(ctx, step) + awaitEditProbeFirstProbe(t, run.states[0]) + run.doEdit(ctx, step) + record := run.states[0].record + checkEditProbeNativeRecord(t, record, "editor_save") + native := record.NativeWrite + if !native.Replaced || native.Before.Size != native.Written.Size || len(record.Readbacks) != 3 || record.Readbacks[2].Kind != "old_source_absent" { + t.Fatalf("editor save did not replace same-size bytes: %+v %+v", native, record.Readbacks) + } + if native.Before.Inode != nil && (native.Written.Inode == nil || *native.Before.Inode == *native.Written.Inode) { + t.Fatal("editor save wrote in place instead of renaming") + } + if temps, _ := filepath.Glob(filepath.Join(dir, ".*.gortex-save.tmp")); len(temps) != 0 { + t.Fatal("editor save left its temporary file", temps) + } + // The terminal native record admits the timed restore batch. + if err := run.finish(); err != nil { + t.Fatal(err) + } + if err := run.restore(ctx); err != nil || run.phase.EditProbe.RestoreMS.Value == nil { + t.Fatalf("restore after native save: %v", err) + } + if data, _ := os.ReadFile(path); string(data) != c.EditInitialContent { + t.Fatal("restore did not return original bytes") + } +} + +func TestEditProbeNativeDependencyFailureIsExplicit(t *testing.T) { + c := editProbeCorpus() + c.EditSender = editProbeShellAfterTool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + path := filepath.Join(dir, c.File) + original := "package sample\n\nfunc Example() int { return 1 }\n\n" + c.EditAnchor + "\n" + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatal(err) + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 2) + run.activate(ctx, plan.Steps[1]) + run.states[0].signalAck() // the tool edit ended without its own ACK + run.doEdit(ctx, plan.Steps[1]) + record := run.states[1].record + if record.Outcome != "diagnostic_failure" || !slices.Contains(record.Errors, "native write dependency failed: ordinal 0 has no own acknowledgement") || record.NativeWrite == nil || record.NativeWrite.StartNS != nil || record.NativeWrite.Window.Value != nil || record.Ack.Value != nil { + t.Fatalf("dependency failure was not explicit: %+v", record) + } + if data, _ := os.ReadFile(path); string(data) != original { + t.Fatal("native write ran without its dependency") + } +} + +func TestEditProbeNativeSenderRefusesUnownedPathsAndWrongFixtures(t *testing.T) { + shell := editProbeCorpus() + shell.EditSender = editProbeShellAfterTool + save := editProbeEditorSaveCorpus(t, 100) + for name, build := range map[string]func() (editProbePlan, error){ + "shell_rename": func() (editProbePlan, error) { + c := editProbeRenameCorpus(t, "go", 100) + c.EditTool, c.EditSender = "edit.file", editProbeShellAfterTool + return makeEditProbePlan(c, 42, 100) + }, + "shell_write_tool": func() (editProbePlan, error) { + c := editProbeWriteCorpus(t, "go") + c.EditSender = editProbeShellAfterTool + return makeEditProbePlan(c, 42, 100) + }, + "shell_one_edit": func() (editProbePlan, error) { return makeEditProbePlan(shell, 42, 1) }, + "shell_two_editors": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(shell, 42, 100, 2) }, + "save_addition": func() (editProbePlan, error) { + c := editProbeCorpus() + c.EditSender = editProbeEditorSave + return makeEditProbePlan(c, 42, 100) + }, + "save_two_editors": func() (editProbePlan, error) { return makeEditProbeTrafficPlan(save, 42, 100, 2) }, + "save_without_placeholders": func() (editProbePlan, error) { + c := editProbeRenameCorpus(t, "go", 100) + c.EditTool, c.EditSender = "edit.file", editProbeEditorSave + return makeEditProbePlan(c, 42, 100) + }, + "unknown_sender": func() (editProbePlan, error) { + c := editProbeCorpus() + c.EditSender = "watch" + return makeEditProbePlan(c, 42, 100) + }, + } { + if _, err := build(); err == nil { + t.Errorf("%s: wrong native fixture planned", name) + } + } + + c := editProbeEditorSaveCorpus(t, 1) + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + step.Request.Arguments = map[string]any{"path": "other.go", "match": step.Request.Arguments["match"], "replacement": step.Request.Arguments["replacement"]} + plan.Steps[0] = step + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + dir := t.TempDir() + for _, name := range []string{c.File, "other.go"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + } + run := newEditProbeNativeRun(t, ctx, c, plan, dir, 1) + run.activate(ctx, step) + run.doEdit(ctx, step) + if record := run.states[0].record; record.Outcome != "diagnostic_failure" || !slices.Contains(record.Errors, "native sender writes only the registered owned fixture file") { + t.Fatalf("unowned native path accepted: %+v", record) + } + for _, name := range []string{c.File, "other.go"} { + if data, _ := os.ReadFile(filepath.Join(dir, name)); string(data) != c.EditInitialContent { + t.Fatal("refused native write changed", name) + } + } +} diff --git a/bench/load/edit_probe_native_unix.go b/bench/load/edit_probe_native_unix.go new file mode 100644 index 000000000..c2939f4ed --- /dev/null +++ b/bench/load/edit_probe_native_unix.go @@ -0,0 +1,17 @@ +//go:build linux || darwin + +package main + +import ( + "os" + "syscall" +) + +func editProbeInode(info os.FileInfo) *uint64 { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + inode := stat.Ino + return &inode +} diff --git a/bench/load/edit_probe_observe_test.go b/bench/load/edit_probe_observe_test.go new file mode 100644 index 000000000..72437c5a6 --- /dev/null +++ b/bench/load/edit_probe_observe_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" +) + +func TestEditProbeDelayedObservationUsesActualReplyNotObserverOrder(t *testing.T) { + for _, beforeEdit := range []bool{false, true} { + t.Run(map[bool]string{false: "after_edit", true: "before_edit"}[beforeEdit], func(t *testing.T) { + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step = step + state.record = newEditProbeRecord(corpus, step) + state.record.Errors = nil + origin := time.Now() + state.started = origin.Add(10 * time.Millisecond) + run := &editProbeRuntime{origin: origin, plan: plan, p: &privateRun{Repos: []privateRepo{{Corpus: corpus}}}} + body, err := json.Marshal(map[string]any{"name": step.Marker, "source": strings.ReplaceAll(corpus.EditTemplate, "{{marker}}", step.Marker)}) + if err != nil { + t.Fatal(err) + } + wire, err := json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": string(body)}}}) + if err != nil { + t.Fatal(err) + } + finished := origin.Add(15 * time.Millisecond) + if beforeEdit { + finished = origin.Add(5 * time.Millisecond) + } + identity := wireCallIdentity{WriteStarted: origin.Add(time.Millisecond), Finished: finished} + if err := run.observeProbe(state, editProbeOracle{name: "source"}, origin, wire, identity, nil, false); err != nil { + t.Fatal(err) + } + first := state.record.FirstCorrect["source"] + if beforeEdit { + if first.Value != nil || len(state.record.Errors) != 1 { + t.Fatalf("late observer hid pre-edit reply: first%+v errors%v", first, state.record.Errors) + } + } else { + if first.Value == nil || *first.Value != 5 { + t.Fatalf("observer timing replaced actual reply: %+v", first) + } + identity.Finished = origin.Add(20 * time.Millisecond) + if err := run.observeProbe(state, editProbeOracle{name: "source"}, origin, wire, identity, nil, false); err != nil { + t.Fatal(err) + } + if *state.record.FirstCorrect["source"].Value != 5 { + t.Fatal("later paired probe replaced first success") + } + } + }) + } +} diff --git a/bench/load/edit_probe_output.go b/bench/load/edit_probe_output.go new file mode 100644 index 000000000..4e392d548 --- /dev/null +++ b/bench/load/edit_probe_output.go @@ -0,0 +1,197 @@ +package main + +import "fmt" + +func editProbeOutputMode(c Corpus) (string, error) { + switch c.EditOutput { + case "", "json": + return "json", nil + case "toon": + return "toon", nil + case "compact": + return "compact", nil + case "cli": + return "cli", nil + case "gcx": + return "", fmt.Errorf("edit output mode gcx: %w", errEditOutputGCXUnavailable) + default: + return "", fmt.Errorf("unsupported edit output mode %q (want json, toon, compact or cli)", c.EditOutput) + } +} + +// Every output mode is accounted in each report, including the ones this +// harness cannot select, so absent coverage is explicit rather than silent. +var editProbeOutputModes = []string{"json", "toon", "compact", "cli", "gcx"} + +type editProbeModes map[string]editProbeModeCoverage + +type editProbeModeCoverage struct { + State string `json:"state"` + Reason string `json:"reason,omitempty"` + ProbeAnswers int `json:"probe_answers"` + DecodeFailures int `json:"decode_failures"` + ClaimsFreshObservable bool `json:"claims_fresh_observable"` + ClaimsFresh int `json:"claims_fresh"` + StaleClaimedFresh Metric `json:"stale_claimed_fresh"` + PrecommitMismatches int `json:"precommit_fresh_mismatches_unassessed"` +} + +type editProbeModeCounts struct { + answers, decodeFailures, claimsFresh, stale, precommit int +} + +// Answers count by each oracle request's actual output mode. A mode without +// answers, or whose payload cannot carry a freshness claim, reports its stale +// claimed-fresh count unavailable rather than zero. +func editProbeOutputCoverage(selected string, counts map[string]editProbeModeCounts) editProbeModes { + out := make(editProbeModes, len(editProbeOutputModes)) + for _, mode := range editProbeOutputModes { + count := counts[mode] + coverage := editProbeModeCoverage{State: "not_selected", ProbeAnswers: count.answers, DecodeFailures: count.decodeFailures, ClaimsFreshObservable: true, ClaimsFresh: count.claimsFresh, PrecommitMismatches: count.precommit} + switch mode { + case "gcx": + coverage.State, coverage.Reason, coverage.ClaimsFreshObservable = "unavailable", errEditOutputGCXUnavailable.Error(), false + case "compact": + coverage.Reason, coverage.ClaimsFreshObservable = "compact text has no freshness field; it proves call-addition presence only", false + } + if mode == selected { + coverage.State = "selected" + } + switch { + case !coverage.ClaimsFreshObservable: + coverage.StaleClaimedFresh = unavailable("answers", "mode cannot carry a freshness claim here") + case count.answers == 0: + coverage.StaleClaimedFresh = unavailable("answers", "no probe answers in this mode") + default: + coverage.StaleClaimedFresh = measured(float64(count.stale), "answers") + } + out[mode] = coverage + } + return out +} + +// A CLI probe sends the same JSON facade request through its own process, so +// its answers are decoded and counted as CLI stdout, never as MCP JSON. +func (r *editProbeRuntime) probeOutput(oracle editProbeOracle) string { + if r.plan.Mode == "cli" { + return "cli" + } + return editProbeRequestOutput(oracle.call) +} + +func (r *editProbeRuntime) countMode(mode string, answered, decoded, fresh, stale, precommit bool) { + r.mu.Lock() + defer r.mu.Unlock() + if r.modeCounts == nil { + r.modeCounts = map[string]editProbeModeCounts{} + } + count := r.modeCounts[mode] + if answered { + count.answers++ + if !decoded { + count.decodeFailures++ + } + } + if fresh { + count.claimsFresh++ + } + if stale { + count.stale++ + } + if precommit { + count.precommit++ + } + r.modeCounts[mode] = count +} + +func editProbeRequestOutput(req request) string { + output, exists := req.Arguments["output"] + if !exists { + return "json" + } + fields, ok := output.(map[string]any) + if !ok { + return "invalid" + } + value := "json" + if format, exists := fields["format"]; exists { + if value, ok = format.(string); !ok { + return "invalid" + } + } + if compact, exists := fields["compact"]; exists { + if compact != true || value != "json" { + return "invalid" + } + return "compact" + } + return value +} + +// The dispatched output keeps each request's selected wire explicit with no +// byte budget; any request that selected nothing else stays JSON. +func editProbeDispatchOutput(req request) map[string]any { + switch editProbeRequestOutput(req) { + case "toon": + return map[string]any{"format": "toon", "max_bytes": 0} + case "compact": + return map[string]any{"format": "json", "compact": true, "max_bytes": 0} + } + return map[string]any{"format": "json", "max_bytes": 0} +} + +// Compact is requested only from the operations with a compact producer; +// every other request, including edits and readbacks, keeps its JSON wire. +func editProbeWithOutput(req request, format string) request { + output := map[string]any{"format": "toon", "max_bytes": 0} + switch { + case format == "compact" && editProbeCompactOperations[req.name()]: + output = map[string]any{"format": "json", "compact": true, "max_bytes": 0} + case format != "toon": + return req + } + req.Arguments = copyJSON(req.Arguments) + if req.Arguments == nil { + req.Arguments = make(map[string]any) + } + req.Arguments["output"] = output + return req +} +func editProbeOutputOracles(c Corpus, oracles []editProbeOracle) []editProbeOracle { + if c.EditOutput != "toon" && c.EditOutput != "compact" { + return oracles + } + for i := range oracles { + oracles[i].call = editProbeWithOutput(oracles[i].call, c.EditOutput) + } + return oracles +} + +// This certifies only the actual format's complete emitted row census. It +// never inserts a JSON total_nodes/total_edges value that TOON did not serve. +// The known compact traversal producer emits total+truncated, retains all edge +// rows, and omits total_edges precisely when no edge rows were clipped. +func editProbeTOONCompleteRows(body map[string]any, key, total string) bool { + proof, ok := body[editProbeTOONWireKey].(editProbeTOONWire) + if !ok || body["truncated"] != false { + return false + } + switch proof.operation { + case "relations.callers", "trace.call_chain": + default: + return false + } + if truncated, exists := body["_truncated_by_budget"]; exists && truncated != false { + return false + } + nodes, exists := body["nodes"] + rows, array := nodes.([]any) + if exists && nodes == nil { + rows, array = []any{}, true + } + declared, err := strconvEditProbeUint(body["total"]) + if !exists || !array || err != nil || declared != uint64(len(rows)) { + return false + } + return key == "nodes" && total == "total_nodes" || key == "edges" && total == "total_edges" +} diff --git a/bench/load/edit_probe_output_test.go b/bench/load/edit_probe_output_test.go new file mode 100644 index 000000000..8adb20e57 --- /dev/null +++ b/bench/load/edit_probe_output_test.go @@ -0,0 +1,337 @@ +package main + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + toon "github.com/toon-format/toon-go" +) + +func outputWire(text string, structured any) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": text}}, "structuredContent": structured}) + return raw +} +func TestEditProbeTOONEvaluatesActualTextAndPreservesMissingFields(t *testing.T) { + producer := map[string]any{"nodes": []any{map[string]any{"id": "fixture/main.go::Added", "file_path": "fixture/main.go", "kind": "function", "name": "Added", "start_line": 7}}, "edges": []any{}, "total": 1, "truncated": false} + encoded, err := toon.Marshal(producer) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeOutput(outputWire(string(encoded), map[string]any{"nodes": []any{}}), "toon", "read.summary") + if err != nil { + t.Fatal(err) + } + if _, exists := body["total_nodes"]; exists { + t.Fatal("invented omitted JSON total_nodes") + } + if _, exists := body["total_edges"]; exists { + t.Fatal("invented omitted edge count") + } + node := body["nodes"].([]any)[0].(map[string]any) + if node["id"] != "fixture/main.go::Added" || node["start_line"] != json.Number("7") || body["total"] != json.Number("1") { + t.Fatal(body) + } + if _, exists := node["meta"]; exists { + t.Fatal("invented compact-row metadata") + } + if proof, ok := body[editProbeTOONWireKey].(editProbeTOONWire); !ok || proof.operation != "read.summary" { + t.Fatal("requested decoder proof missing") + } +} +func TestEditProbeTOONRefusesWrongFormatSiblingAndIncompleteWire(t *testing.T) { + validJSON := map[string]any{"id": "fixture/main.go::Added"} + for name, text := range map[string]string{"JSON_fallback": `{"id":"fixture/main.go::Added"}`, "scalar": "false", "empty": "", "truncated_table": "nodes[2]{id}:\n one\n", "unsafe_integer": "owner: 9007199254740993", "malformed_indent": "a:\n b: 1\n c: 2"} { + t.Run(name, func(t *testing.T) { + if body, err := decodeEditProbeOutput(outputWire(text, validJSON), "toon", "read.source"); err == nil { + t.Fatal("wrong actual wire credited", body) + } + }) + } + duplicate := json.RawMessage(`{"content":[{"type":"text","text":"id: first"},{"type":"text","text":"id: second"}]}`) + if _, err := decodeEditProbeOutput(duplicate, "toon", "read.source"); err == nil { + t.Fatal("ambiguous text content accepted") + } + failed := json.RawMessage(`{"isError":true,"content":[{"type":"text","text":"id: parseable"}]}`) + if _, err := decodeEditProbeOutput(failed, "toon", "read.source"); err == nil { + t.Fatal("tool failure counted as data") + } + if _, err := decodeEditProbeOutput(outputWire("id: added", nil), "json", "read.source"); err == nil { + t.Fatal("TOON body credited to JSON mode") + } + if _, err := decodeEditProbeOutput(outputWire("id: added", nil), "gcx", "read.source"); err == nil { + t.Fatal("unsupported mode accepted") + } +} + +func TestEditProbeOutputModePreservesDefaultPlanAndActualRequestIntent(t *testing.T) { + c := editProbeCorpus() + original, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + c.EditOutput = "json" + explicitJSON, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(original, explicitJSON) { + t.Fatal("explicit default JSON changed normalized offered plan") + } + c.EditOutput = "toon" + compact, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if compact.Mode != "toon" || len(compact.Steps) != 100 { + t.Fatal(compact) + } + for i, step := range compact.Steps { + baseline := original.Steps[i] + if step.DueNS != baseline.DueNS || step.ActivateNS != baseline.ActivateNS || step.MutationID != baseline.MutationID || step.Marker != baseline.Marker || step.RequestHash == baseline.RequestHash || editProbeRequestOutput(step.Request) != "toon" { + t.Fatalf("format altered offered schedule or failed to bind request: %+v", step) + } + args := dispatchArgs(step.Request, privateRepo{Prefix: "fixture"}) + if args["output"].(map[string]any)["format"] != "toon" { + t.Fatal("actual dispatch erased requested output") + } + for _, oracle := range editProbeOracles(c, step) { + if editProbeRequestOutput(oracle.call) != "toon" { + t.Fatal("probe silently requested JSON", oracle.name) + } + } + } + if original.Mode != "json" || editProbeRequestOutput(original.Steps[0].Request) != "json" { + t.Fatal("TOON setup mutated the original JSON plan") + } + // Measurement controls remain exact JSON even for a TOON corpus/session. + control := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": "commit-1"}}} + if dispatchArgs(control, privateRepo{Prefix: "fixture", Corpus: c})["output"].(map[string]any)["format"] != "json" { + t.Fatal("format changed timing-status control") + } + for _, unsupported := range []string{"TOON", "text", "gcx", "invalid"} { + c.EditOutput = unsupported + if _, err := makeEditProbePlan(c, 42, 100); err == nil { + t.Fatal("unsupported output mode admitted", unsupported) + } + } +} + +func TestEditProbeTOONCensusKeepsOmittedCountsUnavailable(t *testing.T) { + producer := map[string]any{"nodes": []any{map[string]any{"id": "fixture/main.go::Survivor"}}, "edges": nil, "total": 1, "truncated": false} + raw, err := toon.Marshal(producer) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeOutput(outputWire(string(raw), nil), "toon", "relations.callers") + if err != nil { + t.Fatal(err) + } + if rows, ok := editProbeCompleteCount(body, "nodes", "total_nodes"); !ok || len(rows) != 1 { + t.Fatal("explicit TOON node census not recognized", body) + } + if rows, ok := editProbeCompleteCount(body, "edges", "total_edges"); !ok || len(rows) != 0 { + t.Fatal("explicit null untrimmed edge collection not recognized", body) + } + if _, present := body["total_nodes"]; present { + t.Fatal("fabricated JSON node count") + } + if _, present := body["total_edges"]; present { + t.Fatal("fabricated omitted full edge count") + } + for _, name := range []string{"missing_total", "mismatched_total", "truncated", "missing_truncation", "budget_clipped", "missing_edges", "explicit_wrong_edges", "wrong_operation", "summary_cannot_borrow_traversal_counts", "forged_JSON_proof"} { + t.Run(name, func(t *testing.T) { + candidate, err := decodeEditProbeOutput(outputWire(string(raw), nil), "toon", "relations.callers") + if err != nil { + t.Fatal(err) + } + switch name { + case "missing_total": + delete(candidate, "total") + case "mismatched_total": + candidate["total"] = json.Number("2") + case "truncated": + candidate["truncated"] = true + case "missing_truncation": + delete(candidate, "truncated") + case "budget_clipped": + candidate["_truncated_by_budget"] = true + case "missing_edges": + delete(candidate, "edges") + case "explicit_wrong_edges": + candidate["total_edges"] = json.Number("1") + case "wrong_operation": + candidate[editProbeTOONWireKey] = editProbeTOONWire{operation: "search.symbols"} + case "summary_cannot_borrow_traversal_counts": + candidate[editProbeTOONWireKey] = editProbeTOONWire{operation: "read.summary"} + case "forged_JSON_proof": + candidate[editProbeTOONWireKey] = map[string]any{"operation": "relations.callers"} + } + if _, ok := editProbeCompleteCount(candidate, "edges", "total_edges"); ok { + t.Fatal("incomplete/foreign census accepted", name, candidate) + } + }) + } +} + +func TestEditProbeTOONSignatureUsesActualCompactRelativeWitness(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeSignatureCorpus(t, language, 1) + c.EditOutput = "toon" + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + oracles := editProbeOracles(c, step) + depth, depthOK := oracles[6].call.Arguments["options"].(map[string]any)["depth"].(float64) + if oracles[5].call.Arguments["target"].(map[string]any)["symbol"] != step.TargetSymbol || oracles[6].call.Arguments["target"].(map[string]any)["symbol"] != step.FollowingSymbol || !depthOK || depth != 2 { + t.Fatal("lost actual following-caller traversal") + } + for _, oracle := range []string{"callers", "call_chain"} { + operation := map[string]string{"callers": "relations.callers", "call_chain": "trace.call_chain"}[oracle] + for _, variant := range []string{"valid", "stale_gap", "wrong_proof", "truncated", "wrong_callee", "missing_total"} { + gap := step.ExpectedLineGap + if variant == "stale_gap" { + gap-- + } + callee := c.Symbol + if variant == "wrong_callee" { + callee = c.File + "::Foreign" + } + producer := map[string]any{"nodes": []any{map[string]any{"id": step.TargetSymbol, "start_line": 10}, map[string]any{"id": step.FollowingSymbol, "start_line": 10 + gap}}, "edges": []any{map[string]any{"from": step.TargetSymbol, "to": callee, "kind": "calls"}}, "total": 2, "truncated": variant == "truncated"} + if variant == "missing_total" { + delete(producer, "total") + } + encoded, err := toon.Marshal(producer) + if err != nil { + t.Fatal(err) + } + if variant == "wrong_proof" { + operation = "read.summary" + } + body, err := decodeEditProbeOutput(outputWire(string(encoded), nil), "toon", operation) + if err != nil { + t.Fatal(err) + } + correct := editProbeCorrect(oracle, body, c, step, plan.Callee) + if correct != (variant == "valid") { + t.Fatalf("%s/%s correct=%v", oracle, variant, correct) + } + if _, exists := body["total_nodes"]; exists { + t.Fatal("fabricated full count") + } + } + } + }) + } +} + +func TestEditProbeOutputSeparatesOnlyActualSessionNotes(t *testing.T) { + nudge := "(Session note: graph read #6. Every location returned so far is real and citeable — if you are localizing or answering a question, consider writing your conclusion from what you already hold and fetching only what it still lacks. This note appears once.)" + for _, format := range []string{"json", "toon"} { + payload := `{"id":"owned"}` + if format == "toon" { + payload = "id: owned" + } + for name, extras := range map[string][]string{ + "known": {nudge}, "unknown": {"Session note: arbitrary"}, "second_payload": {payload}, + "duplicate": {nudge, nudge}, "changed": {nudge + " trailing"}, + } { + t.Run(format+"/"+name, func(t *testing.T) { + content := []any{map[string]any{"type": "text", "text": payload}} + for _, extra := range extras { + content = append(content, map[string]any{"type": "text", "text": extra}) + } + raw, err := json.Marshal(map[string]any{"content": content}) + if err != nil { + t.Fatal(err) + } + before := string(raw) + body, err := decodeEditProbeOutput(raw, format, "relations.callers") + if name == "known" { + if err != nil || body["id"] != "owned" { + t.Fatal(body, err) + } + } else if err == nil { + t.Fatal("extra payload or unknown note accepted", body) + } + if string(raw) != before { + t.Fatal("raw evidence changed") + } + }) + } + } +} + +func TestEditProbeGCXIsUnavailableNeverTOON(t *testing.T) { + encoded, err := toon.Marshal(map[string]any{"id": "fixture/main.go::Added"}) + if err != nil { + t.Fatal(err) + } + if _, err := decodeEditProbeOutput(outputWire(string(encoded), nil), "gcx", "read.file"); !errors.Is(err, errEditOutputGCXUnavailable) { + t.Fatalf("valid TOON text was not refused as GCX: %v", err) + } + c := editProbeCorpus() + c.EditOutput = "gcx" + if _, err := editProbeOutputMode(c); !errors.Is(err, errEditOutputGCXUnavailable) { + t.Fatalf("gcx selection is not the explicit unavailable refusal: %v", err) + } + c.EditOutput = "invalid" + if _, err := editProbeOutputMode(c); err == nil || errors.Is(err, errEditOutputGCXUnavailable) { + t.Fatalf("unsupported mode reported as GCX: %v", err) + } + coverage := editProbeOutputCoverage("json", nil) + if gcx := coverage["gcx"]; gcx.State != "unavailable" || !strings.Contains(gcx.Reason, "TOON") || gcx.StaleClaimedFresh.Value != nil { + t.Fatalf("gcx coverage: %+v", gcx) + } + if len(coverage) != len(editProbeOutputModes) || coverage["json"].State != "selected" || coverage["toon"].State != "not_selected" { + t.Fatalf("every mode must be accounted: %+v", coverage) + } +} + +func TestEditProbeModeCoverageCountsStaleClaimsPerActualMode(t *testing.T) { + corpus := editProbeCorpus() + corpus.EditOutput = "toon" + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step, state.record = step, newEditProbeRecord(corpus, step) + origin := time.Now() + state.started, state.acknowledged = origin.Add(time.Millisecond), origin.Add(2*time.Millisecond) + run := &editProbeRuntime{origin: origin, plan: plan, p: &privateRun{Repos: []privateRepo{{Corpus: corpus}}}} + oracle := editProbeOracles(corpus, step)[0] + encoded, err := toon.Marshal(map[string]any{"name": "Unrelated", "freshness": "fresh"}) + if err != nil { + t.Fatal(err) + } + identity := wireCallIdentity{WriteStarted: origin.Add(3 * time.Millisecond), Finished: origin.Add(4 * time.Millisecond)} + // A wrong answer claiming freshness after the ACK, then one JSON body sent + // to a TOON request (a decode failure, never TOON credit). + if err := run.observeProbe(state, oracle, identity.WriteStarted, outputWire(string(encoded), nil), identity, nil, false); err != nil { + t.Fatal(err) + } + if err := run.observeProbe(state, oracle, identity.WriteStarted, outputWire(`{"freshness":"fresh"}`, nil), identity, nil, false); err != nil { + t.Fatal(err) + } + coverage := editProbeOutputCoverage(plan.Mode, run.modeCounts) + toonMode := coverage["toon"] + if toonMode.State != "selected" || toonMode.ProbeAnswers != 2 || toonMode.DecodeFailures != 1 || toonMode.ClaimsFresh != 1 || toonMode.StaleClaimedFresh.Value == nil || *toonMode.StaleClaimedFresh.Value != 1 { + t.Fatalf("toon coverage: %+v", toonMode) + } + if json := coverage["json"]; json.ProbeAnswers != 0 || json.StaleClaimedFresh.Value != nil || json.StaleClaimedFresh.Reason != "no probe answers in this mode" { + t.Fatalf("unselected JSON reported a measured zero: %+v", json) + } + if !state.record.StaleClaimedFresh { + t.Fatal("record lost its stale claim") + } +} diff --git a/bench/load/edit_probe_phases.go b/bench/load/edit_probe_phases.go new file mode 100644 index 000000000..52267bd6b --- /dev/null +++ b/bench/load/edit_probe_phases.go @@ -0,0 +1,45 @@ +package main + +import ( + "cmp" + "fmt" + "math" + "strconv" +) + +// The daemon's ACK laps followed by its patch laps, as mutation_status names +// them in phases_ms. +var editProbePhaseNames = []string{ + "dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait", "edit_control", "caller_result", + "debounce", "cohort_admission", "batch_admission", "lane_wait", "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", +} + +// Every expected lap is reported for every record. A status without phases_ms +// (a build that predates it, or no status at all) leaves each lap unavailable +// with that reason, and a producer null keeps the producer's own phase state as +// the reason. Missing laps are never reported as a measured zero. +func editProbePhases(status map[string]any, missing string) map[string]Metric { + phases, present := status["phases_ms"].(map[string]any) + states, _ := status["phase_states"].(map[string]any) + out := make(map[string]Metric, len(editProbePhaseNames)) + for _, name := range editProbePhaseNames { + value, listed := phases[name] + state, _ := states[name].(string) + switch { + case !present: + out[name] = unavailable("ms", missing) + case !listed: + out[name] = unavailable("ms", "status phases_ms omits "+name) + case value == nil: + out[name] = unavailable("ms", "producer phase state "+cmp.Or(state, "unreported")) + default: + ms, err := strconv.ParseFloat(fmt.Sprint(value), 64) + if err != nil || ms < 0 || math.IsInf(ms, 0) || math.IsNaN(ms) { + out[name] = unavailable("ms", "invalid phases_ms value") + } else { + out[name] = measured(ms, "ms") + } + } + } + return out +} diff --git a/bench/load/edit_probe_phases_test.go b/bench/load/edit_probe_phases_test.go new file mode 100644 index 000000000..40d3b0af4 --- /dev/null +++ b/bench/load/edit_probe_phases_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestEditProbePhasesAreUnavailableWithReasonsNeverZero(t *testing.T) { + decode := func(raw string) map[string]any { + status, err := decodeEditProbeResult(outputWire(raw, nil)) + if err != nil { + t.Fatal(err) + } + return status + } + // A baseline build that predates phases_ms reports every lap unavailable. + for name, metric := range editProbePhases(decode(`{"receipt":"r","graph_status":"fresh"}`), "status exposes no phases_ms") { + if metric.Value != nil || metric.Reason != "status exposes no phases_ms" || metric.Unit != "ms" { + t.Fatalf("absent phase %s became %+v", name, metric) + } + } + phases := editProbePhases(decode(`{"phases_ms":{"parse":1.25,"debounce":0,"derived":null,"apply":null,"resolve":"x"},"phase_states":{"derived":"pending","debounce":"measured"}}`), "unused") + if len(phases) != len(editProbePhaseNames) { + t.Fatalf("expected every ACK and patch lap: %d", len(phases)) + } + for name, want := range map[string]string{"derived": "producer phase state pending", "apply": "producer phase state unreported", "resolve": "invalid phases_ms value", "mtimes": "status phases_ms omits mtimes"} { + if phases[name].Value != nil || phases[name].Reason != want { + t.Fatalf("%s: %+v, want reason %q", name, phases[name], want) + } + } + // A producer's measured zero stays a measurement; only absence is unavailable. + if phases["parse"].Value == nil || *phases["parse"].Value != 1.25 || phases["debounce"].Value == nil || *phases["debounce"].Value != 0 { + t.Fatalf("measured laps lost: %+v %+v", phases["parse"], phases["debounce"]) + } + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + baseline, candidate := newEditProbeReport(plan, corpus, false), newEditProbeReport(plan, corpus, true) + if baseline.Role != "baseline" || candidate.Role != "candidate" { + t.Fatalf("roles: %q %q", baseline.Role, candidate.Role) + } + data, err := json.Marshal(baseline.Records[0].PhasesMS["parse"]) + if err != nil || string(data) != `{"value":null,"unit":"ms","reason":"no receipt status observed"}` { + t.Fatalf("unsent edit phase is not explicitly unavailable: %s %v", data, err) + } +} + +func TestEditProbeRoleFlagIsEditProbeOnlyAndNotPaired(t *testing.T) { + base := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", "1", "--worktrees", "0"} + for role, baseline := range map[string]bool{"": false, "candidate": false, "baseline": true} { + args := base + if role != "" { + args = append(append([]string{}, base...), "--edit-probe-role", role) + } + o, err := parseRun(args) + if err != nil || o.editProbeBaseline != baseline { + t.Fatalf("role %q: baseline=%v err=%v", role, o.editProbeBaseline, err) + } + } + for _, extra := range [][]string{{"--edit-probe-role", "main"}, {"--edit-probe-role", "baseline", "--scenario", "edit"}} { + if _, err := parseRun(append(append([]string{}, base...), extra...)); err == nil { + t.Fatalf("invalid role accepted: %v", extra) + } + } + paired := []string{"--build", "main=/private/main", "--build", "candidate=/private/candidate", "--manifest", "/private/manifest.json", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", "1", "--worktrees", "0", "--edit-probe-role", "baseline"} + if _, _, err := parsePaired(paired); err == nil || !strings.Contains(err.Error(), "build order") { + t.Fatalf("paired accepted an explicit role: %v", err) + } +} diff --git a/bench/load/edit_probe_plan.go b/bench/load/edit_probe_plan.go new file mode 100644 index 000000000..4e7ab041b --- /dev/null +++ b/bench/load/edit_probe_plan.go @@ -0,0 +1,448 @@ +package main + +import ( + "cmp" + "context" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "math" + "math/rand/v2" + "path/filepath" + "slices" + "strings" + "sync" + "time" +) + +const editProbeScenario = "edit-and-probe" +const editProbeMean = 15 * time.Second +const editProbePeriod = 25 * time.Millisecond +const editProbeMaxEditors = 32 +const editProbeFixedEditorModel = "fixed_editor_per_ordinal_plus_one_separate_prober_per_edit" + +type editProbeStep struct { + Ordinal int `json:"ordinal"` + ActivateNS int64 `json:"activate_ns"` + DueNS int64 `json:"due_ns"` + Editor int `json:"editor,omitempty"` + Marker string `json:"marker"` + MutationID string `json:"mutation_id"` + Request request `json:"request"` + RequestHash string `json:"request_sha256"` + BeforeSymbol string `json:"before_symbol,omitempty"` + Deleted bool `json:"deleted,omitempty"` + TargetSymbol string `json:"target_symbol,omitempty"` + FollowingSymbol string `json:"following_symbol,omitempty"` + ExpectedLineGap int `json:"expected_line_gap,omitempty"` + Relocated bool `json:"relocated,omitempty"` + DestinationFile string `json:"destination_file,omitempty"` + ExpectedRefusal bool `json:"expected_refusal,omitempty"` + Sender string `json:"sender,omitempty"` + After *int `json:"after,omitempty"` + AfterEvent string `json:"after_event,omitempty"` + File string `json:"file,omitempty"` +} + +type editProbePlan struct { + Version int `json:"version"` + Arrival string `json:"arrival"` + MeanNS int64 `json:"mean_interarrival_ns"` + ProbePeriodNS int64 `json:"probe_period_ns"` + Count int `json:"count"` + Editors int `json:"editors,omitempty"` + Readers int `json:"readers,omitempty"` + NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` + Seed int64 `json:"seed"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` + Sender string `json:"sender,omitempty"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + SessionModel string `json:"session_model"` + NotSelected []string `json:"not_selected_not_tested"` + Steps []editProbeStep `json:"steps"` +} + +type editProbeArrival struct { + DueNS int64 + Editor int +} + +// Each editor offers its own open-loop Poisson stream with the single-editor +// mean. Editor 0 keeps the original seeding, so one editor offers exactly the +// original arrivals. Streams merge by due time (ties by editor) and the merge +// order is the global ordinal: every ordinal belongs to one fixed editor. +func editProbeArrivals(seed int64, count, editors int) ([]editProbeArrival, error) { + var merged []editProbeArrival + for editor := range editors { + rng := rand.New(rand.NewPCG(uint64(seed)^0xa0761d6478bd642f^uint64(editor)*0x9e3779b97f4a7c15, uint64(seed)^0xe7037ed1a0b428db)) + due := int64(editProbePeriod) + for range count { + interval := rng.ExpFloat64() * float64(editProbeMean) + if math.IsNaN(interval) || math.IsInf(interval, 0) || interval >= float64(math.MaxInt64) { + return nil, errors.New("poisson interval overflow") + } + delta := max(int64(1), int64(interval)) + if due > math.MaxInt64-delta { + return nil, errors.New("poisson offered-plan duration overflow") + } + due += delta + merged = append(merged, editProbeArrival{DueNS: due, Editor: editor}) + } + } + slices.SortFunc(merged, func(a, b editProbeArrival) int { + return cmp.Or(cmp.Compare(a.DueNS, b.DueNS), cmp.Compare(a.Editor, b.Editor)) + }) + return merged[:count], nil +} + +func makeEditProbePlan(c Corpus, seed int64, count int) (editProbePlan, error) { + return makeEditProbeTrafficPlan(c, seed, count, 1) +} + +func planEditProbe(c Corpus, o runOptions) (editProbePlan, error) { + p, err := makeEditProbeTrafficPlan(c, o.Seed, o.EditCount, o.Editors) + if err != nil || o.Agents == 0 { + return p, err + } + if c.EditSender == editProbeRestart { + return editProbePlan{}, errors.New("the restart cell has no pure readers") + } + if o.Agents < 0 || o.Agents > editProbeMaxReaders { + return editProbePlan{}, fmt.Errorf("edit-and-probe accepts 0..%d pure readers", editProbeMaxReaders) + } + // Readers first run a same-run no-edit window; the offered edit schedule, + // relative to the edit origin after that window, is unchanged. + p.Readers, p.NoEditWindow = o.Agents, int64(editProbeNoEditWindow) + return p, nil +} + +func makeEditProbeTrafficPlan(c Corpus, seed int64, count, editors int) (editProbePlan, error) { + if err := validateCorpus(c); err != nil { + return editProbePlan{}, err + } + if count != 1 && count != 100 { + return editProbePlan{}, errors.New("edit-and-probe requires 100 edits, or explicit 1-edit diagnostic") + } + if editors < 1 || editors > editProbeMaxEditors { + return editProbePlan{}, fmt.Errorf("edit-and-probe requires 1..%d fixed editors", editProbeMaxEditors) + } + if count == 1 && editors > 1 { + return editProbePlan{}, errors.New("the 1-edit diagnostic has exactly one editor") + } + mode, err := editProbeOutputMode(c) + if err != nil { + return editProbePlan{}, err + } + tool, err := editProbeTool(c) + if err != nil { + return editProbePlan{}, err + } + kind := c.EditKind + if kind == "" { + kind = "add_intrafile_call" + } + if kind != "add_intrafile_call" && kind != "append_leaf" && kind != "rename" && kind != "delete" && kind != "modify_body" && kind != "modify_caller_body" && kind != "change_signature" && kind != "add_crossfile_call" && kind != "relocate" { + return editProbePlan{}, errors.New("unsupported edit kind") + } + if mode == "compact" && kind != "add_intrafile_call" && kind != "add_crossfile_call" { + return editProbePlan{}, fmt.Errorf("compact output cannot check %s: compact text omits node IDs, edge rows, signatures and freshness, so it proves only a call addition's presence", kind) + } + if mode == "cli" && (kind == "rename" || kind == "delete" || kind == "relocate") { + return editProbePlan{}, fmt.Errorf("CLI probes cannot check %s: an expected absence must be the exact scoped not-found refusal, and the CLI reports a tool refusal only as an exit status and stderr text", kind) + } + + if kind != "relocate" && (c.EditDestination != "" || c.EditDestinationContent != "") { + return editProbePlan{}, errors.New("edit_destination applies only to the explicit move fixture") + } + if c.EditExpect != "" && tool != "refactor.move" { + return editProbePlan{}, errors.New("edit_expect refusal applies only to TypeScript/Python relocation") + } + if kind == "relocate" && tool != "refactor.move" { + return editProbePlan{}, errors.New("relocation requires the explicit Go move adapter") + } + // Each write carries whole-file content precomputed in ordinal order. With + // several editors an out-of-order write silently drops another editor's + // marker, and the next ordinal's write restores it; a batch's old_string + // precondition fails visibly instead. + if tool == "edit.write" && editors > 1 { + return editProbePlan{}, errors.New("edit.write with several editors would silently drop a concurrent editor's marker: use edit.file or edit.batch") + } + if err := editProbeNativeSender(c, kind, tool, count, editors); err != nil { + return editProbePlan{}, err + } + language := "go" + if filepath.Ext(c.File) == ".ts" { + language = "typescript" + } + if filepath.Ext(c.File) == ".py" { + language = "python" + } + if (filepath.Ext(c.File) != ".go" && filepath.Ext(c.File) != ".ts" && filepath.Ext(c.File) != ".py") || c.EditAnchor == "" || strings.Count(c.EditTemplate, "{{marker}}") != 1 { + return editProbePlan{}, errors.New("edit-and-probe cell requires Go or explicit TypeScript/Python fixture/stable anchor/one marker declaration") + } + selectedCallee := c.Symbol + if kind == "add_crossfile_call" { + if _, _, err := editProbeCrossFileCallee(c); err != nil { + return editProbePlan{}, err + } + selectedCallee = c.EditCallee + if prefix := editProbeCrossFileImport(c); prefix != "" && !strings.HasPrefix(c.EditInitialContent, prefix) { + return editProbePlan{}, errors.New("crossfile fixture requires exact leading import") + } + } else if c.EditCallee != "" { + return editProbePlan{}, errors.New("edit_callee applies only to explicit crossfile fixture") + } + separator := strings.LastIndex(selectedCallee, "::") + if separator < 0 { + return editProbePlan{}, errors.New("initial intrafile cell requires one plain callee symbol") + } + callee := selectedCallee[separator+2:] + if !token.IsIdentifier(callee) { + return editProbePlan{}, errors.New("initial intrafile cell requires one plain callee symbol") + } + probeMarker := "GortexEditProbeDeclaration" + templateCallee := callee + templateSource := strings.ReplaceAll(c.EditTemplate, "{{marker}}", probeMarker) + if kind == "change_signature" { + templateSource = editProbeSignaturePlain(c.File, templateSource) + } + if kind == "append_leaf" { + templateCallee = "" + } + switch language { + case "typescript": + valid, correct := editProbeTypeScriptFixture(templateSource, probeMarker, templateCallee) + if !valid || !correct { + return editProbePlan{}, errors.New("TypeScript template must be a numeric zero-argument marker function returning the intrafile call") + } + case "python": + valid, correct := editProbePythonFixture(templateSource, probeMarker, templateCallee) + if !valid || !correct { + return editProbePlan{}, errors.New("python template must be a top-level marker with one indented return of the intrafile call") + } + default: + parsed, err := parser.ParseFile(token.NewFileSet(), "template.go", "package probe\n"+strings.ReplaceAll(c.EditTemplate, "{{marker}}", probeMarker), 0) + if err != nil { + return editProbePlan{}, fmt.Errorf("edit template is not standalone Go declarations: %w", err) + } + declaration, call := false, false + for _, decl := range parsed.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Name.Name == probeMarker && fn.Body != nil { + declaration = true + ast.Inspect(fn.Body, func(n ast.Node) bool { + if expr, ok := n.(*ast.CallExpr); ok { + if name, ok := expr.Fun.(*ast.Ident); ok && name.Name == callee { + call = true + } + } + return true + }) + } + } + if !declaration || (kind != "append_leaf" && !call) || (kind == "append_leaf" && !editProbeLeafSource(c.File, strings.ReplaceAll(c.EditTemplate, "{{marker}}", probeMarker), probeMarker)) { + return editProbePlan{}, errors.New("template must declare the unique marker function and actually call the selected intrafile callee") + } + } + native := "native save/create/delete/hotfile/restart" + if c.EditSender != "" { + native = "other native senders run as their own cells" + } + p := editProbePlan{Version: 1, Arrival: "precomputed_open_loop_poisson", MeanNS: int64(editProbeMean), ProbePeriodNS: int64(editProbePeriod), Count: count, Seed: seed, Kind: kind, Tool: tool, Language: language, Mode: mode, Repo: c.Label, Commit: c.Commit, File: c.File, Callee: callee, SessionModel: "one_editor_plus_one_separate_prober_per_edit", NotSelected: []string{"other edit kinds/tools/languages/output modes", native, "largest-file/reference configurations require their own reviewed manifest"}} + if kind == "change_signature" && language != "go" { + p.NotSelected = append(p.NotSelected, "parameter detail is proved by exact source; native TS/Python signature metadata omits it and graph freshness uses the relative line witness") + } + if c.EditExpect == "refusal" { + p.NotSelected = append(p.NotSelected, "TypeScript/Python relocation is offered only as the product's explicit refusal; Go relocation is the supported move cell") + } + p.Sender = c.EditSender + if editors > 1 { + p.Editors, p.SessionModel = editors, editProbeFixedEditorModel + p.NotSelected = append(p.NotSelected, "several editors share one repository; 6-repository and linked-worktree routing are not implemented") + } + arrivals, err := editProbeArrivals(seed, count, editors) + if c.EditSender == editProbeHotFile { + arrivals, err = editProbeHotFileArrivals(seed, count) + } + if err != nil { + return editProbePlan{}, err + } + pkg := "" + if c.EditSender == editProbeCreateDelete { + parsed, err := parser.ParseFile(token.NewFileSet(), c.File, c.EditInitialContent, parser.PackageClauseOnly) + if err != nil { + return editProbePlan{}, err + } + pkg = parsed.Name.Name + } + writeContent := c.EditInitialContent + for i, arrival := range arrivals { + due := arrival.DueNS + marker := editProbeMarker(seed, i) + id := fmt.Sprintf("load-edit-%016x-%03d", uint64(seed), i) + r := editRequest(c, marker) + if tool == "edit.symbol" { + r = editProbeSymbolRequest(c, marker) + } + if (tool == "edit.write" || tool == "edit.batch") && kind != "rename" && kind != "delete" && kind != "modify_body" && kind != "modify_caller_body" && kind != "change_signature" { + if strings.Contains(c.EditInitialContent, marker) { + return editProbePlan{}, errors.New("planned marker already exists in write fixture") + } + if tool == "edit.batch" { + r, writeContent = editProbeBatchRequest(c, writeContent, marker, fmt.Sprintf("load-batch-%016x-%03d", uint64(seed), i)) + } else { + r, writeContent = editProbeWriteRequest(c, writeContent, marker) + } + } + beforeSymbol := "" + targetSymbol, followingSymbol := "", "" + expectedGap := 0 + if kind == "rename" || kind == "delete" || kind == "modify_body" || kind == "modify_caller_body" || kind == "change_signature" { + oldName := marker + "Before" + if c.EditSender == editProbeEditorSave { + oldName = editProbePlaceholder(seed, i) + } + beforeTemplate := c.EditTemplate + if kind == "modify_body" || kind == "modify_caller_body" || kind == "change_signature" { + beforeTemplate = c.EditBeforeTemplate + if strings.Count(beforeTemplate, "{{marker}}") != 1 { + return editProbePlan{}, errors.New("body change requires one explicit before-template declaration") + } + } + before := strings.TrimSpace(strings.ReplaceAll(beforeTemplate, "{{marker}}", oldName)) + if strings.Count(c.EditInitialContent, before) != 1 || strings.Contains(c.EditInitialContent, marker+"(") { + return editProbePlan{}, errors.New("rename/delete requires each exact original declaration and absent new declaration") + } + replacement := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", marker)) + if kind == "delete" { + replacement = "" + } + if kind == "modify_body" || kind == "modify_caller_body" || kind == "change_signature" { + replacement = strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", oldName)) + following := marker + "After" + nextDeclaration := editProbeFollowingDeclaration(c.File, following) + if kind == "change_signature" { + nextDeclaration = editProbeFollowingCaller(c.File, following, oldName) + } + if strings.Count(c.EditInitialContent, before+"\n\n"+nextDeclaration) != 1 || strings.Count(replacement, "\n") <= strings.Count(before, "\n") { + return editProbePlan{}, errors.New("body fixture requires adjacent retained follower and actual body line growth") + } + beforeValid := editProbeLeafSource(c.File, before, oldName) + if kind == "modify_caller_body" { + beforeValid = editProbeFixtureSource(c.File, before, oldName, "Alternate") && editProbeLeafSource(c.File, c.EditInitialContent, "Alternate") + } + if kind == "change_signature" { + beforeValid = editProbeFixtureSource(c.File, before, oldName, callee) && editProbeSignatureHeaders(c.File, before, replacement, oldName) + } + if !beforeValid { + return editProbePlan{}, errors.New("before body is not the declared numeric leaf or Alternate caller") + } + targetSymbol = c.File + "::" + oldName + followingSymbol = c.File + "::" + following + expectedGap = strings.Count(replacement, "\n") + 2 + } + beforeSymbol = c.File + "::" + oldName + switch tool { + case "edit.file": + r = request{Tool: "edit", Operation: "file", Arguments: map[string]any{"operation": "file", "target": map[string]any{"file": c.File}, "match": before, "replacement": replacement}} + case "edit.symbol": + r = request{Tool: "edit", Operation: "symbol", Arguments: map[string]any{"operation": "symbol", "target": map[string]any{"symbol": beforeSymbol}, "match": before, "replacement": replacement}} + case "edit.write", "edit.batch": + next := strings.Replace(writeContent, before, replacement, 1) + if tool == "edit.write" { + r = request{Tool: "edit", Operation: "write", Arguments: map[string]any{"operation": "write", "target": map[string]any{"file": c.File}, "content": next}} + } else { + r = editProbeBatchContentRequest(c, writeContent, next, fmt.Sprintf("load-batch-%016x-%03d", uint64(seed), i)) + } + writeContent = next + } + } + if kind == "relocate" { + beforeName := marker + "Before" + before := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", beforeName)) + if strings.Count(c.EditInitialContent, before) != 1 || strings.Contains(c.EditDestinationContent, beforeName) { + return editProbePlan{}, errors.New("move fixture requires exact original declaration and absent destination identity") + } + beforeSymbol = c.File + "::" + beforeName + targetSymbol = c.EditDestination + "::" + beforeName + r = request{Tool: "refactor", Operation: "move", Arguments: map[string]any{"operation": "move", "target": map[string]any{"symbol": beforeSymbol}, "destination": c.EditDestination}} + } + sender, after, afterEvent := editProbeNativeStep(c.EditSender, i) + file, deleted := "", kind == "delete" + if c.EditSender == editProbeCreateDelete { + r, file, beforeSymbol = editProbeLifecycleStep(c, pkg, seed, i) + deleted = file != "" && beforeSymbol != "" + } + if sender != "" { + if r.Tool != "native" { + r = editProbeNativeRequest(c.File, sender, r) + } + id = "" + } else { + if tool != "edit.batch" && tool != "refactor.move" { + r.Arguments["options"] = map[string]any{"mutation_id": id} + } + r = editProbeWithOutput(r, mode) + } + hash, err := hashJSON(r) + if err != nil { + return editProbePlan{}, err + } + p.Steps = append(p.Steps, editProbeStep{Ordinal: i, ActivateNS: due - int64(editProbePeriod), DueNS: due, Editor: arrival.Editor, Marker: marker, MutationID: id, Request: r, RequestHash: hash, BeforeSymbol: beforeSymbol, Deleted: deleted, TargetSymbol: targetSymbol, FollowingSymbol: followingSymbol, ExpectedLineGap: expectedGap, Relocated: kind == "relocate", DestinationFile: c.EditDestination, ExpectedRefusal: c.EditExpect == "refusal", Sender: sender, After: after, AfterEvent: afterEvent, File: file}) + } + return p, nil +} + +func (p editProbePlan) budget(deadline time.Duration, builds int) (time.Duration, error) { + if len(p.Steps) == 0 || deadline <= 0 || builds < 1 || builds > 2 { + return 0, errors.New("invalid finite edit-probe budget") + } + tail := p.Steps[len(p.Steps)-1].DueNS + if int64(deadline) > math.MaxInt64/int64(builds) { + return 0, errors.New("edit-probe deadline overflow") + } + drain := int64(deadline) * int64(builds) + if p.NoEditWindow < 0 || tail > math.MaxInt64-drain || tail+drain > math.MaxInt64-p.NoEditWindow { + return 0, errors.New("edit-probe run budget overflow") + } + return time.Duration(p.NoEditWindow + tail + drain), nil +} + +func editProbeOperations(plan editProbePlan, c Corpus) []request { + var operations []request + for _, step := range plan.Steps { + operations = append(operations, step.Request) + for _, oracle := range editProbeOracles(c, step) { + operations = append(operations, oracle.call) + } + } + return operations +} + +// Each finite edit task waits against the same absolute origin. A blocked ACK +// cannot shift another offered arrival. Callbacks must honor the common context; +// all owned tasks join before the caller tears down sessions or the daemon. +func runEditProbeArrivals(ctx context.Context, p editProbePlan, origin time.Time, activate, dispatch func(context.Context, editProbeStep), sleep func(context.Context, time.Duration) error) error { + var workers sync.WaitGroup + for _, step := range p.Steps { + workers.Go(func() { + if sleep(ctx, time.Until(origin.Add(time.Duration(step.ActivateNS)))) != nil { + return + } + activate(ctx, step) + if sleep(ctx, time.Until(origin.Add(time.Duration(step.DueNS)))) != nil { + return + } + dispatch(ctx, step) + }) + } + workers.Wait() + return ctx.Err() +} diff --git a/bench/load/edit_probe_plan_test.go b/bench/load/edit_probe_plan_test.go new file mode 100644 index 000000000..90e3916ce --- /dev/null +++ b/bench/load/edit_probe_plan_test.go @@ -0,0 +1,282 @@ +package main + +import ( + "context" + "math" + "reflect" + "strings" + "sync" + "testing" + "time" +) + +func editProbeCorpus() Corpus { + c := sampleCorpus() + c.EditAnchor = "func Caller() int { return Example() }" + c.EditTemplate = "\nfunc {{marker}}() int { return Example() }\n" + return c +} + +func TestEditProbePlanIsFiniteSeededAndSourceIndependent(t *testing.T) { + c := editProbeCorpus() + a, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + c.Source = "/another/private/source" + b, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(a, b) || len(a.Steps) != 100 { + t.Fatal("same pinned workload/seed changed offered plan") + } + hashes := map[string]bool{} + last := int64(0) + for i, step := range a.Steps { + if step.Ordinal != i || step.DueNS <= last || step.ActivateNS != step.DueNS-int64(editProbePeriod) || hashes[step.RequestHash] { + t.Fatalf("bad unique planned step%+v", step) + } + hashes[step.RequestHash] = true + last = step.DueNS + } + changed, err := makeEditProbePlan(c, 43, 100) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(a.Steps, changed.Steps) { + t.Fatal("seed did not change schedule") + } + budget, err := a.budget(time.Minute, 2) + if err != nil || budget != time.Duration(last)+2*time.Minute { + t.Fatalf("budget%v %v", budget, err) + } + if _, err := a.budget(time.Duration(math.MaxInt64), 2); err == nil { + t.Fatal("overflow budget accepted") + } +} + +func TestEditProbePlanRejectsUnselectedOrInvalidCells(t *testing.T) { + for _, name := range []string{"count", "language", "missing_anchor", "no_callee", "missing_call", "no_body", "two_markers", "bad_go"} { + t.Run(name, func(t *testing.T) { + c := editProbeCorpus() + count := 100 + switch name { + case "count": + count = 99 + case "language": + c.File = "sample.rs" + case "missing_anchor": + c.EditAnchor = "" + case "no_callee": + c.Symbol = "" + case "missing_call": + c.EditTemplate = "func {{marker}}() int { return 1 }" + case "no_body": + c.EditTemplate = "func {{marker}}() int" + case "two_markers": + c.EditTemplate += "// {{marker}}" + case "bad_go": + c.EditTemplate = "func {{marker}}( {" + } + if _, err := makeEditProbePlan(c, 1, count); err == nil { + t.Fatal("invalid cell accepted") + } + }) + } +} + +func TestEditProbeAbsoluteArrivalsDoNotWaitForEarlierACK(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + firstEntered, secondEntered, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(unblock) + plan := editProbePlan{Steps: []editProbeStep{{Ordinal: 0}, {Ordinal: 1}}} + done := make(chan error, 1) + go func() { + done <- runEditProbeArrivals(ctx, plan, time.Now(), func(context.Context, editProbeStep) {}, func(ctx context.Context, step editProbeStep) { + if step.Ordinal == 0 { + close(firstEntered) + select { + case <-release: + case <-ctx.Done(): + } + } else { + close(secondEntered) + } + }, waitContext) + }() + select { + case <-firstEntered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case <-secondEntered: + case <-ctx.Done(): + t.Fatal("blocked earlier ACK shifted offered later arrival") + } + unblock() + if err := <-done; err != nil { + t.Fatal(err) + } +} + +// The default cell (one editor, no readers, JSON, MCP sender) is the paired +// no-regression workload. Optional dimensions must leave its offered plan, plan +// hash and operation list byte-identical; changing a constant below needs an +// explicit compatibility decision, not a regenerated value. +func TestEditProbeDefaultPlanIdentityIsPinned(t *testing.T) { + c := editProbeCorpus() + hundred, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + one, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + toonCorpus := c + toonCorpus.EditOutput = "toon" + toonPlan, err := makeEditProbePlan(toonCorpus, 42, 100) + if err != nil { + t.Fatal(err) + } + o := runOptions{Scenario: editProbeScenario, Profile: "agent", Agents: 0, Editors: 1, Seed: 42, Duration: time.Minute, Deadline: 120 * time.Second, WarmTimeout: 30 * time.Minute, StopTimeout: 30 * time.Second, SweepRepetitions: 15, SweepBudget: 10 * time.Minute, EditCount: 100} + report, err := newRunReport(o, []Corpus{c}, "", Host{}, "run") + if err != nil { + t.Fatal(err) + } + pinned := []struct { + name string + value any + want string + }{ + {"100-edit plan", hundred, "0b583c59bde46c199bb4b7baa674222b29a0dc17177f0cb46e243e9e4e648ef0"}, + {"1-edit plan", one, "616f10cc464b12060ae6b24e26218a92cee77fcc897d769f51eac20e8b7e6061"}, + {"TOON plan", toonPlan, "15d4158514a8dd1312ca426f38765e09386f27144eaf0bc9d548640649fd100d"}, + {"operations", editProbeOperations(hundred, c), "2fd3b48bf5c4357757b96972c99ec5e6bdd79b0ab369ea229390f73fdb205b73"}, + } + for _, pin := range pinned { + got, err := hashJSON(pin.value) + if err != nil { + t.Fatal(err) + } + if got != pin.want { + t.Errorf("%s identity changed: %s", pin.name, got) + } + } + if report.RequestPlan != "eefb64b2f6272b3094fcbc908ebf6df6d0d5bfdc7d1b27570b7f0bd960b5f43d" { + t.Errorf("request plan hash changed: %s", report.RequestPlan) + } +} + +func TestEditProbeEditorStreamsAreIndependentOpenLoopPoisson(t *testing.T) { + c := editProbeCorpus() + one, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + streams := func(p editProbePlan) map[int][]int64 { + out := map[int][]int64{} + for _, step := range p.Steps { + out[step.Editor] = append(out[step.Editor], step.DueNS) + } + return out + } + plans := map[int]editProbePlan{} + for _, editors := range []int{2, 4} { + p, err := makeEditProbeTrafficPlan(c, 42, 100, editors) + if err != nil { + t.Fatal(err) + } + again, err := makeEditProbeTrafficPlan(c, 42, 100, editors) + if err != nil || !reflect.DeepEqual(p, again) { + t.Fatal("same seed/editor count changed the offered plan", err) + } + if p.Editors != editors || p.SessionModel != editProbeFixedEditorModel || len(p.Steps) != 100 { + t.Fatalf("editor plan identity: %d %q %d", p.Editors, p.SessionModel, len(p.Steps)) + } + last := editProbeStep{DueNS: -1} + for i, step := range p.Steps { + // Markers and mutation IDs stay keyed by the global ordinal. + if step.Ordinal != i || step.Marker != one.Steps[i].Marker || step.MutationID != one.Steps[i].MutationID || step.ActivateNS != step.DueNS-int64(editProbePeriod) || step.Editor < 0 || step.Editor >= editors { + t.Fatalf("bad merged step %+v", step) + } + if step.DueNS < last.DueNS || step.DueNS == last.DueNS && step.Editor <= last.Editor { + t.Fatalf("merge order is not due time then editor: %+v after %+v", step, last) + } + last = step + } + byEditor := streams(p) + if len(byEditor) != editors { + t.Fatalf("an editor offered no arrivals: %v", byEditor) + } + // Editor 0 keeps the one-editor stream; adding editors cannot shift it. + for k, due := range byEditor[0] { + if due != one.Steps[k].DueNS { + t.Fatalf("editor 0 arrival %d moved: %d != %d", k, due, one.Steps[k].DueNS) + } + } + plans[editors] = p + } + // Each editor's gaps come from its own stream, independent of how many + // other editors exist; the merged plan only truncates each stream. + two, four := streams(plans[2]), streams(plans[4]) + for k := range min(len(two[1]), len(four[1])) { + if two[1][k] != four[1][k] { + t.Fatalf("editor 1 stream depends on editor count at arrival %d", k) + } + } + if reflect.DeepEqual(four[0][:min(len(four[0]), len(four[1]))], four[1][:min(len(four[0]), len(four[1]))]) { + t.Fatal("editors share one arrival stream") + } + shifted, err := makeEditProbeTrafficPlan(c, 43, 100, 4) + if err != nil || reflect.DeepEqual(shifted.Steps, plans[4].Steps) { + t.Fatal("seed did not change the several-editor schedule", err) + } +} + +func TestEditProbeEditorCountRefusals(t *testing.T) { + c := editProbeCorpus() + for _, cell := range []struct{ count, editors int }{{100, 0}, {100, editProbeMaxEditors + 1}, {1, 2}, {100, -1}} { + if _, err := makeEditProbeTrafficPlan(c, 42, cell.count, cell.editors); err == nil { + t.Fatalf("invalid editor cell accepted: %+v", cell) + } + } + for _, cell := range []struct{ count, editors int }{{1, 1}, {100, 1}, {100, editProbeMaxEditors}} { + if _, err := makeEditProbeTrafficPlan(c, 42, cell.count, cell.editors); err != nil { + t.Fatalf("valid editor cell refused: %+v: %v", cell, err) + } + } + // Precomputed whole-file writes would silently drop a concurrent editor's + // marker; a batch's old_string precondition fails visibly instead. + write, batch := editProbeWriteCorpus(t, "go"), editProbeWriteCorpus(t, "go") + batch.EditTool = "edit.batch" + if _, err := makeEditProbeTrafficPlan(write, 42, 100, 1); err != nil { + t.Fatalf("one-editor edit.write refused: %v", err) + } + if _, err := makeEditProbeTrafficPlan(write, 42, 100, 2); err == nil || !strings.Contains(err.Error(), "silently drop") { + t.Fatalf("several-editor edit.write planned: %v", err) + } + if _, err := makeEditProbeTrafficPlan(batch, 42, 100, 2); err != nil { + t.Fatalf("several-editor edit.batch refused: %v", err) + } + base := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--worktrees", "0"} + for _, editors := range []string{"1", "4", "32"} { + if o, err := parseRun(append(base, "--edit-agents", editors)); err != nil || o.Editors < 1 { + t.Fatalf("--edit-agents %s refused: %v", editors, err) + } + } + for _, extra := range [][]string{{"--edit-agents", "0"}, {"--edit-agents", "33"}, {"--edit-agents", "2", "--edit-count", "1"}, {"--edit-agents", "8", "--host", "floor"}} { + if _, err := parseRun(append(base, extra...)); err == nil { + t.Fatalf("invalid edit-and-probe editors accepted: %v", extra) + } + } + paired := []string{"--build", "main=/private/main", "--build", "candidate=/private/candidate", "--manifest", "/private/manifest.json", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--worktrees", "0", "--edit-agents", "2"} + if _, _, err := parsePaired(paired); err == nil { + t.Fatal("paired no-regression cell widened past one editor") + } +} diff --git a/bench/load/edit_probe_python.go b/bench/load/edit_probe_python.go new file mode 100644 index 000000000..2c97312be --- /dev/null +++ b/bench/load/edit_probe_python.go @@ -0,0 +1,61 @@ +package main + +import ( + "go/token" + "strings" +) + +// This fixture cell permits only top-level zero-argument definitions with one +// four-space-indented return statement. Extra/nested bodies, strings, inline +// comments and unsupported syntax are refused rather than interpreted loosely. +func editProbePythonFixture(source, marker, callee string) (valid, correct bool) { + lines := strings.Split(source, "\n") + functions, matches := 0, 0 + for i := 0; i < len(lines); i++ { + line := strings.TrimSuffix(lines[i], "\r") + if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { + continue + } + if !strings.HasPrefix(line, "def ") || !strings.HasSuffix(line, "():") { + return false, false + } + name := strings.TrimSuffix(strings.TrimPrefix(line, "def "), "():") + if !token.IsIdentifier(name) { + return false, false + } + i++ + for i < len(lines) && strings.TrimSpace(lines[i]) == "" { + i++ + } + if i >= len(lines) { + return false, false + } + body := strings.TrimSuffix(lines[i], "\r") + if !strings.HasPrefix(body, " return ") { + return false, false + } + expression := strings.TrimSpace(strings.TrimPrefix(body, " return ")) + called := "" + if strings.HasSuffix(expression, "()") { + called = strings.TrimSuffix(expression, "()") + if !token.IsIdentifier(called) { + return false, false + } + } else { + if expression == "" { + return false, false + } + for _, digit := range expression { + if digit < '0' || digit > '9' { + return false, false + } + } + } + functions++ + if name == marker { + matches++ + correct = called == callee + } + } + return functions > 0, matches == 1 && correct +} diff --git a/bench/load/edit_probe_python_test.go b/bench/load/edit_probe_python_test.go new file mode 100644 index 000000000..c8c6a866d --- /dev/null +++ b/bench/load/edit_probe_python_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func editProbePythonCorpus() Corpus { + c := sampleCorpus() + c.File, c.Symbol = "main.py", "main.py::Example" + c.EditAnchor = "def Caller():\n return Example()" + c.EditTemplate = "\ndef {{marker}}():\n return Example()\n" + return c +} + +func TestEditProbePythonExplicitCellAndFixture(t *testing.T) { + c := editProbePythonCorpus() + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Language != "python" || plan.Kind != "add_intrafile_call" || plan.Tool != "edit.file" || plan.Mode != "json" || len(plan.Steps) != 100 { + t.Fatalf("wrong cell: %+v", plan) + } + goPlan, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + for i := range plan.Steps { + if plan.Steps[i].DueNS != goPlan.Steps[i].DueNS || plan.Steps[i].ActivateNS != goPlan.Steps[i].ActivateNS { + t.Fatal("language changed offered traffic") + } + } + fixture, err := os.ReadFile("testdata/edit-probe/python/main.py") + if err != nil { + t.Fatal(err) + } + valid, _ := editProbePythonFixture(string(fixture), "", "") + if !valid || strings.Count(string(fixture), "def ") != 10 || strings.Count(string(fixture), c.EditAnchor) != 1 { + t.Fatal("fixture does not meet explicit syntax/initial symbol gate") + } + step := plan.Steps[0] + source := strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + for oracle, body := range map[string]map[string]any{ + "source": {"id": "sample/main.py::" + step.Marker, "name": step.Marker, "file_path": "sample/main.py", "source": source}, + "file": {"path": "sample/main.py", "content": string(fixture) + source}, + "text": {"matches": []any{map[string]any{"path": "sample/main.py", "line": json.Number("31"), "text": "def " + step.Marker + "():"}}}, + } { + if !editProbeCorrect(oracle, body, c, step, plan.Callee) { + t.Fatalf("actual %s fixture rejected", oracle) + } + } + for _, oracle := range []string{"callers", "call_chain"} { + body := map[string]any{"edges": []any{map[string]any{"from": "sample/main.py::" + step.Marker, "to": "sample/main.py::Example", "kind": "calls"}}} + if !editProbeCorrect(oracle, body, c, step, plan.Callee) { + t.Fatalf("actual %s edge rejected", oracle) + } + } +} + +func TestEditProbePythonRejectsFalseSourceAndBoundaries(t *testing.T) { + c := editProbePythonCorpus() + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + source := strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + for name, changed := range map[string]string{ + "wrong_callee": strings.ReplaceAll(source, "Example()", "Other()"), + "comment_only": "# " + strings.ReplaceAll(strings.TrimSpace(source), "\n", "\n# "), + "string_only": "\"\"\"" + source + "\"\"\"", + "stale_body": strings.ReplaceAll(source, "Example()", "1"), + "comment_call": strings.ReplaceAll(source, "return Example()", "return 1 # Example()"), + "tab_body": strings.ReplaceAll(source, " return", "\treturn"), + "wrong_indent": strings.ReplaceAll(source, " return", " return"), + "nested_definition": "def Outer():\n " + strings.ReplaceAll(strings.TrimSpace(source), "\n", "\n ") + "\n", + "extra_body": source + " return Other()\n", + "duplicate": source + source, + } { + t.Run(name, func(t *testing.T) { + body := map[string]any{"id": "sample/main.py::" + step.Marker, "name": step.Marker, "file_path": "sample/main.py", "source": changed, "fresh": true} + if editProbeCorrect("source", body, c, step, plan.Callee) { + t.Fatal("false source became actual-call proof") + } + if editProbeCorrect("file", map[string]any{"path": "sample/main.py", "content": changed, "fresh": true}, c, step, plan.Callee) { + t.Fatal("false file became actual-call proof") + } + }) + } + for _, field := range []string{"id", "file_path"} { + body := map[string]any{"id": "sample/main.py::" + step.Marker, "name": step.Marker, "file_path": "sample/main.py", "source": source} + body[field] = "sample/foreign.py::" + step.Marker + if editProbeCorrect("source", body, c, step, plan.Callee) { + t.Fatalf("wrong %s accepted", field) + } + } + if editProbeCorrect("file", map[string]any{"path": "sample/foreign.py", "content": source}, c, step, plan.Callee) { + t.Fatal("wrong file accepted") + } + for _, text := range []string{"# def " + step.Marker + "():", "\"def " + step.Marker + "():\"", " def " + step.Marker + "():"} { + body := map[string]any{"matches": []any{map[string]any{"path": "sample/main.py", "line": json.Number("1"), "text": text}}} + if editProbeCorrect("text", body, c, step, plan.Callee) { + t.Fatal("comment/string/nested declaration became top-level text proof") + } + } +} + +func TestEditProbePythonRejectsInvalidTemplates(t *testing.T) { + base := editProbePythonCorpus() + for _, template := range []string{ + strings.ReplaceAll(base.EditTemplate, "Example()", "Other()"), + strings.ReplaceAll(base.EditTemplate, " return", "\treturn"), + "# " + strings.ReplaceAll(strings.TrimSpace(base.EditTemplate), "\n", "\n# "), + strings.ReplaceAll(base.EditTemplate, "Example()", "\"Example()\""), + } { + c := base + c.EditTemplate = template + if _, err := makeEditProbePlan(c, 42, 1); err == nil { + t.Fatal("invalid Python fixture template admitted") + } + } +} diff --git a/bench/load/edit_probe_read_repair.go b/bench/load/edit_probe_read_repair.go new file mode 100644 index 000000000..3c48fb2fc --- /dev/null +++ b/bench/load/edit_probe_read_repair.go @@ -0,0 +1,112 @@ +package main + +import ( + "encoding/hex" + "errors" + "math" +) + +var editProbeRepairDeepPhases = [...]string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived"} + +func isEditProbeRepairDeepPhase(name string) bool { + for _, phase := range editProbeRepairDeepPhases { + if name == phase { + return true + } + } + return false +} + +// R is checked against its own accepted owner/bytes/window. It is never added +// to B or used to replace P's null deep values. Wire/session fields are bounded +// admission causes, not proof of a uniquely identified reader RPC. +func checkEditProbeReadRepair(raw any, status map[string]any, receipt string, requested uint64) (editProbeWindow, error) { + unavailable := editProbeWindow{} + r, owner, wall, err := editProbeReadRepairIdentity(raw, status, receipt, requested) + if err != nil { + return unavailable, err + } + a, _ := r["phase_accounting"].(map[string]any) + if !editProbeReadRepairPhaseOwner(a, owner, wall) || a["sum_state"] != "partial" || a["observed_partition_complete"] != true { + return unavailable, errors.New("own-window phase evidence unavailable") + } + phases, _ := a["phases_ms"].(map[string]any) + states, _ := a["phase_states"].(map[string]any) + for _, name := range append(editProbeRepairDeepPhases[:], "repair_control") { + if phases[name] == nil || states[name] != "measured" { + return unavailable, errors.New("missing applicable publisher phase " + name) + } + } + for name, value := range phases { + if value == nil && states[name] == "measured" { + return unavailable, errors.New("missing measured R value " + name) + } + if value != nil && (!isEditProbeRepairDeepPhase(name) && name != "repair_control" && name != "patch_analysis" || states[name] != "measured") { + return unavailable, errors.New("unexpected/unmeasured R phase " + name) + } + } + window, err := checkEditProbeWindowNS(wall, phases) + subtotal, subtotalErr := strconvEditProbeUint(a["observed_subtotal_ns"]) + residual, residualErr := strconvEditProbeUint(a["unattributed_ns"]) + if err != nil || window.ResidualNS != 0 || subtotalErr != nil || subtotal != uint64(window.SumNS) || residualErr != nil || residual != 0 { + return unavailable, errors.New("read-repair scalar partition disagrees with its own window") + } + return window, nil +} + +func editProbeReadRepairIdentity(raw any, status map[string]any, receipt string, requested uint64) (map[string]any, uint64, int64, error) { + r, _ := raw.(map[string]any) + owner, ownerErr := strconvEditProbeUint(r["owner_generation"]) + req, reqErr := strconvEditProbeUint(r["requested_generation"]) + wall, wallErr := strconvEditProbeUint(r["window_ns"]) + blob, _ := r["accepted_blob_sha"].(string) + _, blobErr := hex.DecodeString(blob) + // The public commit receipt and caller-chosen mutation_id are not the + // internal admission/publication key. Use the original requested timeline, + // never the independently keyed coalesced physical winner. + original, _ := status["publication_phases"].(map[string]any) + internal, _ := original["key"].(string) + ticket, ticketErr := strconvEditProbeUint(original["ticket"]) + if receipt == "" || status["receipt"] != receipt || internal == "" || r["requested_receipt"] != internal || + ticketErr != nil || ticket != requested || original["source"] != "mcp_edit" || original["clock"] != "monotonic" || + ownerErr != nil || owner == 0 || reqErr != nil || req != requested || + wallErr != nil || wall > math.MaxInt64 || len(blob) != 40 || blobErr != nil || blob != status["new_sha"] || + r["scope"] != "read_assisted_reconcile" || r["clock"] != "monotonic" || + r["additive_with_requested_background"] != false || r["background_relationship"] != "unavailable" { + return nil, 0, 0, errors.New("accepted owner/bytes/request/window relation unavailable") + } + causes, _ := r["reader_causes"].([]any) + if len(causes) == 0 || len(causes) > 8 { + return nil, 0, 0, errors.New("bounded reader admissions unavailable") + } + seen := map[uint64]bool{} + for _, value := range causes { + cause, _ := value.(map[string]any) + generation, err := strconvEditProbeUint(cause["admission_generation"]) + session, _ := cause["session"].(string) + key, _ := cause["request_id_key"].(string) + if err != nil || generation == 0 || generation > owner || seen[generation] || session == "" || key == "" { + return nil, 0, 0, errors.New("invalid reader admission identity") + } + seen[generation] = true + } + return r, owner, int64(wall), nil +} + +func editProbeReadRepairPhaseOwner(a map[string]any, owner uint64, wall int64) bool { + phaseOwner, phaseOwnerErr := strconvEditProbeUint(a["owner_generation"]) + phaseWall, phaseWallErr := strconvEditProbeUint(a["window_ns"]) + return phaseOwnerErr == nil && phaseOwner == owner && phaseWallErr == nil && phaseWall == uint64(wall) && + a["scope"] == "read_assisted_reconcile" && a["clock"] == "monotonic" && a["additive_with_requested_background"] == false +} + +// Unlike absent late R/B, an owned frozen R phase failure cannot gain laps. +func editProbeImmutablePartialReadRepair(raw any, status map[string]any, receipt string, requested uint64) bool { + r, owner, wall, err := editProbeReadRepairIdentity(raw, status, receipt, requested) + if err != nil { + return false + } + a, _ := r["phase_accounting"].(map[string]any) + return editProbeReadRepairPhaseOwner(a, owner, wall) && a["observed_partition_complete"] == false && + (a["sum_state"] == "missing_laps" || a["sum_state"] == "invalid_order" || a["sum_state"] == "unclassified_gap") +} diff --git a/bench/load/edit_probe_read_repair_test.go b/bench/load/edit_probe_read_repair_test.go new file mode 100644 index 000000000..043e86048 --- /dev/null +++ b/bench/load/edit_probe_read_repair_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func editProbeRepairAccountingFixture() map[string]any { + body := editProbeAccountingFixture() + body["new_sha"] = "ce013625030ba8dba906f756967f9e9ca394464a" + body["mutation_id"] = "caller-logical-id" + original := body["publication_phases"].(map[string]any) + original["ticket"], original["source"], original["clock"] = json.Number("1"), "mcp_edit", "monotonic" + winner := body["coalesced_publication"].(map[string]any) + pub := winner["publication_phases"].(map[string]any) + pub["clock"], pub["terminal"] = "monotonic", true + physical := winner["timings"].(map[string]any)["phase_timing"].(map[string]any) + physical["clock"], physical["patch_wall_ms"] = "monotonic", json.Number("5") + a := physical["patch_accounting"].(map[string]any) + a["scope"], a["sum_state"], a["execution_disposition"] = "accepted_winning_physical_attempt", "partial", "native_mtime_duplicate" + phases, states := a["phases_ms"].(map[string]any), a["phase_states"].(map[string]any) + b := body["requested_background"].(map[string]any) + b["background_wall_ms"] = json.Number("12") + bPhases := b["phases_ms"].(map[string]any) + for _, name := range []string{"gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", "executor_control"} { + phases[name], states[name] = nil, "not_applicable" + delete(bPhases, name) + } + rPhases, rStates := map[string]any{}, map[string]any{} + for _, name := range append(editProbeRepairDeepPhases[:], "repair_control") { + rPhases[name], rStates[name] = json.Number("1"), "measured" + } + body["read_assisted_publication"] = map[string]any{ + "scope": "read_assisted_reconcile", "clock": "monotonic", "owner_generation": json.Number("3"), + "requested_receipt": "original", "requested_generation": json.Number("1"), "accepted_blob_sha": body["new_sha"], + "window_ns": json.Number("8000000"), "additive_with_requested_background": false, "background_relationship": "unavailable", + "reader_causes": []any{ + map[string]any{"session": "reader", "request_id_key": "n:7", "admission_generation": json.Number("2")}, + map[string]any{"session": "reader", "request_id_key": "n:7", "admission_generation": json.Number("3")}, + }, + "phase_accounting": map[string]any{ + "scope": "read_assisted_reconcile", "clock": "monotonic", "owner_generation": json.Number("3"), + "window_ns": json.Number("8000000"), "observed_subtotal_ns": json.Number("8000000"), "unattributed_ns": json.Number("0"), + "sum_state": "partial", "observed_partition_complete": true, "additive_with_requested_background": false, + "phases_ms": rPhases, "phase_states": rStates, + }, + } + return body +} + +func TestEditProbeValidatesIndependentPublisherWithoutSummingOverlappingWindows(t *testing.T) { + body := editProbeRepairAccountingFixture() + got := checkEditProbeAccounting(body, "receipt-a") + if got.State != "observed_single_window_checks_pass" || got.R == nil || got.R.WallNS != 8000000 || got.B == nil || got.B.WallNS != 12000000 || got.P == nil || got.P.WallNS != 5000000 { + t.Fatalf("independent R3 confused with requested1/watcher2 or summed into B/P: %+v", got) + } + // Reused wire IDs are still only causes; their admission generations differ. + physical := body["coalesced_publication"].(map[string]any)["timings"].(map[string]any)["phase_timing"].(map[string]any) + if physical["patch_accounting"].(map[string]any)["phases_ms"].(map[string]any)["apply"] != nil { + t.Fatal("R retroactively filled the no-op P") + } +} + +func TestEditProbeMissingOrContradictoryPublisherRefusesSuccess(t *testing.T) { + for _, bad := range []string{"missing", "bytes", "receipt", "requested", "owner", "bounds", "missing_phase", "phase_state", "extra_phase", "sum", "additive", "duplicate_admission", "missing_internal", "public_receipt", "caller_id", "missing_original", "original_ticket", "original_source", "original_clock", "swapped_winner"} { + t.Run(bad, func(t *testing.T) { + body := editProbeRepairAccountingFixture() + r := body["read_assisted_publication"].(map[string]any) + a := r["phase_accounting"].(map[string]any) + switch bad { + case "missing": + delete(body, "read_assisted_publication") + case "bytes": + r["accepted_blob_sha"] = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391" + case "receipt": + r["requested_receipt"] = "foreign" + case "missing_internal": + delete(r, "requested_receipt") + case "public_receipt": + r["requested_receipt"] = body["receipt"] + case "caller_id": + r["requested_receipt"] = body["mutation_id"] + case "missing_original": + delete(body, "publication_phases") + case "original_ticket": + body["publication_phases"].(map[string]any)["ticket"] = json.Number("2") + case "original_source": + body["publication_phases"].(map[string]any)["source"] = "canonical_coalesced_attempt" + case "original_clock": + body["publication_phases"].(map[string]any)["clock"] = "unavailable" + case "swapped_winner": + body["publication_phases"] = body["coalesced_publication"].(map[string]any)["publication_phases"] + case "requested": + r["requested_generation"] = json.Number("2") + case "owner": + a["owner_generation"] = json.Number("4") + case "bounds": + a["window_ns"] = json.Number("9000000") + case "missing_phase": + a["phases_ms"].(map[string]any)["apply"] = nil + case "phase_state": + a["phase_states"].(map[string]any)["apply"] = "not_observed" + case "extra_phase": + a["phases_ms"].(map[string]any)["lane_wait"] = json.Number("1") + case "sum": + a["observed_subtotal_ns"] = json.Number("7") + case "additive": + r["additive_with_requested_background"] = true + case "duplicate_admission": + r["reader_causes"].([]any)[1].(map[string]any)["admission_generation"] = json.Number("2") + } + got := checkEditProbeAccounting(body, "receipt-a") + if got.State != "unavailable" || got.R != nil || len(got.Reasons) == 0 { + t.Fatalf("unproven R was accepted: %+v", got) + } + }) + } +} + +func TestEditProbeCollectsLateReadRepairAndBackgroundInEitherOrder(t *testing.T) { + for _, order := range []string{"P_then_R_then_B", "P_and_B_then_R"} { + t.Run(order, func(t *testing.T) { + body := editProbeRepairAccountingFixture() + r, b := body["read_assisted_publication"], body["requested_background"] + delete(body, "read_assisted_publication") + if order == "P_then_R_then_B" { + delete(body, "requested_background") + } + early := checkEditProbeAccounting(body, "receipt-a") + if early.State != "unavailable" || editProbeFinished(true, true, true, early) { + t.Fatalf("complete duplicate P stopped before late evidence: %+v", early) + } + body["read_assisted_publication"] = r + if order == "P_then_R_then_B" { + middle := checkEditProbeAccounting(body, "receipt-a") + if middle.State != "unavailable" || middle.R == nil || editProbeFinished(true, true, true, middle) { + t.Fatalf("valid R stopped before late B: %+v", middle) + } + } + body["requested_background"] = b + late := checkEditProbeAccounting(body, "receipt-a") + if late.State != "observed_single_window_checks_pass" || !editProbeFinished(true, true, true, late) { + t.Fatalf("late matching evidence never completed: %+v", late) + } + if early.R != nil || early.State != "unavailable" { + t.Fatal("late collection changed earlier evidence") + } + }) + } +} + +func TestEditProbeStopsOnlyOwnedImmutableReadRepairPhaseFailures(t *testing.T) { + for _, state := range []string{"missing_laps", "invalid_order", "unclassified_gap"} { + t.Run(state, func(t *testing.T) { + body := editProbeRepairAccountingFixture() + delete(body, "requested_background") + r := body["read_assisted_publication"].(map[string]any) + a := r["phase_accounting"].(map[string]any) + a["observed_partition_complete"], a["sum_state"] = false, state + a["phases_ms"].(map[string]any)["apply"] = nil + a["phase_states"].(map[string]any)["apply"] = "not_observed" + got := checkEditProbeAccounting(body, "receipt-a") + if got.State != "terminal_partial_physical" || !editProbeFinished(true, true, true, got) { + t.Fatalf("known owned final R cannot gain missing laps: %+v", got) + } + r["accepted_blob_sha"] = "foreign" + got = checkEditProbeAccounting(body, "receipt-a") + if got.State != "unavailable" || editProbeFinished(true, true, true, got) { + t.Fatalf("foreign R was used as terminal evidence: %+v", got) + } + }) + } +} diff --git a/bench/load/edit_probe_reads.go b/bench/load/edit_probe_reads.go new file mode 100644 index 000000000..0ebda8657 --- /dev/null +++ b/bench/load/edit_probe_reads.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "sync" + "time" +) + +const editProbeNoEditWindow = 30 * time.Second +const editProbeMaxReaders = 32 + +// Pure readers run the ordinary reading plan beside the edits. Each call is +// classed when it is issued: before the edit origin it is the same run's +// no-edit baseline, afterwards it is under edit while any sent edit's receipt +// is pending (not yet observed terminal), and between edits otherwise. An edit +// sent while the read is in flight does not reclass it. +type editProbeReadStats struct { + Calls int `json:"calls"` + Busy int `json:"busy_refusals"` + Errors int `json:"errors"` + Censored int `json:"censored"` + Latency editProbeQuantiles `json:"latency_ms"` +} + +// Read stats per operation, then per class. +type editProbeReads map[string]map[string]editProbeReadStats + +type editProbeRead struct { + operation, class, outcome string + latency float64 +} + +func (r *editProbeRuntime) readClass(at time.Time) string { + switch { + case at.Before(r.origin): + return "no_edit_baseline" + case r.pendingEdits.Load() > 0: + return "under_edit" + default: + return "between_edits" + } +} + +func (r *editProbeRuntime) startReaders(ctx context.Context, readers *sync.WaitGroup) { + repo := r.p.Repos[0] + plan, _ := readingPlan(repo.Corpus) + if len(plan) == 0 { + return + } + for reader, client := range r.readers { + readers.Go(func() { + pace := newPacing(profile(r.options.Profile), uint64(r.options.Seed)+uint64(reader)) + for step := 0; ctx.Err() == nil; step++ { + if waitContext(ctx, pace.think()) != nil { + return + } + var burst sync.WaitGroup + for slot := range pace.burst() { + burst.Go(func() { + req := plan[(reader+step*5+slot)%len(plan)] + class := r.readClass(time.Now()) + _, rows, _ := callTool(ctx, client, req, dispatchArgs(req, repo), "reader", reader, step, slot, r.options.Deadline, waitContext) + r.recordRead(req.name(), class, rows) + }) + } + burst.Wait() + } + }) + } +} + +func (r *editProbeRuntime) recordRead(operation, class string, rows []Attempt) { + read := editProbeRead{operation: operation, class: class, outcome: "error"} + if len(rows) > 0 { + last := rows[len(rows)-1] + read.outcome, read.latency = last.Outcome, last.LogicalMS + } + r.mu.Lock() + r.phase.Attempts = append(r.phase.Attempts, rows...) + r.reads = append(r.reads, read) + r.mu.Unlock() +} + +// Only successful calls enter a latency sample. A call cut off by its +// deadline or the run end is censored in the latency ranking; busy and failed +// calls have no latency. None is ever a zero latency. +func summarizeEditProbeReads(reads []editProbeRead, seed int64) editProbeReads { + if len(reads) == 0 { + return nil + } + type cell struct{ operation, class string } + samples := map[cell][]float64{} + expired := map[cell]int{} + stats := map[cell]*editProbeReadStats{} + for _, read := range reads { + key := cell{read.operation, read.class} + if stats[key] == nil { + stats[key] = &editProbeReadStats{} + } + s := stats[key] + s.Calls++ + switch read.outcome { + case "success": + samples[key] = append(samples[key], read.latency) + case "busy": + s.Busy++ + case "phase_censored": + s.Censored++ + case "product_deadline": + s.Errors++ + expired[key]++ + default: + s.Errors++ + } + } + out := editProbeReads{} + for key, s := range stats { + censored := s.Censored + expired[key] + s.Latency = editProbeQuantilesOf(samples[key], censored, s.Calls-len(samples[key])-censored, "ms", seed, "reads."+key.operation+"."+key.class) + if out[key.operation] == nil { + out[key.operation] = map[string]editProbeReadStats{} + } + out[key.operation][key.class] = *s + } + return out +} diff --git a/bench/load/edit_probe_reads_test.go b/bench/load/edit_probe_reads_test.go new file mode 100644 index 000000000..acf83ffe5 --- /dev/null +++ b/bench/load/edit_probe_reads_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestEditProbeNoEditWindowPrecedesActivationWithoutShift(t *testing.T) { + c := editProbeCorpus() + base, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + withReaders, err := planEditProbe(c, runOptions{Seed: 42, EditCount: 100, Editors: 1, Agents: 4}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(withReaders.Steps, base.Steps) || withReaders.Readers != 4 || withReaders.NoEditWindow != int64(editProbeNoEditWindow) { + t.Fatalf("readers shifted the offered edits or lost the window: readers=%d window=%d", withReaders.Readers, withReaders.NoEditWindow) + } + baseBudget, err := base.budget(time.Minute, 1) + if err != nil { + t.Fatal(err) + } + if budget, err := withReaders.budget(time.Minute, 1); err != nil || budget != baseBudget+editProbeNoEditWindow { + t.Fatalf("no-edit window not budgeted: %v %v", budget, err) + } + if none, err := planEditProbe(c, runOptions{Seed: 42, EditCount: 100, Editors: 1}); err != nil || !reflect.DeepEqual(none, base) { + t.Fatal("zero readers changed the default plan", err) + } + if _, err := planEditProbe(c, runOptions{Seed: 42, EditCount: 100, Editors: 1, Agents: editProbeMaxReaders + 1}); err == nil { + t.Fatal("reader cap not enforced") + } + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--edit-agents", "2", "--worktrees", "0"} + if _, err := parseRun(append(args, "--agents", "4")); err != nil { + t.Fatal(err) + } + if _, err := parseRun(append(args, "--agents", "33")); err == nil { + t.Fatal("33 readers accepted") + } + paired := []string{"--build", "main=/private/main", "--build", "candidate=/private/candidate", "--manifest", "/private/manifest.json", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "1", "--edit-agents", "1", "--worktrees", "0"} + if _, _, err := parsePaired(paired); err == nil { + t.Fatal("paired no-regression cell accepted readers") + } +} + +func TestEditProbeReadsClassifyByPendingEditsAndNeverSampleFailures(t *testing.T) { + r := &editProbeRuntime{origin: time.Now().Add(time.Hour)} + if got := r.readClass(time.Now()); got != "no_edit_baseline" { + t.Fatal(got) + } + r.origin = time.Now().Add(-time.Second) + r.pendingEdits.Add(1) + if got := r.readClass(time.Now()); got != "under_edit" { + t.Fatal(got) + } + r.pendingEdits.Add(-1) + if got := r.readClass(time.Now()); got != "between_edits" { + t.Fatal(got) + } + reads := []editProbeRead{ + {"read.summary", "under_edit", "success", 7}, + {"read.summary", "under_edit", "success", 5}, + {"read.summary", "under_edit", "busy", 0}, + {"read.summary", "under_edit", "phase_censored", 0}, + {"read.summary", "under_edit", "product_deadline", 0}, + {"read.summary", "no_edit_baseline", "success", 3}, + } + summary := summarizeEditProbeReads(reads, 1) + under := summary["read.summary"]["under_edit"] + // Deadline and run-end cuts rank above both successes; the busy read has + // no latency at all. + if under.Calls != 5 || under.Busy != 1 || under.Censored != 1 || under.Errors != 1 || under.Latency.Count != 2 || under.Latency.Censored != 2 || under.Latency.NotApplicable != 1 || *under.Latency.P50.Value != 7 || under.Latency.P95.Value != nil { + t.Fatalf("failed reads entered the latency sample: %+v", under) + } + if baseline := summary["read.summary"]["no_edit_baseline"]; baseline.Calls != 1 || *baseline.Latency.Max.Value != 3 { + t.Fatalf("baseline cell: %+v", baseline) + } + if summarizeEditProbeReads(nil, 1) != nil { + t.Fatal("no readers reported an empty read table") + } +} + +// Reads are under edit while a sent edit's receipt is pending. Once its own +// status is terminal, reads issued while the edit still waits for its probes +// are between edits. +func TestEditProbeReadsUnderEditEndWhenTheReceiptTurnsTerminal(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c := editProbeCorpus() + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte("package sample\n\n"+c.EditAnchor+"\n"), 0o600); err != nil { + t.Fatal(err) + } + var terminal atomic.Bool + run := newEditProbeStubRun(t, ctx, c, plan, dir, 1, func(frame stubFrame) (any, *rpcError) { + switch frame.Params["name"] { + case "edit": + return stubContent(map[string]any{"mutation_receipt": "receipt-1"}), nil + case "change": + return stubContent(map[string]any{"mutation_id": plan.Steps[0].MutationID, "graph_status": "fresh", "graph_status_terminal": terminal.Load()}), nil + } + return stubContent(map[string]any{}), nil // probes and readbacks never answer correctly + }) + run.options.Deadline = time.Second + run.activate(ctx, plan.Steps[0]) + awaitEditProbeFirstProbe(t, run.states[0]) + done := make(chan struct{}) + go func() { + defer close(done) + run.doEdit(ctx, plan.Steps[0]) + }() + await := func(want string) { + t.Helper() + for deadline := time.Now().Add(2 * time.Second); run.readClass(time.Now()) != want; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("read class never became %s", want) + } + } + } + await("under_edit") + terminal.Store(true) + await("between_edits") + select { + case <-done: + t.Fatal("the edit ended before its probes; a terminal receipt was not observed mid-loop") + default: + } + <-done + if got := run.readClass(time.Now()); got != "between_edits" || run.pendingEdits.Load() != 0 { + t.Fatalf("pending edits after the loop: %s %d", got, run.pendingEdits.Load()) + } +} + +func TestEditProbeReadersUseOwnSessionsAndClassAtIssue(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + var mu sync.Mutex + connections := map[int]int{} + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Method == "tools/call" { + mu.Lock() + connections[frame.Connection]++ + mu.Unlock() + } + return stubContent(map[string]any{}), nil + }) + var readers []*socketClient + for range 2 { + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + readers = append(readers, client) + } + // The saturation profile removes think time so the test stays short. + r := &editProbeRuntime{options: runOptions{Profile: "saturation", Seed: 1, Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Prefix: "sample", Corpus: editProbeCorpus()}}}, phase: &Phase{}, readers: readers} + r.origin = time.Now().Add(150 * time.Millisecond) + readCtx, stop := context.WithTimeout(ctx, 600*time.Millisecond) + defer stop() + var group sync.WaitGroup + r.startReaders(readCtx, &group) + group.Wait() + classes := map[string]int{} + for _, read := range r.reads { + classes[read.class]++ + } + if classes["no_edit_baseline"] == 0 || classes["between_edits"] == 0 || classes["under_edit"] != 0 { + t.Fatalf("reads not classed by issue time: %v", classes) + } + for _, attempt := range r.phase.Attempts { + if attempt.Role != "reader" || attempt.Agent < 0 || attempt.Agent > 1 { + t.Fatalf("reader attempt identity: %+v", attempt) + } + } + mu.Lock() + defer mu.Unlock() + if len(connections) != 2 || connections[1] == 0 || connections[2] == 0 { + t.Fatalf("readers did not use their own sessions: %v", connections) + } +} diff --git a/bench/load/edit_probe_refusal.go b/bench/load/edit_probe_refusal.go new file mode 100644 index 000000000..c500b1d37 --- /dev/null +++ b/bench/load/edit_probe_refusal.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + "unicode/utf8" +) + +// TypeScript and Python relocation is offered as the actual move, which the +// product refuses; Go relocation is supported and measured as a move. The +// refusal cell is explicit (edit_expect: "refusal"), never inferred. +func editProbeRelocationRefusalFixture(c Corpus) error { + ext := filepath.Ext(c.File) + if c.EditExpect != "refusal" || c.EditKind != "relocate" || ext != ".ts" && ext != ".py" { + return errors.New("unsupported language: move_symbol fixture supports Go only; TypeScript/Python relocation is an explicit edit_expect refusal cell") + } + if filepath.Base(c.File) != c.File || filepath.Ext(c.EditDestination) != ext || filepath.Base(c.EditDestination) != c.EditDestination || c.File == c.EditDestination || !utf8.ValidString(c.EditDestinationContent) { + return errors.New("relocation refusal fixture requires distinct root source and destination files of one language") + } + return editProbeInitialFixture(c) +} + +// The refusal must be the producer's one scoped message as the only payload, +// followed at most by the known auxiliary session notes. +func editProbeExactToolRefusal(raw json.RawMessage, want string) (string, bool) { + var envelope struct { + IsError bool `json:"isError"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &envelope) != nil || len(envelope.Content) == 0 { + return "", false + } + message := envelope.Content[0].Text + lastNoteKind := 0 + for _, item := range envelope.Content[1:] { + kind := editProbeSessionNoteKind(item.Text) + if item.Type != "text" || kind <= lastNoteKind { + return message, false + } + lastNoteKind = kind + } + return message, envelope.IsError && envelope.Content[0].Type == "text" && message == want +} + +// A refusal cell completes only on the exact scoped refusal, then the same +// editing session's immediate file readback and the original symbol's source +// show no change, and the source and destination bytes on disk are unchanged. +// Any acknowledgement, receipt or other message is a diagnostic failure. +func (r *editProbeRuntime) finishRefusal(ctx, callCtx context.Context, s *editProbeState, step editProbeStep, started time.Time, raw json.RawMessage, identity wireCallIdentity, err error) { + want := fmt.Sprintf("unsupported language: move_symbol only supports Go files (got %q)", r.p.Repos[0].Prefix+"/"+r.plan.File) + refusal := editProbeReadback{Kind: "relocation_refusal", Latency: unavailable("ms", "response endpoint unavailable"), ExpectedOutcome: "refused"} + if !identity.Finished.IsZero() { + refusal.Latency = measured(float64(identity.Finished.Sub(started))/float64(time.Millisecond), "ms") + } + message, exact := editProbeExactToolRefusal(raw, want) + refusal.Refusal, refusal.Correct = message, err != nil && exact + if !refusal.Correct && err != nil { + refusal.Error = err.Error() + } + s.mu.Lock() + s.record.Receipt, s.record.MutationID = "", "" + s.record.Ack = unavailable("ms", "relocation refused; its latency is the relocation_refusal readback") + s.record.Accounting = editProbeAccounting{State: "not_applicable_refused"} + s.record.PhasesMS = editProbePhases(nil, "refusal cell made no change") + s.record.Derived = unavailable("ms", "refusal cell made no change") + s.record.Readback = unavailable("ms", "refusal readbacks missing or incorrect") + s.record.Readbacks = append(s.record.Readbacks, refusal) + if !refusal.Correct { + s.record.Errors = append(s.record.Errors, "relocation was not refused with the exact producer message") + } + s.mu.Unlock() + readbacksCorrect, readbackEnd := refusal.Correct, identity.Finished + for index, req := range []request{ + editProbeWithOutput(request{Tool: "read", Operation: "file", Arguments: map[string]any{"operation": "file", "target": map[string]any{"file": r.plan.File}}}, r.plan.Mode), + {Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": step.BeforeSymbol}, "options": map[string]any{"context_lines": 0}}}, + } { + begin := time.Now() + data, readIdentity, readErr := r.perform(callCtx, s, r.editorFor(step), req, "readback:"+req.Operation, "readback", 100+index) + back := editProbeReadback{Kind: "file_unchanged", Latency: unavailable("ms", "response endpoint unavailable")} + if req.Operation == "source" { + back.Kind, back.Correct = "original_source_present", readErr == nil && editProbePresenceProof(data, r.p.Repos[0].Corpus, step.BeforeSymbol) + } else { + body, decodeErr := decodeEditProbeOutput(data, editProbeRequestOutput(req), req.name()) + content, _ := body["content"].(string) + path, _ := body["path"].(string) + back.Correct = readErr == nil && decodeErr == nil && content == string(r.original) && editProbeFixturePath(path, r.plan.File) + if decodeErr != nil { + back.Error = decodeErr.Error() + } + } + if readErr != nil { + back.Error = readErr.Error() + } + if !readIdentity.Finished.IsZero() { + back.Latency = measured(float64(readIdentity.Finished.Sub(begin))/float64(time.Millisecond), "ms") + readbackEnd = readIdentity.Finished + } + readbacksCorrect = readbacksCorrect && back.Correct && !readIdentity.Finished.IsZero() + s.mu.Lock() + s.record.Readbacks = append(s.record.Readbacks, back) + if !back.Correct { + s.record.Errors = append(s.record.Errors, "immediate "+back.Kind+" readback incorrect") + } + s.mu.Unlock() + } + unchanged := r.confirmRefusalUnchanged() + s.mu.Lock() + defer s.mu.Unlock() + if readbacksCorrect { + s.record.Readback = measured(float64(readbackEnd.Sub(started))/float64(time.Millisecond), "ms") + } + if unchanged != nil { + s.record.Errors = append(s.record.Errors, unchanged.Error()) + } + switch { + case ctx.Err() != nil: + s.record.Outcome = "phase_censored" + case callCtx.Err() != nil: + s.record.Outcome = "product_deadline" + case len(s.record.Errors) == 0: + s.record.Outcome = "complete" + default: + s.record.Outcome = "diagnostic_failure" + } +} + +// A refusal cell has nothing to restore; its post-window step only proves +// that the source and destination bytes are still the offered originals. +func (r *editProbeRuntime) confirmRefusalUnchanged() error { + if r.phase != nil && r.phase.EditProbe != nil { + r.phase.EditProbe.RestoreMS = unavailable("ms", "refusal cell made no change") + } + source, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) + if err != nil { + return err + } + destination, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.p.Repos[0].Corpus.EditDestination))) + if err != nil { + return err + } + if string(source) != string(r.original) || string(destination) != string(r.moveOriginal) { + return errors.New("refused relocation changed source or destination bytes") + } + return nil +} diff --git a/bench/load/edit_probe_refusal_test.go b/bench/load/edit_probe_refusal_test.go new file mode 100644 index 000000000..02e341264 --- /dev/null +++ b/bench/load/edit_probe_refusal_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "strings" + "testing" +) + +func TestEditProbeMoveRefusalsDistinguishApplicabilityLanguageAndMissingEvidence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, tool := range []string{"move_symbol", "refactor.move"} { + t.Run(language+"/"+tool, func(t *testing.T) { + c := editProbeSymbolCorpus(t, language) + c.EditTool = tool + c.EditKind = "relocate" + _, err := makeEditProbePlan(c, 42, 1) + want := "unsupported language" + if language == "go" { + want = "move fixture requires" + } + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("missing explicit refusal: %v", err) + } + if language == "go" { + for _, kind := range []string{"append_leaf", "modify_body", "change_signature", "rename", "delete"} { + c.EditKind = kind + if _, err := makeEditProbePlan(c, 42, 1); err == nil || !strings.Contains(err.Error(), "applies to relocation") { + t.Fatalf("inapplicable move kind %s admitted: %v", kind, err) + } + } + } + }) + } + } +} diff --git a/bench/load/edit_probe_relocation_refusal_test.go b/bench/load/edit_probe_relocation_refusal_test.go new file mode 100644 index 000000000..4c5950a8b --- /dev/null +++ b/bench/load/edit_probe_relocation_refusal_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func editProbeRelocationRefusalCorpus(t *testing.T, language string) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, language) + ext := filepath.Ext(c.File) + c.EditTool, c.EditKind, c.EditExpect = "refactor.move", "relocate", "refusal" + c.EditDestination = "destination" + ext + c.EditDestinationContent = map[string]string{".ts": "export function DestinationAnchor(): number { return 7; }\n", ".py": "def DestinationAnchor():\n return 7\n"}[ext] + for i := range 100 { + c.EditInitialContent += strings.ReplaceAll(c.EditTemplate, "{{marker}}", fmt.Sprintf("GortexEditProbe%016xN%03dBefore", uint64(42), i)) + } + return c +} + +func TestEditProbeRelocationRefusalCellRequiresExactProducerRefusal(t *testing.T) { + for _, language := range []string{"typescript", "python"} { + c := editProbeRelocationRefusalCorpus(t, language) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(language, err) + } + for _, step := range plan.Steps { + if !step.ExpectedRefusal || !step.Relocated || step.Request.name() != "refactor.move" || editProbeOracles(c, step) != nil { + t.Fatalf("%s refusal step: %+v", language, step) + } + } + if len(editProbeOperations(plan, c)) != len(plan.Steps) { + t.Fatal("a refusal cell offered probes") + } + unmarked := c + unmarked.EditExpect = "" + if _, err := makeEditProbePlan(unmarked, 42, 100); err == nil || !strings.HasPrefix(err.Error(), "unsupported language") { + t.Fatalf("%s relocation without the explicit refusal cell: %v", language, err) + } + misspelled := c + misspelled.EditExpect = "refused" + if _, err := makeEditProbePlan(misspelled, 42, 100); err == nil { + t.Fatal("unknown edit_expect admitted") + } + other := editProbeSymbolCorpus(t, language) + other.EditExpect = "refusal" + if _, err := makeEditProbePlan(other, 42, 100); err == nil || !strings.Contains(err.Error(), "applies only to TypeScript/Python relocation") { + t.Fatalf("edit_expect admitted for %s: %v", other.EditTool, err) + } + } + golang := moveProbeCorpus(42) + golang.EditExpect = "refusal" + if _, err := makeEditProbePlan(golang, 42, 100); err == nil || !strings.Contains(err.Error(), "Go relocation is supported") { + t.Fatalf("Go relocation offered as a refusal: %v", err) + } +} + +func TestEditProbeRefusalCellKeepsBytesAndSkipsRestore(t *testing.T) { + for _, outcome := range []string{"exact", "other_path", "extra_content", "acknowledged", "bytes_changed"} { + t.Run(outcome, func(t *testing.T) { + c := editProbeRelocationRefusalCorpus(t, "typescript") + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + dir := t.TempDir() + for name, content := range map[string]string{c.File: c.EditInitialContent, c.EditDestination: c.EditDestinationContent} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + before := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:])) + refusal := fmt.Sprintf("unsupported language: move_symbol only supports Go files (got %q)", "fixture/"+c.File) + var mu sync.Mutex + var batches int + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + switch frame.Params["name"] { + case "refactor": + switch outcome { + case "other_path": + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": strings.Replace(refusal, "fixture/", "other/", 1)}}}, nil + case "extra_content": + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": refusal}, map[string]any{"type": "text", "text": "retry with move_file"}}}, nil + case "acknowledged": + return stubContent(map[string]any{"mutation_receipt": "moved"}), nil + case "bytes_changed": + if err := os.WriteFile(filepath.Join(dir, c.EditDestination), []byte(c.EditDestinationContent+before+"\n"), 0o600); err != nil { + t.Error(err) + } + } + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": refusal}}}, nil + case "edit": + mu.Lock() + batches++ + mu.Unlock() + return stubContent(map[string]any{}), nil + } + if args["operation"] == "file" { + return stubContent(map[string]any{"path": "fixture/" + c.File, "content": c.EditInitialContent}), nil + } + return stubContent(map[string]any{"id": "fixture/" + step.BeforeSymbol, "file_path": "fixture/" + c.File, "name": step.BeforeSymbol[strings.LastIndex(step.BeforeSymbol, "::")+2:], "source": before}), nil + }) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + editor, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer editor.Close() + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step, state.record = step, newEditProbeRecord(c, step) + state.record.Outcome, state.record.Errors = "not_started", nil // as prepared + phase := &Phase{Metrics: map[string]Metric{}, EditProbe: newEditProbeReport(plan, c, true)} + run := &editProbeRuntime{options: runOptions{Deadline: 2 * time.Second, Seed: 42}, p: &privateRun{Repos: []privateRepo{{Prefix: "fixture", Path: dir, Corpus: c}}}, phase: phase, plan: plan, editors: []*socketClient{editor}, states: []*editProbeState{state}, original: []byte(c.EditInitialContent), moveOriginal: []byte(c.EditDestinationContent), origin: time.Now(), requireAccounting: true} + run.activate(ctx, step) + run.doEdit(ctx, step) + record := state.record + if outcome == "exact" { + if record.Outcome != "complete" || len(record.Errors) != 0 || record.Accounting.State != "not_applicable_refused" || record.Receipt != "" || record.Ack.Value != nil || record.Readback.Value == nil || len(record.Readbacks) != 3 || record.Readbacks[0].Refusal != refusal { + t.Fatalf("exact refusal cell: %+v", record) + } + for name, lap := range record.PhasesMS { + if lap.Value != nil { + t.Fatal("refusal invented a phase", name) + } + } + } else if record.Outcome != "diagnostic_failure" || len(record.Errors) == 0 { + t.Fatalf("%s passed as a refusal: %+v", outcome, record) + } + // The post-window step proves the bytes, sends no restore batch and + // reports no restore time. + err = run.restore(ctx) + if (err != nil) != (outcome == "bytes_changed") || phase.EditProbe.RestoreMS.Value != nil || phase.EditProbe.RestoreMS.Reason != "refusal cell made no change" { + t.Fatalf("refusal restore: %v %+v", err, phase.EditProbe.RestoreMS) + } + mu.Lock() + defer mu.Unlock() + if batches != 0 { + t.Fatal("refusal cell sent an edit batch") + } + }) + } +} diff --git a/bench/load/edit_probe_rename_test.go b/bench/load/edit_probe_rename_test.go new file mode 100644 index 000000000..63d9f9777 --- /dev/null +++ b/bench/load/edit_probe_rename_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func editProbeRenameCorpus(t *testing.T, language string, count int) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, language) + c.EditKind = "rename" + for i := 0; i < count; i++ { + name := fmt.Sprintf("GortexEditProbe%016xN%03dBefore", uint64(42), i) + c.EditInitialContent += strings.ReplaceAll(c.EditTemplate, "{{marker}}", name) + } + return c +} + +func TestEditProbeRenameUsesPreexistingDeclarationAndExplicitAbsence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeRenameCorpus(t, language, 100) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + additive := c + additive.EditKind = "" + original, err := makeEditProbePlan(additive, 42, 100) + if err != nil { + t.Fatal(err) + } + for i, step := range plan.Steps { + if step.DueNS != original.Steps[i].DueNS || step.ActivateNS != original.Steps[i].ActivateNS { + t.Fatal("rename changed offered arrival schedule") + } + before := c.File + "::" + step.Marker + "Before" + if step.BeforeSymbol != before || step.Request.Arguments["target"].(map[string]any)["symbol"] != before { + t.Fatal("rename did not bind exact original declaration") + } + old, _ := step.Request.Arguments["match"].(string) + replacement, _ := step.Request.Arguments["replacement"].(string) + if strings.Count(c.EditInitialContent, old) != 1 || strings.Contains(replacement, step.Marker+"Before") || !editProbeFixtureSource(c.File, replacement, step.Marker, plan.Callee) { + t.Fatalf("invalid exact rename splice: %q -> %q", old, replacement) + } + oracles := editProbeOracles(c, step) + if len(oracles) != 8 || oracles[7].name != "old_source_absent" || oracles[7].call.ExpectedAbsentSymbol != before { + t.Fatal("positive graph/source probes replaced instead of explicit absence added") + } + } + }) + } +} + +func TestEditProbeRenameRejectsMissingAndAlreadyRenamedFixtures(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + base := editProbeRenameCorpus(t, language, 1) + for _, name := range []string{"missing_original", "duplicate_original", "already_new", "wrong_tool", "unsupported_kind"} { + t.Run(name, func(t *testing.T) { + c := base + old := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", "GortexEditProbe000000000000002aN000Before")) + switch name { + case "missing_original": + c.EditInitialContent = strings.Replace(c.EditInitialContent, old, "", 1) + case "duplicate_original": + c.EditInitialContent += "\n" + old + case "already_new": + c.EditInitialContent += strings.ReplaceAll(c.EditTemplate, "{{marker}}", "GortexEditProbe000000000000002aN000") + case "wrong_tool": + c.EditTool = "unsupported.tool" + case "unsupported_kind": + c.EditKind = "unsupported_future_kind" + } + if _, err := makeEditProbePlan(c, 42, 1); err == nil { + t.Fatal("unsupported/contradictory rename admitted") + } + }) + } + }) + } +} + +func TestEditProbeRenameDeleteUseActualFileWriteSymbolBatchSemantics(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, kind := range []string{"rename", "delete"} { + for _, tool := range []string{"edit.file", "edit.write", "edit.symbol", "edit.batch"} { + t.Run(language+"/"+kind+"/"+tool, func(t *testing.T) { + c := editProbeRenameCorpus(t, language, 100) + c.EditKind = kind + c.EditTool = tool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + offered := c.EditInitialContent + for _, step := range plan.Steps { + before := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker+"Before")) + replacement := strings.TrimSpace(strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker)) + if kind == "delete" { + replacement = "" + } + next := strings.Replace(offered, before, replacement, 1) + switch tool { + case "edit.file", "edit.symbol": + if step.Request.Arguments["match"] != before || step.Request.Arguments["replacement"] != replacement { + t.Fatal("splice semantics changed") + } + case "edit.write": + if step.Request.Arguments["content"] != next { + t.Fatal("whole-file offered prefix was regenerated/lost") + } + case "edit.batch": + changes := step.Request.Arguments["changes"].([]any) + change := changes[0].(map[string]any) + if len(changes) != 1 || change["op"] != "edit_file" || change["path"] != c.File || change["old_string"] != offered || change["new_string"] != next { + t.Fatal("batch did not retain exact whole-file offered before/after") + } + } + offered = next + } + }) + } + } + } +} + +func TestEditProbeRenameRejectsRetainedOldCallerBesideNewEdge(t *testing.T) { + c := editProbeRenameCorpus(t, "go", 1) + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + from := "owned/" + c.File + "::" + step.Marker + to := "owned/" + c.Symbol + old := "owned/" + step.BeforeSymbol + for _, variant := range []string{"valid", "old_node", "old_edge", "truncated", "missing_total"} { + t.Run(variant, func(t *testing.T) { + nodes := []any{map[string]any{"id": from}, map[string]any{"id": to}} + edges := []any{map[string]any{"from": from, "to": to, "kind": "calls"}} + body := map[string]any{"nodes": nodes, "edges": edges, "total_nodes": json.Number("2"), "total_edges": json.Number("1"), "truncated": false} + switch variant { + case "old_node": + body["nodes"] = append(nodes, map[string]any{"id": old}) + body["total_nodes"] = json.Number("3") + case "old_edge": + body["edges"] = append(edges, map[string]any{"from": old, "to": to, "kind": "calls"}) + body["total_edges"] = json.Number("2") + case "truncated": + body["truncated"] = true + case "missing_total": + delete(body, "total_nodes") + } + if got := editProbeCorrect("callers", body, c, step, plan.Callee); got != (variant == "valid") { + t.Fatalf("correct=%v", got) + } + }) + } +} diff --git a/bench/load/edit_probe_restart.go b/bench/load/edit_probe_restart.go new file mode 100644 index 000000000..1ae3b4e29 --- /dev/null +++ b/bench/load/edit_probe_restart.go @@ -0,0 +1,296 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "time" +) + +// The restart cell kills the owned daemon while the edit's ACK and its own +// receipt are pending and restarts it on the same store. Answers before the +// kill are kept apart; first-correct times come only from probes of the +// restarted daemon. +type editProbeRestartRecord struct { + PendingBeforeKill bool `json:"receipt_pending_before_kill"` + StatusBeforeKill json.RawMessage `json:"status_before_kill"` + KillNS *int64 `json:"kill_ns"` + ReadyNS *int64 `json:"ready_ns"` + Downtime Metric `json:"downtime_ms"` + StatusAfterRestart json.RawMessage `json:"status_after_restart"` + StatusAfterError string `json:"status_after_restart_error,omitempty"` + BeforeKill map[string]Metric `json:"first_correct_before_kill_ms"` + FromReady map[string]Metric `json:"first_correct_from_ready_ms"` + DiskHasMarker bool `json:"disk_has_marker"` + Qualification string `json:"qualification,omitempty"` +} + +type editProbeRestarter interface { + kill() error + start(ctx context.Context) error // returns once the restarted daemon is ready +} + +type editProbeDaemonRestarter struct{ p *privateRun } + +func (d editProbeDaemonRestarter) kill() error { + if d.p.process == nil { + return errors.New("private daemon is not running") + } + err := killOwnedProcess(d.p.process) + <-d.p.done + d.p.process = nil + return errors.Join(err, d.p.stdout.Close(), d.p.stderr.Close()) +} + +func (d editProbeDaemonRestarter) start(ctx context.Context) error { + begin := time.Now() + if err := d.p.start(); err != nil { + return err + } + if err := d.p.awaitSocket(ctx); err != nil { + return err + } + _, _, err := d.p.awaitReady(ctx, begin, d.p.Repos) + return err +} + +// The window between the edit's disk commit and its graph publication is +// short, so the cell polls its edit's status faster than the probe period. +const editProbeRestartPoll = 5 * time.Millisecond + +// doRestartEdit sends the cell's one edit and kills the owned daemon while +// that edit's ACK is still outstanding: once a status read by its mutation_id +// shows the bytes committed and the graph receipt not yet terminal. The ACK +// itself waits for the receipt, so after the ACK the receipt is rarely still +// pending. The harness stops waiting for that ACK just before the kill. +func (r *editProbeRuntime) doRestartEdit(ctx context.Context, step editProbeStep) { + s, callCtx, cancel, started := r.beginEdit(ctx, step) + r.markPending(s) + defer r.settlePending(s) + defer s.signalAck() + defer s.stopProbes() + defer cancel() + type reply struct { + ack json.RawMessage + identity wireCallIdentity + err error + } + editCtx, abandon := context.WithCancel(callCtx) + defer abandon() + replies := make(chan reply, 1) + go func() { + ack, identity, err := r.perform(editCtx, s, r.editorFor(step), step.Request, "edit", "workload", 0) + replies <- reply{ack, identity, err} + }() + restart := &editProbeRestartRecord{Downtime: unavailable("ms", "daemon not restarted"), BeforeKill: map[string]Metric{}, FromReady: map[string]Metric{}} + statusReq := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"mutation_id": step.MutationID}}} + readStatus := func(stream int) (json.RawMessage, map[string]any) { + raw, _, err := r.perform(callCtx, s, s.client, statusReq, "status", "probe", stream) + status, decodeErr := decodeEditProbeResult(raw) + if err != nil || decodeErr != nil { + return raw, nil + } + return raw, status + } + var edit reply + answered := false + for !answered && callCtx.Err() == nil { + raw, status := readStatus(200) + if status["disk_status"] == "committed" { + restart.StatusBeforeKill = raw + restart.PendingBeforeKill = status["graph_status_terminal"] == false + break + } + select { + case edit = <-replies: + answered = true + case <-time.After(editProbeRestartPoll): + case <-callCtx.Done(): + } + } + if !answered { + abandon() + edit = <-replies + } + killedOutstanding := !answered && edit.err != nil && callCtx.Err() == nil + if edit.err != nil && !killedOutstanding { + s.mu.Lock() + switch { + case ctx.Err() != nil: + s.record.Outcome = "phase_censored" + case callCtx.Err() != nil: + s.record.Outcome = "product_deadline" + default: + s.record.Outcome = "edit_error" + } + s.record.Errors = append(s.record.Errors, edit.err.Error()) + s.mu.Unlock() + return + } + receipt := "" + if edit.err == nil { + body, err := decodeEditProbeOutput(edit.ack, editProbeRequestOutput(step.Request), r.plan.Tool) + if err != nil { + s.mu.Lock() + s.record.Outcome = "invalid_ack" + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + return + } + receipt, _ = body["mutation_receipt"].(string) + s.mu.Lock() + s.record.Ack = measured(float64(edit.identity.Finished.Sub(started))/float64(time.Millisecond), "ms") + s.acknowledged = edit.identity.Finished + s.mu.Unlock() + // The ACK came back first, so its receipt may already be terminal: the + // status read just before the kill decides. + raw, status := readStatus(201) + restart.StatusBeforeKill = raw + restart.PendingBeforeKill = status["graph_status_terminal"] == false + } else { + status, _ := decodeEditProbeResult(restart.StatusBeforeKill) + receipt, _ = status["receipt"].(string) + s.mu.Lock() + s.record.Ack = unavailable("ms", "the daemon was killed while this edit's ACK was outstanding") + s.mu.Unlock() + } + s.mu.Lock() + s.record.Receipt = receipt + s.mu.Unlock() + r.finishRestart(ctx, callCtx, s, step, started, restart, statusReq) +} + +// finishRestart kills and restarts the daemon, reopens the sessions, then +// runs the immediate readbacks and resumed probes against the restarted +// daemon. +func (r *editProbeRuntime) finishRestart(ctx, callCtx context.Context, s *editProbeState, step editProbeStep, started time.Time, restart *editProbeRestartRecord, statusReq request) { + var failure error + s.mu.Lock() + if s.record.Receipt == "" { + failure = errors.New("restart cell observed no receipt for its committed edit") + } + s.mu.Unlock() + var ready time.Time + if failure == nil { + if !restart.PendingBeforeKill { + restart.Qualification = "receipt terminal before kill" + } + // No probe is sent to a dead socket; the loops resume once ready. + s.stopProbes() + s.mu.Lock() + for name, value := range s.record.FirstCorrect { + restart.BeforeKill[name] = value + s.record.FirstCorrect[name] = unavailable("ms", "not observed after restart") + } + s.mu.Unlock() + kill := time.Now() + failure = r.restarter.kill() + if failure == nil { + failure = r.restarter.start(callCtx) + } + ready = time.Now() + killNS := kill.Sub(r.origin).Nanoseconds() + restart.KillNS = &killNS + if failure == nil { + readyNS := ready.Sub(r.origin).Nanoseconds() + restart.ReadyNS = &readyNS + restart.Downtime = measured(float64(ready.Sub(kill))/float64(time.Millisecond), "ms") + failure = r.reopenSessions(callCtx, s) + } + if failure != nil { + failure = errors.Join(errors.New("daemon restart failed"), failure) + } + } + if failure == nil { + raw, _, err := r.perform(callCtx, s, s.client, statusReq, "status", "probe", 202) + restart.StatusAfterRestart = raw + if err != nil { + restart.StatusAfterError = err.Error() + } + probeCtx, cancel := context.WithCancel(ctx) + s.mu.Lock() + s.cancel = cancel + s.mu.Unlock() + r.probe(probeCtx, cancel, s) + end, correct := r.immediateReadbacks(callCtx, s, step, []string{"file", "source"}, nil) + if correct { + s.mu.Lock() + s.record.Readback = measured(float64(end.Sub(started))/float64(time.Millisecond), "ms") + s.mu.Unlock() + } + for callCtx.Err() == nil && !s.firstCorrectComplete() { + if waitContext(callCtx, editProbePeriod) != nil { + break + } + } + disk, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) + restart.DiskHasMarker = err == nil && editProbeFixtureSource(r.plan.File, string(disk), step.Marker, r.plan.Callee) + } + s.mu.Lock() + defer s.mu.Unlock() + readyOffset := float64(ready.Sub(started)) / float64(time.Millisecond) + for name, value := range s.record.FirstCorrect { + restart.FromReady[name] = unavailable("ms", "daemon not restarted") + if failure == nil { + restart.FromReady[name] = value + } + if failure == nil && value.Value != nil { + restart.FromReady[name] = measured(*value.Value-readyOffset, "ms") + } + } + s.record.Restart = restart + s.record.Accounting = editProbeAccounting{State: "not_applicable_restart"} + s.record.PhasesMS = editProbePhases(nil, "the daemon was killed while this receipt was pending") + s.record.Derived = unavailable("ms", "the daemon was killed while this receipt was pending") + if failure != nil { + s.record.Errors = append(s.record.Errors, failure.Error()) + } + if restart.Qualification != "" { + s.record.Errors = append(s.record.Errors, restart.Qualification) + } + if failure == nil && !restart.DiskHasMarker { + s.record.Errors = append(s.record.Errors, "committed marker missing from disk after restart") + } + if s.firstProbeWrite.IsZero() || s.editWrite.IsZero() || !s.firstProbeWrite.Before(s.editWrite) { + s.record.Errors = append(s.record.Errors, "no actual prober wire start before edit wire") + } + for name, value := range s.record.FirstCorrect { + if value.Value == nil { + s.record.Errors = append(s.record.Errors, "first-correct oracle unavailable: "+name) + } + } + if s.record.StaleClaimedFresh { + s.record.Errors = append(s.record.Errors, "stale answer explicitly claimed fresh after acknowledgement") + } + switch { + case ctx.Err() != nil: + s.record.Outcome = "phase_censored" + case callCtx.Err() != nil: + s.record.Outcome = "product_deadline" + case len(s.record.Errors) == 0: + s.record.Outcome = "complete" + default: + s.record.Outcome = "diagnostic_failure" + } +} + +// The killed daemon's sessions are dead. The edit's fixed editor and its +// prober reconnect to the restarted daemon; the runtime closes them. +func (r *editProbeRuntime) reopenSessions(ctx context.Context, s *editProbeState) error { + editor, err := openMCP(ctx, r.p.Socket, r.p.Repos[0].Path) + if err != nil { + return err + } + prober, err := openMCP(ctx, r.p.Socket, r.p.Repos[0].Path) + if err != nil { + return errors.Join(err, editor.Close()) + } + r.restarted = append(r.restarted, editor, prober) + r.editors = []*socketClient{editor} + s.mu.Lock() + s.client = prober + s.mu.Unlock() + return nil +} diff --git a/bench/load/edit_probe_restart_test.go b/bench/load/edit_probe_restart_test.go new file mode 100644 index 000000000..8d3d277cf --- /dev/null +++ b/bench/load/edit_probe_restart_test.go @@ -0,0 +1,229 @@ +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +type editProbeFakeRestarter struct { + mu sync.Mutex + calls []string + startErr error + onKill func() + onStart func() +} + +func (f *editProbeFakeRestarter) kill() error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "kill") + if f.onKill != nil { + f.onKill() + } + return nil +} + +func (f *editProbeFakeRestarter) start(context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "start") + if f.onStart != nil { + f.onStart() + } + return f.startErr +} + +// The edit splices the fixture on disk and, unless its receipt is already +// terminal, holds its ACK until the restart, as a daemon waiting on the +// receipt would. Status by mutation_id reads committed and pending (or +// terminal) before the kill and is unknown to the restarted daemon. Sessions +// opened before the kill (connections 1 and 2) must not be used after it. +func editProbeRestartStub(t *testing.T, c Corpus, dir string, terminal bool, restarted *atomic.Bool) func(stubFrame) (any, *rpcError) { + disk := editProbeDiskStub(t, c, dir) + var committed atomic.Bool + return func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + if frame.Method != "" && restarted.Load() && frame.Connection <= 2 { + t.Errorf("pre-kill session %d used after restart: %s", frame.Connection, frame.Method) + } + switch { + case frame.Method == "resources/read": + return map[string]any{"contents": []any{map[string]any{"type": "text", "text": `{"total_nodes":1}`}}}, nil + case frame.Params["name"] == "edit" && args["operation"] == "file": + path := filepath.Join(dir, c.File) + data, err := os.ReadFile(path) + if err == nil { + err = os.WriteFile(path, []byte(strings.Replace(string(data), args["match"].(string), args["replacement"].(string), 1)), 0o600) + } + if err != nil { + t.Error(err) + } + committed.Store(true) + for deadline := time.Now().Add(2 * time.Second); !terminal && !restarted.Load() && time.Now().Before(deadline); { + time.Sleep(time.Millisecond) + } + return stubContent(map[string]any{"mutation_receipt": "receipt-1", "graph_status": "fresh"}), nil + case frame.Params["name"] == "change": + options, _ := args["options"].(map[string]any) + id, _ := options["mutation_id"].(string) + if restarted.Load() || !committed.Load() || id == "" { + return map[string]any{"isError": true, "content": []any{map[string]any{"type": "text", "text": "no mutation recorded for mutation_id " + id}}}, nil + } + graph := "pending" + if terminal { + graph = "fresh" + } + return stubContent(map[string]any{"found": true, "receipt": "receipt-1", "mutation_id": id, "disk_status": "committed", "graph_status": graph, "graph_status_terminal": terminal}), nil + } + return disk(frame) + } +} + +func runEditProbeRestartCell(t *testing.T, terminal bool, startErr error) (*editProbeRuntime, *editProbeFakeRestarter, []*socketClient) { + t.Helper() + c := editProbeCorpus() + c.EditSender = editProbeRestart + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte("package sample\n\nfunc Example() int { return 1 }\n\n"+c.EditAnchor+"\n"), 0o600); err != nil { + t.Fatal(err) + } + var restarted atomic.Bool + run := newEditProbeStubRun(t, ctx, c, plan, dir, 1, editProbeRestartStub(t, c, dir, terminal, &restarted)) + // The kill takes every session opened before it down with the daemon. + before := []*socketClient{run.editors[0], run.states[0].client} + fake := &editProbeFakeRestarter{startErr: startErr, onStart: func() { restarted.Store(true) }, onKill: func() { + for _, client := range before { + _ = client.Close() + } + }} + run.restarter = fake + run.activate(ctx, plan.Steps[0]) + awaitEditProbeFirstProbe(t, run.states[0]) + run.doEdit(ctx, plan.Steps[0]) + t.Cleanup(func() { + for _, client := range run.restarted { + _ = client.Close() + } + }) + return run, fake, before +} + +func TestEditProbeRestartReopensSessionsAndNeverReusesPreKillAnswers(t *testing.T) { + run, fake, before := runEditProbeRestartCell(t, false, nil) + record := run.states[0].record + restart := record.Restart + if record.Outcome != "complete" || len(record.Errors) != 0 || restart == nil || !restart.PendingBeforeKill || restart.Qualification != "" || !reflect.DeepEqual(fake.calls, []string{"kill", "start"}) { + t.Fatalf("restart cell did not complete after a pending kill: %+v %+v %v", record, restart, fake.calls) + } + beforeStatus, err := decodeEditProbeResult(restart.StatusBeforeKill) + if err != nil || beforeStatus["disk_status"] != "committed" || beforeStatus["receipt"] != record.Receipt { + t.Fatalf("restart did not retain its committed receipt status: %s (%v)", restart.StatusBeforeKill, err) + } + if restart.KillNS == nil || restart.ReadyNS == nil || *restart.ReadyNS < *restart.KillNS || restart.Downtime.Value == nil || !strings.Contains(restart.StatusAfterError, "no mutation recorded") || len(restart.StatusAfterRestart) == 0 || !restart.DiskHasMarker { + t.Fatalf("restart evidence missing: %+v", restart) + } + // The daemon was killed while the edit's ACK was outstanding: no ACK time, + // the receipt comes from the committed status, and the abandoned call is + // censored rather than a workload error. + if record.Ack.Value != nil || record.Ack.Reason != "the daemon was killed while this edit's ACK was outstanding" || record.Receipt != "receipt-1" || record.Readback.Value == nil || record.Accounting.State != "not_applicable_restart" || record.Derived.Value != nil { + t.Fatalf("restart record timings: %+v", record) + } + var edits []string + for _, attempt := range run.phase.Attempts { + if attempt.Role == "workload" { + edits = append(edits, attempt.Outcome) + } + } + if !reflect.DeepEqual(edits, []string{"phase_censored"}) { + t.Fatalf("abandoned edit call reported as %v", edits) + } + for name, lap := range record.PhasesMS { + if lap.Value != nil { + t.Fatal("restart invented a phase for the interrupted receipt", name) + } + } + for name, value := range record.FirstCorrect { + fromReady := restart.FromReady[name] + if _, kept := restart.BeforeKill[name]; !kept || value.Value == nil || fromReady.Value == nil || *fromReady.Value < 0 { + t.Fatalf("%s first-correct is not a post-restart answer: %+v %+v", name, value, fromReady) + } + } + if run.editors[0] == before[0] || run.states[0].client == before[1] || len(run.restarted) != 2 { + t.Fatal("restart did not reopen the editor and prober sessions") + } + // The killed daemon's committed write is on disk, so restore proceeds. + run.phase.EditProbe.Records = []editProbeRecord{record} + if err := run.restore(t.Context()); err != nil || run.phase.EditProbe.RestoreMS.Value == nil { + t.Fatalf("restore after restart: %v", err) + } +} + +// After the workload the run-level receipt is taken on the reopened editor: +// the scenario's first session died with the killed daemon. +func TestEditProbeRestartClosingReceiptUsesTheReopenedEditor(t *testing.T) { + run, _, before := runEditProbeRestartCell(t, false, nil) + dead := &Phase{} + captureReceipt(t.Context(), run.p, before[0], dead) + if len(dead.Stats) != 0 || len(dead.Errors) != 1 || !strings.HasPrefix(dead.Errors[0], "stats receipt: ") { + t.Fatalf("the pre-kill editor still answered: %v", dead.Errors) + } + run.phase.editProbeRuntime = run + captureReceipt(t.Context(), run.p, editProbeReceiptClient(run.phase, before[:1]), run.phase) + if len(run.phase.Errors) != 0 || len(run.phase.Stats) != 1 || len(run.phase.Status) != 1 { + t.Fatalf("closing receipt after the restart: %v", run.phase.Errors) + } +} + +func TestEditProbeRestartRequiresObservedPendingReceipt(t *testing.T) { + run, fake, _ := runEditProbeRestartCell(t, true, nil) + record := run.states[0].record + if record.Outcome != "diagnostic_failure" || record.Restart.Qualification != "receipt terminal before kill" || !slices.Contains(record.Errors, "receipt terminal before kill") || len(fake.calls) != 2 { + t.Fatalf("terminal receipt passed as a restart: %+v", record) + } + + run, _, before := runEditProbeRestartCell(t, false, errors.New("socket never came back")) + record = run.states[0].record + if record.Outcome != "diagnostic_failure" || record.Restart.ReadyNS != nil || record.Restart.Downtime.Value != nil || run.editors[0] != before[0] || len(run.restarted) != 0 { + t.Fatalf("failed restart reported as measured: %+v", record.Restart) + } + for name, value := range record.Restart.FromReady { + if value.Value != nil || value.Reason != "daemon not restarted" { + t.Fatalf("%s invented a time from a restart that never became ready", name) + } + } + + c := editProbeCorpus() + c.EditSender = editProbeRestart + toon, write, rename := c, editProbeWriteCorpus(t, "go"), editProbeRenameCorpus(t, "go", 1) + toon.EditOutput = "toon" + write.EditSender = editProbeRestart + rename.EditTool, rename.EditSender = "edit.file", editProbeRestart + for name, build := range map[string]func() (editProbePlan, error){ + "hundred_edits": func() (editProbePlan, error) { return makeEditProbePlan(c, 42, 100) }, + "readers": func() (editProbePlan, error) { + return planEditProbe(c, runOptions{Seed: 42, EditCount: 1, Editors: 1, Agents: 1}) + }, + "toon": func() (editProbePlan, error) { return makeEditProbePlan(toon, 42, 1) }, + "write_tool": func() (editProbePlan, error) { return makeEditProbePlan(write, 42, 1) }, + "rename": func() (editProbePlan, error) { return makeEditProbePlan(rename, 42, 1) }, + } { + if _, err := build(); err == nil { + t.Errorf("%s: restart cell planned", name) + } + } +} diff --git a/bench/load/edit_probe_retry_test.go b/bench/load/edit_probe_retry_test.go new file mode 100644 index 000000000..7dffebdb7 --- /dev/null +++ b/bench/load/edit_probe_retry_test.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +type editProbeRetryStub struct{ calls int } + +func (s *editProbeRetryStub) call(context.Context, string, any) (json.RawMessage, error) { + s.calls++ + if s.calls == 1 { + // Actual cmd/gortex/daemon_mcp_repo_loading.go JSON-RPC error shape, + // not a synthetic top-level retryable property. + var reply wireReply + if err := json.Unmarshal([]byte(`{"id":1,"error":{"code":-32000,"message":"repository loading","data":{"error_code":"repo_loading","retryable":true}}}`), &reply); err != nil { + return nil, err + } + return nil, reply.Error + } + return json.RawMessage(`{"content":[{"type":"text","text":"{\"ok\":true}"}]}`), nil +} + +func TestEditProbeRefusalClassificationUsesActualWireFields(t *testing.T) { + for _, tc := range []struct { + name, wire string + retry bool + }{ + {"busy", `{"id":1,"error":{"code":-32002,"message":"busy","data":{"outcome":"server_busy","retryable":true,"delivery_unknown":false}}}`, true}, + {"loading", `{"id":1,"error":{"code":-32000,"message":"loading","data":{"error_code":"repo_loading","retryable":true}}}`, true}, + {"unknown_delivery", `{"id":1,"error":{"code":-32002,"message":"busy","data":{"retryable":true,"delivery_unknown":true}}}`, false}, + {"unknown_code", `{"id":1,"error":{"code":-32000,"message":"unknown","data":{"error_code":"unknown","retryable":true}}}`, false}, + {"wrong_level", `{"id":1,"error":{"code":-32000,"message":"loading","retryable":true}}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var reply wireReply + if err := json.Unmarshal([]byte(tc.wire), &reply); err != nil { + t.Fatal(err) + } + if reply.Error == nil || reply.Error.retryEditProbe() != tc.retry { + t.Fatalf("classification mismatch%+v", reply.Error) + } + }) + } + blocked := json.RawMessage(`{"isError":true,"content":[{"text":"{\"error_code\":\"tool_blocked_by_mode\",\"message\":\"switch mode\",\"retriable\":true}"}]}`) + err := checkToolResult(blocked) + remote, ok := err.(*rpcError) + if !ok || remote.retryEditProbe() { + t.Fatalf("action-required guidance became automatic retry%v", err) + } +} + +func TestEditProbeTypedRetryPolicyDoesNotChangeLegacyCalls(t *testing.T) { + r := request{Tool: "edit", Operation: "file"} + sleep := func(context.Context, time.Duration) error { return nil } + legacy := &editProbeRetryStub{} + _, rows, err := callTool(t.Context(), legacy, r, nil, "workload", 0, 0, 0, time.Second, sleep) + if err == nil || legacy.calls != 1 || len(rows) != 1 || rows[0].Outcome != "error" { + t.Fatalf("legacy retry behavior changed: %+v %v", rows, err) + } + scoped := &editProbeRetryStub{} + _, rows, err = callEditProbeTool(t.Context(), scoped, r, nil, "workload", 0, 0, 0, time.Second, sleep) + if err != nil || scoped.calls != 2 || len(rows) != 2 || rows[0].Outcome != "retryable" || rows[1].Outcome != "success" { + t.Fatalf("typed probe retry missing: %+v %v", rows, err) + } + metrics := summarize(rows, 1, "workload")["edit.file"] + if metrics.RetryableRefusals != 1 || metrics.BusyRefusals != 0 || metrics.Successes != 1 || len(metrics.Errors) != 0 { + t.Fatalf("retry refusal misclassified: %+v", metrics) + } +} diff --git a/bench/load/edit_probe_run.go b/bench/load/edit_probe_run.go new file mode 100644 index 000000000..c6f9596f8 --- /dev/null +++ b/bench/load/edit_probe_run.go @@ -0,0 +1,1095 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" +) + +const editProbeRawLimit = 8 << 20 + +var errEditProbeEvidenceIncomplete = errors.New("harness evidence incomplete") + +type editProbeReadback struct { + Kind string `json:"kind"` + Latency Metric `json:"latency_ms"` + Correct bool `json:"correct"` + Error string `json:"error"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` + Refusal string `json:"refusal,omitempty"` + ObservationOnly bool `json:"observation_only,omitempty"` +} +type editProbeRecord struct { + Ordinal int `json:"ordinal"` + Editor int `json:"editor,omitempty"` + Marker string `json:"marker"` + MutationID string `json:"mutation_id"` + PlannedDueNS int64 `json:"planned_due_ns"` + ActivatedNS *int64 `json:"activated_ns"` + StartedNS *int64 `json:"started_ns"` + StartLatenessNS *int64 `json:"start_lateness_ns"` + Outcome string `json:"outcome"` + Receipt string `json:"mutation_receipt"` + Batch *editProbeBatchReferences `json:"batch,omitempty"` + Move *editProbeMoveReferences `json:"move,omitempty"` + MoveAccounting *editProbeMoveAccounting `json:"move_accounting,omitempty"` + Restart *editProbeRestartRecord `json:"restart,omitempty"` + NativeWrite *editProbeNativeWrite `json:"native_write,omitempty"` + GraphTerminalBasis string `json:"graph_terminal_basis,omitempty"` + Ack Metric `json:"t_ack_ms"` + PhasesMS map[string]Metric `json:"phases_ms,omitempty"` + Readback Metric `json:"t_readback_ms,omitempty"` + FileGraph Metric `json:"t_file_graph_ms,omitempty"` + Relations Metric `json:"t_relations_ms,omitempty"` + Search Metric `json:"t_search_ms,omitempty"` + Derived Metric `json:"t_derived_ms,omitempty"` + Readbacks []editProbeReadback `json:"readbacks"` + FirstCorrect map[string]Metric `json:"first_correct_ms"` + Accounting editProbeAccounting `json:"accounting"` + PendingReads int `json:"reads_while_receipt_observed_pending"` + ExpectedRefusals int `json:"expected_refusals,omitempty"` + ProbeErrors int `json:"probe_errors"` + StaleClaimedFresh bool `json:"stale_claimed_fresh"` + PrecommitFreshMismatches int `json:"precommit_fresh_mismatches_unassessed"` + GraphTerminal bool `json:"graph_terminal"` + Evidence string `json:"raw_evidence"` + EvidenceWriteMS float64 `json:"evidence_write_ms"` + EvidenceEncodeMS float64 `json:"evidence_encode_ms"` + EvidenceBytes int `json:"evidence_bytes"` + EvidenceLimit int `json:"evidence_limit_bytes"` + Errors []string `json:"errors"` +} +type editProbeReport struct { + Role string `json:"role,omitempty"` + Plan editProbePlan `json:"plan"` + PlanHash string `json:"plan_sha256"` + AccountingRequired bool `json:"accounting_required"` + Records []editProbeRecord `json:"edits"` + Summary *editProbeSummary `json:"summary,omitempty"` + OutputModes editProbeModes `json:"output_modes,omitempty"` + Reads editProbeReads `json:"reads,omitempty"` + RestoreMS Metric `json:"restore_batch_ms"` + Restore json.RawMessage `json:"restore_batch"` + StatsBefore json.RawMessage `json:"stats_before"` + StatsAfter json.RawMessage `json:"stats_after"` + Limitations []string `json:"limitations"` +} + +type editProbeState struct { + mu sync.Mutex + step editProbeStep + record editProbeRecord + client *socketClient + sink io.WriteCloser + sinkMu sync.Mutex + rawBytes int + rawAccepted int + rawPending []byte + rawWake chan struct{} + rawClosing bool + rawDone chan struct{} + rawOnce sync.Once + writeNS, encodeNS int64 + sinkErr error + cancel context.CancelFunc + pairedContext context.Context // shared per-edit probes; cleared after joined completion + probes sync.WaitGroup + started, acknowledged, firstProbeWrite, editWrite time.Time + dependencyMet time.Time // a dependency-gated native write may proceed + pending bool // counted in the runtime's pending edits + absenceBefore map[string]bool + ackReady chan struct{} // closed once a dependent may check this edit: its own ACK is known, or it ended + ackOnce sync.Once +} + +type editProbeRuntime struct { + options runOptions + p *privateRun + phase *Phase + plan editProbePlan + editors []*socketClient // fixed per ordinal; editors[0] owns run-level controls + readers []*socketClient + pendingEdits atomic.Int64 // sent edits whose receipt is not yet observed terminal and whose loop has not ended + reads []editProbeRead + modeCounts map[string]editProbeModeCounts + states []*editProbeState + original []byte + moveOriginal []byte + origin time.Time + mu sync.Mutex // Phase.Attempts, reads and mode counts only; request lifetimes are never serialized + requireAccounting bool + finalized bool + restarter editProbeRestarter // restart cell only + restarted []*socketClient // sessions reopened after a restart; closed by close +} + +func newEditProbeRecord(c Corpus, step editProbeStep) editProbeRecord { + record := editProbeRecord{Ordinal: step.Ordinal, Editor: step.Editor, Marker: step.Marker, MutationID: step.MutationID, PlannedDueNS: step.DueNS, Outcome: "phase_censored", Ack: unavailable("ms", "edit not sent"), PhasesMS: editProbePhases(nil, "no receipt status observed"), Readback: unavailable("ms", "edit not acknowledged"), Derived: unavailable("ms", "no receipt status observed"), FirstCorrect: make(map[string]Metric), Accounting: editProbeAccounting{State: "not_observed"}, Errors: []string{"scenario never reached edit dispatch"}} + for _, oracle := range editProbeOracles(c, step) { + record.FirstCorrect[oracle.name] = unavailable("ms", "not observed") + } + record.groupTimes() + return record +} + +func (record *editProbeRecord) groupTimes() { + record.FileGraph = editProbeGroupTime(record.FirstCorrect, "file_graph") + record.Relations = editProbeGroupTime(record.FirstCorrect, "relations") + record.Search = editProbeGroupTime(record.FirstCorrect, "search") +} + +func newEditProbeReport(plan editProbePlan, c Corpus, required bool) *editProbeReport { + hash, _ := hashJSON(plan) + language := "Go" + if plan.Language == "typescript" { + language = "TypeScript" + } + if plan.Language == "python" { + language = "Python" + } + role := "baseline" + if required { + role = "candidate" + } + out := &editProbeReport{Role: role, Plan: plan, PlanHash: hash, AccountingRequired: required, RestoreMS: unavailable("ms", "post-window batch not performed"), Limitations: []string{fmt.Sprintf("one %s/%s/%s/%s cell only", language, plan.Kind, plan.Tool, strings.ToUpper(plan.Mode)), "per-window numeric checks do not establish 100-edit/reference acceptance", "missing/unsupported B stays in the complete offered denominator", "t_readback/file_graph/relations/search are client times from edit send; t_derived is daemon-clock time from receipt origin and is not additive with them"}} + if plan.Readers > 0 { + out.Limitations = append(out.Limitations, "pure reads are classed when issued (no-edit baseline before the edit origin, under edit while any sent edit's receipt is not yet observed terminal, otherwise between edits); an edit without a receipt to observe stays pending until its loop ends, and an edit sent while a read is in flight does not reclass it") + } + if plan.Mode == "toon" { + out.Limitations = append(out.Limitations, "TOON is evaluated from the requested text, never a structured JSON sibling; setup, timing-status and restore controls remain explicit JSON with exact numeric handling", "TOON traversal subgraphs omit JSON total_nodes/full untrimmed total_edges and signature metadata; summaries keep full fields, omitted traversal fields are not synthesized", "complete TOON subgraph checks require explicit nontruncation, exact declared node total and the emitted row census; an omitted full edge-count field is not reported as a measured value") + } + if len(plan.Steps) > 0 && plan.Steps[0].ExpectedRefusal { + out.Limitations = append(out.Limitations, "a relocation refusal cell passes only on the producer's exact scoped refusal with unchanged source and destination bytes and the original symbol present; it has no probes, receipt, phases or restore batch") + } + if plan.Mode == "cli" { + out.Limitations = append(out.Limitations, "CLI oracles run one owned `gortex call --format json` process per request; the edit, readbacks and receipt status stay MCP JSON", "CLI probe cadence is bounded by process start, daemon dial and exit rather than 25 ms, and a probe's start is its process start", "CLI stdout keeps only the first text payload, re-encoded through float64: numbers that cannot be exact and prose wrappers are decode failures, and a tool refusal is a nonzero exit counted as a probe error") + } + if plan.Sender != "" { + out.Limitations = append(out.Limitations, editProbeNativeLimitations(plan.Sender)...) + } + if plan.Mode == "compact" { + out.Limitations = append(out.Limitations, "compact text is requested only from read.summary, search.symbols, relations.callers and trace.call_chain; source, editing_context and text probes, readbacks, the edit ACK and timing controls stay JSON", "compact rows carry no node IDs, edge rows, signatures or freshness fields: they prove the marker's presence only, relation rows prove membership of the traversal answer with a nonzero edge footer but not the edge or its tier, and stale-claimed-fresh is unavailable rather than 0", "an empty compact answer is zero rows; a JSON answer to a compact request, such as not-found guidance, counts as a decode failure") + } + for _, step := range plan.Steps { + out.Records = append(out.Records, newEditProbeRecord(c, step)) + } + return out +} + +func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, editors []*socketClient, phase *Phase) (*editProbeRuntime, []*socketClient, error) { + plan, err := planEditProbe(p.Repos[0].Corpus, o) + if err != nil { + return nil, nil, err + } + if len(editors) != max(1, plan.Editors) || len(readers) != plan.Readers { + return nil, nil, fmt.Errorf("edit-and-probe plan has %d fixed editors and %d readers but %d editor and %d reader sessions", max(1, plan.Editors), plan.Readers, len(editors), len(readers)) + } + original, err := os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(plan.File))) + if err != nil { + return nil, nil, err + } + if err := editProbeWriteAdmission(p.Repos[0].Corpus, original); err != nil { + return nil, nil, err + } + if filepath.Ext(plan.File) == ".ts" { + valid, _ := editProbeTypeScriptFixture(editProbeCrossFilePlain(p.Repos[0].Corpus, string(original)), "", "") + if !valid { + return nil, nil, errors.New("TypeScript cell requires an explicit numeric-function fixture") + } + } + if filepath.Ext(plan.File) == ".py" { + valid, _ := editProbePythonFixture(editProbeCrossFilePlain(p.Repos[0].Corpus, string(original)), "", "") + if !valid { + return nil, nil, errors.New("python cell requires an explicit top-level single-return fixture") + } + } + if strings.Count(string(original), p.Repos[0].Corpus.EditAnchor) != 1 { + return nil, nil, errors.New("edit-probe anchor must occur exactly once in original fixture") + } + for _, step := range plan.Steps { + if step.BeforeSymbol == "" && strings.Contains(string(original), step.Marker) { + return nil, nil, errors.New("planned marker already exists before edit") + } + } + r := &editProbeRuntime{options: o, p: p, phase: phase, plan: plan, editors: editors, readers: readers, original: original, requireAccounting: !o.editProbeBaseline} + if plan.Tool == "refactor.move" { + r.moveOriginal, err = os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(p.Repos[0].Corpus.EditDestination))) + if err != nil { + return r, nil, err + } + if string(r.moveOriginal) != p.Repos[0].Corpus.EditDestinationContent { + return r, nil, errors.New("move destination bytes differ from offered fixture") + } + } + phase.EditProbe = newEditProbeReport(plan, p.Repos[0].Corpus, r.requireAccounting) + if plan.Sender == editProbeRestart { + r.restarter = editProbeDaemonRestarter{p: p} + } + if p.Repos[0].Corpus.EditKind == "add_crossfile_call" { + if err := r.admitCrossFileCallee(ctx); err != nil { + return r, nil, err + } + } + phase.editProbeRuntime = r + dir := filepath.Join(p.Root, "edit-probe") + if err := os.Mkdir(dir, 0o700); err != nil { + return r, nil, err + } + var clients []*socketClient + setup := time.Now() + for _, step := range plan.Steps { + record := newEditProbeRecord(p.Repos[0].Corpus, step) + record.Outcome = "not_started" + record.Errors = nil + state := &editProbeState{step: step, record: record} + r.states = append(r.states, state) + state.record.Evidence = filepath.Join(dir, fmt.Sprintf("edit-%03d.jsonl", step.Ordinal)) + } + for _, state := range r.states { + sink, err := os.OpenFile(state.record.Evidence, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return r, clients, err + } + state.sink = sink + state.startRaw() + state.client, err = openMCP(ctx, p.Socket, p.Repos[0].Path) + if err != nil { + return r, clients, err + } + clients = append(clients, state.client) + // A file created during the run proves its own presence before the + // paired delete is sent; it cannot be admitted here. + if state.step.BeforeSymbol != "" && state.step.File == "" { + req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": state.step.BeforeSymbol}, "options": map[string]any{"context_lines": 0}}} + raw, identity, readErr := state.client.callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, p.Repos[0])}) + if readErr == nil { + readErr = checkToolResult(raw) + } + proven := readErr == nil && editProbePresenceProof(raw, p.Repos[0].Corpus, state.step.BeforeSymbol) + if err := state.emit(map[string]any{"event": "before_presence_admission", "symbol": state.step.BeforeSymbol, "wire_id": identity.ID, "write_started": identity.WriteStarted, "finished": identity.Finished, "payload": raw, "proven": proven}); err != nil { + return r, clients, err + } + if !proven { + return r, clients, fmt.Errorf("original symbol presence admission failed: %s: %v", state.step.BeforeSymbol, readErr) + } + state.absenceBefore = map[string]bool{state.step.BeforeSymbol: true} + } + } + phase.Metrics["edit_probe_session_setup_seconds"] = measured(time.Since(setup).Seconds(), "s") + phase.Metrics["edit_probe_sessions"] = measured(float64(len(clients)), "sessions") + phase.Metrics["edit_probe_editor_sessions"] = measured(float64(len(editors)), "sessions") + if len(readers) > 0 { + phase.Metrics["edit_probe_reader_sessions"] = measured(float64(len(readers)), "sessions") + } + phase.SessionModel = plan.SessionModel + return r, clients, nil +} + +func (s *editProbeState) emit(value any) error { + started := time.Now() + data, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("%w: encode: %v", errEditProbeEvidenceIncomplete, err) + } + s.sinkMu.Lock() + defer func() { s.encodeNS += time.Since(started).Nanoseconds(); s.sinkMu.Unlock() }() + if s.sinkErr != nil { + return s.sinkErr + } + if s.rawClosing { + return fmt.Errorf("%w: raw writer already closing", errEditProbeEvidenceIncomplete) + } + if s.rawAccepted+len(data)+1 > editProbeRawLimit { + s.sinkErr = fmt.Errorf("%w: per-edit raw evidence limit exceeded", errEditProbeEvidenceIncomplete) + return s.sinkErr + } + // Pending and in-flight payloads share the unchanged total byte cap. A + // burst of small rows need not wait for the writer's next timeslice. + s.rawPending = append(s.rawPending, data...) + s.rawPending = append(s.rawPending, '\n') + s.rawAccepted += len(data) + 1 + select { + case s.rawWake <- struct{}{}: + default: // Wake already pending; these bytes are still owned above. + } + return nil +} + +// One owned writer per edit keeps persistence off response/probe timing paths. +// All pending/in-flight bytes share the per-edit cap; exhaustion fails evidence. +func (s *editProbeState) startRaw() { + s.rawWake, s.rawDone = make(chan struct{}, 1), make(chan struct{}) + go func() { + defer close(s.rawDone) + failed := false + for range s.rawWake { + for { + s.sinkMu.Lock() + data := s.rawPending + s.rawPending = nil + s.sinkMu.Unlock() + if len(data) == 0 { + break + } + if failed { + continue // Never credit bytes after a failed write. + } + start := time.Now() + n, err := s.sink.Write(data) + if err == nil && n != len(data) { + err = io.ErrShortWrite + } + s.sinkMu.Lock() + s.writeNS += time.Since(start).Nanoseconds() + s.rawBytes += n + if err != nil { + s.sinkErr = errors.Join(s.sinkErr, fmt.Errorf("%w: raw write: %v", errEditProbeEvidenceIncomplete, err)) + failed = true + } + s.sinkMu.Unlock() + } + } + }() +} + +func (s *editProbeState) finishRaw() { + if s.rawWake != nil { + s.rawOnce.Do(func() { + s.sinkMu.Lock() + s.rawClosing = true + close(s.rawWake) + s.sinkMu.Unlock() + }) + <-s.rawDone + } +} + +func editProbeOffset(at, origin time.Time) any { + if at.IsZero() { + return nil + } + return at.Sub(origin).Nanoseconds() +} + +type editProbeCaller struct { + runtime *editProbeRuntime + state *editProbeState + client *socketClient + purpose string + request request + last wireCallIdentity + role string +} + +func (c *editProbeCaller) responseReturnedAt() time.Time { return c.last.Finished } + +func (c *editProbeCaller) call(ctx context.Context, method string, params any) (json.RawMessage, error) { + var onWriteStart func(time.Time) + if c.purpose == "edit" { + onWriteStart = func(at time.Time) { + c.state.mu.Lock() + if c.state.editWrite.IsZero() { + c.state.editWrite = at + } + c.state.mu.Unlock() + } + } + data, identity, err := c.client.callWithWriteObserver(ctx, method, params, onWriteStart) + c.last = identity + c.state.mu.Lock() + if c.purpose == "edit" && c.state.editWrite.IsZero() && !identity.WriteStarted.IsZero() { + c.state.editWrite = identity.WriteStarted + } + if strings.HasPrefix(c.purpose, "oracle:") && !identity.WriteStarted.IsZero() && (c.state.firstProbeWrite.IsZero() || identity.WriteStarted.Before(c.state.firstProbeWrite)) { + c.state.firstProbeWrite = identity.WriteStarted + } + c.state.mu.Unlock() + if err == nil { + if expected := c.expectedAbsence(data, identity); expected != nil { + err = expected + } + } + row := map[string]any{"event": "rpc", "purpose": c.purpose, "tool": c.request.name(), "wire_id": identity.ID, "call_start_ns": editProbeOffset(identity.Started, c.runtime.origin), "write_start_ns": editProbeOffset(identity.WriteStarted, c.runtime.origin), "write_end_ns": editProbeOffset(identity.WriteFinished, c.runtime.origin), "reply_delivered_ns": editProbeOffset(identity.ReplyDelivered, c.runtime.origin), "call_end_ns": editProbeOffset(identity.Finished, c.runtime.origin), "output_mode": editProbeRequestOutput(c.request), "payload": data} + if err != nil { + row["error"] = err.Error() + if expected, ok := err.(*editProbeExpectedAbsence); ok { + row["expected_outcome"] = "symbol_absent" + row["expected_symbol"] = expected.symbol + } + } + if evidenceErr := c.state.emit(row); evidenceErr != nil { + return data, errors.Join(err, evidenceErr) + } + return data, err +} + +func editProbeRetrySleep(seed int64, ordinal, stream int) func(context.Context, time.Duration) error { + rng := rand.New(rand.NewPCG(uint64(seed)^uint64(ordinal+1)*0x9e3779b97f4a7c15, uint64(stream+1)^0x8ebc6af09c88c6e3)) + return func(ctx context.Context, delay time.Duration) error { + // Independent per logical call; backoff never consumes arrival RNG state. + jitter := time.Duration(rng.Int64N(int64(delay/2) + 1)) + return waitContext(ctx, delay+jitter) + } +} + +func (r *editProbeRuntime) perform(ctx context.Context, s *editProbeState, client *socketClient, req request, purpose, role string, stream int) (json.RawMessage, wireCallIdentity, error) { + data, identity, _, err := r.performObserved(ctx, s, client, req, purpose, role, stream) + return data, identity, err +} + +func (r *editProbeRuntime) performObserved(ctx context.Context, s *editProbeState, client *socketClient, req request, purpose, role string, stream int) (json.RawMessage, wireCallIdentity, []Attempt, error) { + budget := r.options.Deadline + if strings.HasPrefix(purpose, "oracle:") || purpose == "status" { + budget = min(budget, 20*time.Second) + } + if end, ok := ctx.Deadline(); ok { + budget = min(budget, time.Until(end)) + } + caller := &editProbeCaller{runtime: r, state: s, client: client, purpose: purpose, request: req, role: role} + data, attempts, err := callEditProbeTool(ctx, caller, req, dispatchArgs(req, r.p.Repos[0]), role, 0, s.step.Ordinal, stream, budget, editProbeRetrySleep(r.options.Seed, s.step.Ordinal, stream)) + r.mu.Lock() + r.phase.Attempts = append(r.phase.Attempts, attempts...) + r.mu.Unlock() + evidenceErr := s.emit(map[string]any{"event": "logical_call", "purpose": purpose, "tool": req.name(), "attempts": attempts}) + if evidenceErr != nil { + return data, caller.last, attempts, errors.Join(err, evidenceErr) + } + return data, caller.last, attempts, err +} + +func (s *editProbeState) noteProbeStart() bool { + s.mu.Lock() + defer s.mu.Unlock() + pending := !s.started.IsZero() && !s.record.GraphTerminal + if pending { + s.record.PendingReads++ + } + return pending +} + +func (r *editProbeRuntime) activateProbeState(step editProbeStep, cancel context.CancelFunc) *editProbeState { + s := r.states[step.Ordinal] + s.mu.Lock() + at := time.Since(r.origin).Nanoseconds() + s.record.ActivatedNS, s.cancel = &at, cancel + s.mu.Unlock() + return s +} + +func (r *editProbeRuntime) observeProbe(s *editProbeState, oracle editProbeOracle, sent time.Time, data json.RawMessage, identity wireCallIdentity, err error, pending bool) error { + ended := identity.Finished + if !identity.WriteStarted.IsZero() { + sent = identity.WriteStarted + } + body, decodeErr := decodeEditProbeOutput(data, r.probeOutput(oracle), oracle.call.name()) + absence, exactRefusal := err.(*editProbeExpectedAbsence) + expectedAbsence := oracle.call.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+oracle.call.ExpectedAbsentSymbol + correct := expectedAbsence || err == nil && decodeErr == nil && editProbeCorrect(oracle.name, body, r.p.Repos[0].Corpus, s.step, r.plan.Callee) + fresh := decodeErr == nil && editProbeClaimsFresh(body) + stale, precommit := false, false + s.mu.Lock() + if expectedAbsence { + s.record.ExpectedRefusals++ + } else if err != nil { + s.record.ProbeErrors++ + } + if correct { + switch { + case s.step.Deleted && s.step.After != nil && (s.dependencyMet.IsZero() || sent.Before(s.dependencyMet)): + // A native delete probes from its own activation but is sent only + // once its create completed; until then an absence answer says + // only that the created file is not visible yet. + case s.started.IsZero() || ended.IsZero() || ended.Before(s.started): + s.record.Errors = append(s.record.Errors, "marker visible before its own edit") + case s.record.FirstCorrect[oracle.name].Value == nil: + s.record.FirstCorrect[oracle.name] = measured(float64(ended.Sub(s.started))/float64(time.Millisecond), "ms") + } + } + if !correct && fresh { + if !s.acknowledged.IsZero() && !sent.Before(s.acknowledged) { + s.record.StaleClaimedFresh, stale = true, true + } else { + s.record.PrecommitFreshMismatches++ + precommit = true + } + } + s.mu.Unlock() + r.countMode(r.probeOutput(oracle), err == nil || expectedAbsence, err != nil || decodeErr == nil, fresh, stale, precommit) + return s.emit(map[string]any{"event": "probe", "oracle": oracle.name, "start_ns": editProbeOffset(sent, r.origin), "end_ns": editProbeOffset(ended, r.origin), "correct": correct, "claims_fresh": fresh, "receipt_observed_pending": pending}) +} + +// Activation only starts owned loops and returns. It never waits for a probe +// RPC/result; a blocked prober cannot shift this or any other edit's due time. +func (r *editProbeRuntime) activate(ctx context.Context, step editProbeStep) { + probeCtx, cancel := context.WithCancel(ctx) + r.probe(probeCtx, cancel, r.activateProbeState(step, cancel)) +} + +func (r *editProbeRuntime) probe(probeCtx context.Context, cancel context.CancelFunc, s *editProbeState) { + for index, oracle := range editProbeOracles(r.p.Repos[0].Corpus, s.step) { + s.probes.Go(func() { + ticker := time.NewTicker(editProbePeriod) + defer ticker.Stop() + for probeCtx.Err() == nil { + sent := time.Now() + pending := s.noteProbeStart() + var data json.RawMessage + var identity wireCallIdentity + var err error + if r.plan.Mode == "cli" { + data, identity, err = r.performCLI(probeCtx, s, oracle.call, "oracle:"+oracle.name, 10+index) + } else { + data, identity, err = r.perform(probeCtx, s, s.client, oracle.call, "oracle:"+oracle.name, "probe", 10+index) + } + if err := r.observeProbe(s, oracle, sent, data, identity, err, pending); err != nil { + cancel() + return + } + select { + case <-probeCtx.Done(): + return + case <-ticker.C: + } + } + }) + } +} + +func (s *editProbeState) stopProbes() { + s.mu.Lock() + cancel := s.cancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + s.probes.Wait() +} +func (r *editProbeRuntime) beginEdit(ctx context.Context, step editProbeStep) (*editProbeState, context.Context, context.CancelFunc, time.Time) { + s := r.states[step.Ordinal] + started := time.Now() + callCtx, cancel := context.WithTimeout(ctx, r.options.Deadline) + s.mu.Lock() + s.started = started + ns := started.Sub(r.origin).Nanoseconds() + late := ns - step.DueNS + s.record.StartedNS = &ns + s.record.StartLatenessNS = &late + s.record.Outcome = "in_flight" + if s.record.ActivatedNS == nil { + s.record.Errors = append(s.record.Errors, "missing pre-edit activation") + } + s.mu.Unlock() + return s, callCtx, cancel, started +} + +// A sent edit is pending for read classification until its receipt is first +// observed terminal, or until its loop ends without that (a failed edit, or a +// native write, which has no receipt). +func (r *editProbeRuntime) markPending(s *editProbeState) { + s.mu.Lock() + defer s.mu.Unlock() + if !s.pending { + s.pending = true + r.pendingEdits.Add(1) + } +} + +func (r *editProbeRuntime) settlePending(s *editProbeState) { + s.mu.Lock() + defer s.mu.Unlock() + r.settlePendingLocked(s) +} + +// The caller holds s.mu. +func (r *editProbeRuntime) settlePendingLocked(s *editProbeState) { + if s.pending { + s.pending = false + r.pendingEdits.Add(-1) + } +} + +// Each edit and its own immediate readbacks use the ordinal's fixed editor; +// probes and receipt status stay on that edit's separate prober session. +func (r *editProbeRuntime) editorFor(step editProbeStep) *socketClient { + return r.editors[step.Editor] +} + +// The run-level receipt after the workload goes through editor 0, which a +// restart cell reopened on the restarted daemon: the scenario's first session +// may belong to the killed one. +func editProbeReceiptClient(phase *Phase, clients []*socketClient) *socketClient { + if phase.editProbeRuntime != nil { + return phase.editProbeRuntime.editors[0] + } + return clients[0] +} + +func (r *editProbeRuntime) doEdit(ctx context.Context, step editProbeStep) { + if step.Sender != "" { + r.doNativeWrite(ctx, step) + return + } + if r.plan.Sender == editProbeRestart { + r.doRestartEdit(ctx, step) + return + } + s, callCtx, cancel, started := r.beginEdit(ctx, step) + r.markPending(s) + defer r.settlePending(s) + defer s.signalAck() + defer s.stopProbes() + defer cancel() + ack, identity, err := r.perform(callCtx, s, r.editorFor(step), step.Request, "edit", "workload", 0) + r.finishEditAfterACK(ctx, callCtx, s, step, started, ack, identity, err) +} + +// The paired dispatcher can start this owned continuation on this build's own +// ACK, without a barrier on its partner. Standalone retains the same order. +func (r *editProbeRuntime) finishEditAfterACK(ctx, callCtx context.Context, s *editProbeState, step editProbeStep, started time.Time, ack json.RawMessage, identity wireCallIdentity, err error) { + if step.ExpectedRefusal { + r.finishRefusal(ctx, callCtx, s, step, started, ack, identity, err) + return + } + if err != nil { + s.mu.Lock() + s.record.Outcome = "edit_error" + if ctx.Err() != nil { + s.record.Outcome = "phase_censored" + } + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + return + } + body, err := decodeEditProbeOutput(ack, editProbeRequestOutput(step.Request), r.plan.Tool) + if err != nil { + s.mu.Lock() + s.record.Outcome = "invalid_ack" + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + return + } + receipt, _ := body["mutation_receipt"].(string) + var move *editProbeMoveReferences + var batch *editProbeBatchReferences + var batchOffered []byte + if r.plan.Tool == "edit.batch" { + transaction, offered, requestErr := editProbeBatchIntent(step.Request) + if requestErr == nil { + refs, ackErr := decodeEditProbeBatchAcknowledgement(body, transaction, r.p.Repos[0].File, filepath.ToSlash(filepath.Join(r.p.Repos[0].Prefix, r.plan.File)), offered) + requestErr = ackErr + if ackErr == nil { + batch = &refs + batchOffered = offered + } + } + if requestErr != nil { + s.mu.Lock() + s.record.Outcome = "invalid_ack" + s.record.Errors = append(s.record.Errors, requestErr.Error()) + s.mu.Unlock() + return + } + } + if step.Relocated { + refs, moveErr := decodeEditProbeMoveAcknowledgement(body, r.p.Repos[0].Prefix+"/"+step.BeforeSymbol, r.p.Repos[0].Prefix+"/"+step.TargetSymbol, r.movePaths()) + if moveErr != nil { + s.mu.Lock() + s.record.Outcome = "invalid_ack" + s.record.Errors = append(s.record.Errors, moveErr.Error()) + s.mu.Unlock() + return + } + move = &refs + } + s.mu.Lock() + s.record.Receipt = receipt + if batch != nil { + s.record.Batch = batch + if batch.TransactionError != "" { + s.record.Errors = append(s.record.Errors, batch.TransactionError) + } + s.record.Receipt = "" + s.record.MutationID = "" + } + if move != nil { + s.record.Move = move + s.record.Receipt, s.record.MutationID = "", "" + s.record.Accounting = editProbeAccounting{State: "not_applicable_direct_move"} + s.record.PhasesMS = editProbePhases(nil, "relocation reports per-file move accounting, not one receipt's phases_ms") + s.record.Derived = unavailable("ms", "relocation reports per-file move accounting, not one receipt's timeline") + } + if receipt == "" && batch == nil && move == nil { + s.record.Errors = append(s.record.Errors, "ACK mutation receipt unavailable") + } else if !identity.Finished.IsZero() { + s.record.Ack = measured(float64(identity.Finished.Sub(started))/float64(time.Millisecond), "ms") + s.acknowledged = identity.Finished + } + s.mu.Unlock() + s.signalAck() + readbackKinds := []string{"file", "source"} + if step.BeforeSymbol != "" && !step.Deleted && step.TargetSymbol == "" { + readbackKinds = append(readbackKinds, "old_source_absent") + } + if step.Relocated { + readbackKinds = append(readbackKinds, "old_source_absent", "source_file_after_move") + } + readbackEnd, readbacksCorrect := r.immediateReadbacks(callCtx, s, step, readbackKinds, nil) + // The agent has its edit back once the last immediate readback is correct. + s.mu.Lock() + s.record.Readback = unavailable("ms", "immediate readback missing or incorrect") + if readbacksCorrect { + s.record.Readback = measured(float64(readbackEnd.Sub(started))/float64(time.Millisecond), "ms") + } + s.mu.Unlock() + if move != nil { + r.collectMoveEvidence(callCtx, s, *move, identity.ID) + } else if batch != nil { + r.collectBatchEvidence(callCtx, s, *batch, batchOffered) + } else { + options := map[string]any{"mutation_id": step.MutationID} + if receipt != "" { + options = map[string]any{"receipt": receipt} + } + statusReq := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": options}} + for callCtx.Err() == nil { + raw, _, statusErr := r.perform(callCtx, s, s.client, statusReq, "status", "probe", 200) + if errors.Is(statusErr, errEditProbeEvidenceIncomplete) { + break + } + if statusErr == nil { + if status, decodeErr := decodeEditProbeResult(raw); decodeErr == nil { + accounting := checkEditProbeAccountingForTool(status, receipt, r.plan.Tool) + s.mu.Lock() + s.record.Accounting = accounting + s.record.PhasesMS = editProbePhases(status, "status exposes no phases_ms") + s.record.Derived = editProbeDerived(status) + s.record.GraphTerminal = status["graph_status_terminal"] == true + if s.record.GraphTerminal { + r.settlePendingLocked(s) + } + all := true + for _, value := range s.record.FirstCorrect { + if value.Value == nil { + all = false + } + } + terminal := s.record.GraphTerminal + if terminal && status["graph_status"] != "fresh" { + s.record.Errors = append(s.record.Errors, "terminal graph status is not fresh") + } + s.mu.Unlock() + if terminal && status["graph_status"] != "fresh" { + break + } + if status["mutation_id"] != step.MutationID { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "status mutation identity mismatch") + s.mu.Unlock() + break + } + if editProbeFinished(terminal, all, r.requireAccounting, accounting) { + break + } + } + } + if waitContext(callCtx, editProbePeriod) != nil { + break + } + } + } + s.mu.Lock() + if s.firstProbeWrite.IsZero() || s.editWrite.IsZero() || !s.firstProbeWrite.Before(s.editWrite) { + s.record.Errors = append(s.record.Errors, "no actual prober wire start before edit wire") + } + for name, value := range s.record.FirstCorrect { + if value.Value == nil { + s.record.Errors = append(s.record.Errors, "first-correct oracle unavailable: "+name) + } + } + if !s.record.GraphTerminal && (move == nil || r.requireAccounting || !movePublicFactsComplete(s.record)) { + s.record.Errors = append(s.record.Errors, "graph receipt terminal unavailable") + } + if r.requireAccounting && move != nil && (s.record.MoveAccounting == nil || s.record.MoveAccounting.State != "observed_move_windows_checks_pass") { + s.record.Errors = append(s.record.Errors, "move shared A/per-file write/direct accounting unavailable") + } + if r.requireAccounting && move == nil && s.record.Accounting.State != "observed_single_window_checks_pass" { + s.record.Errors = append(s.record.Errors, "A/P/B accounting unavailable: "+strings.Join(s.record.Accounting.Reasons, "; ")) + } + if s.record.StaleClaimedFresh { + s.record.Errors = append(s.record.Errors, "stale answer explicitly claimed fresh after acknowledgement") + } + if ctx.Err() != nil { + s.record.Outcome = "phase_censored" + } else if callCtx.Err() != nil { + s.record.Outcome = "product_deadline" + } else if len(s.record.Errors) == 0 { + s.record.Outcome = "complete" + } else { + s.record.Outcome = "diagnostic_failure" + } + s.mu.Unlock() +} + +// Immediate same editing session, with no sleep/receipt/freshness barrier. +// Observation-only kinds are recorded but neither gate the readback time nor +// fail the record. +func (r *editProbeRuntime) immediateReadbacks(callCtx context.Context, s *editProbeState, step editProbeStep, readbackKinds []string, observeOnly map[string]bool) (time.Time, bool) { + readbackEnd, readbacksCorrect := time.Time{}, true + file := r.plan.File + if step.File != "" { + file = step.File + } + for index, kind := range readbackKinds { + target := map[string]any{"file": file} + if kind == "file" && step.Relocated { + target["file"] = step.DestinationFile + } + if kind == "source" { + target = map[string]any{"symbol": file + "::" + step.Marker} + if step.TargetSymbol != "" { + target = map[string]any{"symbol": step.TargetSymbol} + } + } + req := request{Tool: "read", Operation: kind, Arguments: map[string]any{"operation": kind, "target": target}} + if kind == "source" && step.TargetSymbol != "" { + req.Arguments["options"] = map[string]any{"context_lines": 0} + } + if kind == "old_source_absent" || kind == "source" && step.Deleted { + req.Operation = "source" + req.ExpectedAbsentSymbol = step.BeforeSymbol + req.Arguments = map[string]any{"operation": "source", "target": map[string]any{"symbol": step.BeforeSymbol}} + } + if kind == "source_file_after_move" { + req.Operation = "file" + req.Arguments["operation"] = "file" + } + req = editProbeWithOutput(req, r.plan.Mode) + begin := time.Now() + raw, identity, readErr := r.perform(callCtx, s, r.editorFor(step), req, "readback:"+kind, "readback", 100+index) + payload, decodeErr := decodeEditProbeOutput(raw, editProbeRequestOutput(req), req.name()) + back := editProbeReadback{Kind: kind, Latency: unavailable("ms", "response endpoint unavailable"), Correct: readErr == nil && decodeErr == nil && editProbeCorrect(kind, payload, r.p.Repos[0].Corpus, step, r.plan.Callee)} + if !identity.Finished.IsZero() { + back.Latency = measured(float64(identity.Finished.Sub(begin))/float64(time.Millisecond), "ms") + } + absence, exactRefusal := readErr.(*editProbeExpectedAbsence) + if req.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+req.ExpectedAbsentSymbol { + back.Correct = true + back.ExpectedOutcome = "symbol_absent" + back.Refusal = absence.message + } else { + if readErr != nil { + back.Error = readErr.Error() + } + if decodeErr != nil { + back.Error = decodeErr.Error() + } + } + back.ObservationOnly = observeOnly[kind] + s.mu.Lock() + s.record.Readbacks = append(s.record.Readbacks, back) + if !back.Correct && !back.ObservationOnly { + s.record.Errors = append(s.record.Errors, "immediate "+kind+" readback incorrect") + } + s.mu.Unlock() + if back.ObservationOnly { + continue + } + readbacksCorrect = readbacksCorrect && back.Correct && !identity.Finished.IsZero() + if identity.Finished.After(readbackEnd) { + readbackEnd = identity.Finished + } + } + return readbackEnd, readbacksCorrect +} + +func (r *editProbeRuntime) run(ctx context.Context) error { + budget, err := r.plan.budget(r.options.Deadline, 1) + if err != nil { + return err + } + captureReceipt(ctx, r.p, r.editors[0], r.phase) + before, statsErr := r.stats(ctx) + r.phase.EditProbe.StatsBefore = before + if statsErr != nil { + return statsErr + } + // Pure readers start with the phase; the offered edits keep their own + // schedule relative to an origin after the no-edit window. + r.phase.Started = time.Now() + r.origin = r.phase.Started.Add(time.Duration(r.plan.NoEditWindow)) + measuredCtx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + readCtx, stopReads := context.WithCancel(measuredCtx) + var readers sync.WaitGroup + r.startReaders(readCtx, &readers) + err = runEditProbeArrivals(measuredCtx, r.plan, r.origin, r.activate, r.doEdit, waitContext) + stopReads() + readers.Wait() + finished := r.finish() + after, statsErr := r.stats(measuredCtx) + r.phase.EditProbe.StatsAfter = after + return errors.Join(err, finished, statsErr) +} + +func (r *editProbeRuntime) finish() error { + var failures []error + if r.finalized { + return nil + } + r.finalized = true + r.phase.EditProbe.Records = nil + for _, s := range r.states { + s.stopProbes() + s.finishRaw() + s.mu.Lock() + s.pairedContext = nil + if s.record.Outcome == "not_started" { + s.record.Outcome = "phase_censored" + s.record.Errors = append(s.record.Errors, "planned edit never dispatched") + } + if s.record.Outcome == "complete" && len(s.record.Errors) != 0 { + // A coupled partner can lose evidence after this continuation's + // own result completed, but before all paired probes are joined. + s.record.Outcome = "diagnostic_failure" + } + record := s.record + s.mu.Unlock() + record.groupTimes() + s.sinkMu.Lock() + sinkErr := s.sinkErr + record.EvidenceWriteMS = float64(s.writeNS) / 1e6 + record.EvidenceEncodeMS = float64(s.encodeNS) / 1e6 + record.EvidenceBytes, record.EvidenceLimit = s.rawBytes, editProbeRawLimit + s.sinkMu.Unlock() + if sinkErr != nil { + record.Outcome = "harness_evidence_incomplete" + record.Errors = append(record.Errors, sinkErr.Error()) + } + r.phase.EditProbe.Records = append(r.phase.EditProbe.Records, record) + if record.Outcome != "complete" || len(record.Errors) > 0 { + failures = append(failures, fmt.Errorf("edit %d: %s (%s)", record.Ordinal, record.Outcome, strings.Join(record.Errors, "; "))) + } + } + for _, step := range r.plan.Steps[len(r.states):] { + record := newEditProbeRecord(r.p.Repos[0].Corpus, step) + r.phase.EditProbe.Records = append(r.phase.EditProbe.Records, record) + failures = append(failures, fmt.Errorf("edit %d: setup never completed", step.Ordinal)) + } + finishEditProbeMetrics(r.phase) + r.phase.EditProbe.Summary = summarizeEditProbe(r.phase.EditProbe.Records, r.plan.Seed) + r.mu.Lock() + r.phase.EditProbe.Reads = summarizeEditProbeReads(r.reads, r.plan.Seed) + r.phase.EditProbe.OutputModes = editProbeOutputCoverage(r.plan.Mode, r.modeCounts) + r.mu.Unlock() + return errors.Join(failures...) +} + +// The typed offered records retain failed/missing/censored ACKs. They must not +// be converted to the legacy scalar WriteMS zero or a never-visible success. +func finishEditProbeMetrics(phase *Phase) { + var ack, visible []float64 + phase.NeverSeen, phase.Censored = 0, 0 + for _, record := range phase.EditProbe.Records { + if record.Ack.Value != nil { + ack = append(ack, *record.Ack.Value) + } + if value := record.FirstCorrect["search_visible"].Value; value != nil { + visible = append(visible, *value) + } + if record.Outcome == "phase_censored" { + phase.Censored++ + } + if record.Outcome == "product_deadline" && record.Ack.Value != nil && record.FirstCorrect["search_visible"].Value == nil { + phase.NeverSeen++ + } + } + phase.EditLatency, phase.Visibility = distribution(ack, "ms"), distribution(visible, "ms") +} + +func (r *editProbeRuntime) close() error { + var failures []error + if !r.finalized { + failures = append(failures, r.finish()) + } + for _, s := range r.states { + s.stopProbes() + s.finishRaw() + if s.sink != nil { + failures = append(failures, s.sink.Close()) + } + } + for _, client := range r.restarted { + failures = append(failures, client.Close()) + } + return errors.Join(failures...) +} + +func (r *editProbeRuntime) restore(ctx context.Context) error { + if len(r.plan.Steps) > 0 && r.plan.Steps[0].ExpectedRefusal { + return r.confirmRefusalUnchanged() + } + if r.plan.Sender == editProbeCreateDelete { + return r.restoreCreatedFiles(ctx) + } + // Do not race unknown/uncompleted source effects with a cleanup rewrite. + for _, record := range r.phase.EditProbe.Records { + if record.NativeWrite != nil || record.Restart != nil && record.Restart.DiskHasMarker { + continue // A native write ended before its record did; a killed daemon's committed write is on disk. + } + if (record.Receipt == "" && record.Batch == nil && record.Move == nil) || (!record.GraphTerminal && (record.Move == nil || r.requireAccounting || !movePublicFactsComplete(record))) { + return errors.New("restore batch unavailable while mutation outcome is unknown/pending") + } + } + current, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) + if err != nil { + return err + } + changes := []any{map[string]any{"op": "edit_file", "path": r.p.Repos[0].Prefix + "/" + r.plan.File, "old_string": string(current), "new_string": string(r.original)}} + if r.plan.Tool == "refactor.move" { + destination := r.p.Repos[0].Corpus.EditDestination + currentDestination, readErr := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(destination))) + if readErr != nil { + return readErr + } + changes = append(changes, map[string]any{"op": "edit_file", "path": r.p.Repos[0].Prefix + "/" + destination, "old_string": string(currentDestination), "new_string": string(r.moveOriginal)}) + } + if err := r.restoreBatch(ctx, changes); err != nil { + return err + } + restored, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) + if err != nil { + return err + } + if string(restored) != string(r.original) { + return errors.New("timed public restore batch did not restore original bytes") + } + if r.plan.Tool == "refactor.move" { + destination, readErr := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.p.Repos[0].Corpus.EditDestination))) + if readErr != nil { + return readErr + } + if string(destination) != string(r.moveOriginal) { + return errors.New("timed public restore batch did not restore destination bytes") + } + } + return nil +} + +func (r *editProbeRuntime) stats(ctx context.Context) (json.RawMessage, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + data, err := r.editors[0].call(ctx, "resources/read", map[string]any{"uri": "gortex://stats"}) + if err != nil { + return data, err + } + return textPayload(data, "contents") +} diff --git a/bench/load/edit_probe_run_test.go b/bench/load/edit_probe_run_test.go new file mode 100644 index 000000000..4f07facde --- /dev/null +++ b/bench/load/edit_probe_run_test.go @@ -0,0 +1,357 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestEditProbeCensoredSetupKeepsCompleteOfferedDenominator(t *testing.T) { + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 100) + if err != nil { + t.Fatal(err) + } + report := newEditProbeReport(plan, corpus, true) + if len(report.Records) != 100 { + t.Fatal("startup failure lost planned edits") + } + for i, row := range report.Records { + if row.Ordinal != i || row.Outcome != "phase_censored" || row.Ack.Value != nil || len(row.Errors) == 0 { + t.Fatalf("fabricated startup record%+v", row) + } + } + root := t.TempDir() + file := filepath.Join(root, corpus.File) + if err := os.WriteFile(file, []byte("package p\nfunc Example() int { return 1 }\n"+corpus.EditAnchor+"\n"), 0o600); err != nil { + t.Fatal(err) + } + phase := &Phase{Metrics: make(map[string]Metric)} + p := &privateRun{Root: root, Socket: filepath.Join(root, "absent.sock"), Repos: []privateRepo{{Path: root, Corpus: corpus}}} + ctx, cancel := context.WithCancel(t.Context()) + cancel() + run, clients, err := prepareEditProbe(ctx, runOptions{Seed: 1, EditCount: 100, Editors: 1}, p, nil, []*socketClient{nil}, phase) + if err == nil || run == nil || len(clients) != 0 { + t.Fatalf("setup cancellation missingrun=%v clients=%d err=%v", run != nil, len(clients), err) + } + _ = run.close() // expected censored records are a diagnostic failure + if len(phase.EditProbe.Records) != 100 { + t.Fatal("partial session setup dropped offered edits") + } + for _, row := range phase.EditProbe.Records { + if row.Outcome != "phase_censored" || row.StartedNS != nil || row.Ack.Value != nil { + t.Fatalf("unstarted edit fabricated timing%+v", row) + } + } +} + +func TestEditProbeBlockedProbeResponsesCannotDelayEditDispatch(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + reached, edited, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var probeOnce, editOnce, releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Connection == 2 { + probeOnce.Do(func() { close(reached) }) + <-release + } + if frame.Params["name"] == "edit" { + editOnce.Do(func() { close(edited) }) + return stubContent(map[string]any{"mutation_receipt": "commit-test"}), nil + } + return stubContent(map[string]any{}), nil + }) + editor, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer editor.Close() + prober, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer prober.Close() + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 1) + if err != nil { + t.Fatal(err) + } + plan.Steps[0].ActivateNS = 0 + plan.Steps[0].DueNS = int64(25 * time.Millisecond) + file, err := os.CreateTemp(t.TempDir(), "probe-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer file.Close() + record := newEditProbeRecord(corpus, plan.Steps[0]) + record.Outcome = "not_started" + record.Errors = nil + state := &editProbeState{step: plan.Steps[0], record: record, client: prober, sink: file} + state.startRaw() + defer state.finishRaw() + phase := &Phase{Metrics: make(map[string]Metric), EditProbe: newEditProbeReport(plan, corpus, true)} + run := &editProbeRuntime{options: runOptions{Seed: 1, Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Prefix: "sample", Corpus: corpus}}}, phase: phase, plan: plan, editors: []*socketClient{editor}, states: []*editProbeState{state}, origin: time.Now(), requireAccounting: true} + done := make(chan error, 1) + go func() { done <- runEditProbeArrivals(ctx, plan, run.origin, run.activate, run.doEdit, waitContext) }() + select { + case <-reached: + case <-ctx.Done(): + t.Fatal("prober never dispatched") + } + select { + case <-edited: + case <-ctx.Done(): + t.Fatal("blocked probe response gated actual edit") + } + cancel() + unblock() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("owned edit/probers did not join") + } + state.stopProbes() + if state.record.StartedNS == nil { + t.Fatal("edit start not retained") + } +} + +type editProbeResponseClockStub struct { + finished time.Time + err error +} + +func (s *editProbeResponseClockStub) call(context.Context, string, any) (json.RawMessage, error) { + s.finished = time.Now() + if s.err != nil { + return nil, s.err + } + return json.Marshal(map[string]any{"content": []any{map[string]any{"type": "text", "text": "{}"}}}) +} +func (s *editProbeResponseClockStub) responseReturnedAt() time.Time { return s.finished } + +func TestEditProbeAttemptLatencyUsesResponseEndpoint(t *testing.T) { + client := &editProbeResponseClockStub{} + req := request{Tool: "read", Operation: "file"} + _, attempts, err := callEditProbeTool(t.Context(), client, req, nil, "probe", 0, 0, 0, time.Second, waitContext) + if err != nil || len(attempts) != 1 { + t.Fatalf("call: %v %+v", err, attempts) + } + want := float64(client.finished.Sub(attempts[0].Started)) / float64(time.Millisecond) + if attempts[0].DurationMS != want { + t.Fatalf("response latency includes observer work: got=%v want=%v", attempts[0].DurationMS, want) + } +} + +func TestEditProbeRawWriterDrainsBeforeEvidenceCompletion(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "raw-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer file.Close() + state := &editProbeState{sink: file} + state.startRaw() + for i := 0; i < 8; i++ { + if err := state.emit(map[string]any{"ordinal": i}); err != nil { + t.Fatal(err) + } + } + state.finishRaw() + data, err := os.ReadFile(file.Name()) + if err != nil || strings.Count(string(data), "\n") != 8 || state.rawBytes != len(data) || state.sinkErr != nil { + t.Fatalf("writer not drained: bytes=%d raw=%d err=%v sink=%v", len(data), state.rawBytes, err, state.sinkErr) + } +} + +func TestEditProbeEvidenceFailureIsNotProductFailure(t *testing.T) { + client := &editProbeResponseClockStub{err: errEditProbeEvidenceIncomplete} + req := request{Tool: "edit", Operation: "file"} + _, rows, err := callEditProbeTool(t.Context(), client, req, nil, "workload", 0, 0, 0, time.Second, waitContext) + if !errors.Is(err, errEditProbeEvidenceIncomplete) || len(rows) != 1 || rows[0].Outcome != "harness_incomplete" { + t.Fatalf("evidence classification: %+v %v", rows, err) + } + metrics := summarize(rows, 1, "workload")["edit.file"] + if metrics.HarnessIncomplete != 1 || metrics.Successes != 0 || metrics.ProductExpired != 0 || len(metrics.Errors) != 0 { + t.Fatalf("evidence failure charged to product: %+v", metrics) + } +} + +func TestEditProbeAggregatesDoNotTurnMissingACKOrCensoringIntoSuccess(t *testing.T) { + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 100) + if err != nil { + t.Fatal(err) + } + report := newEditProbeReport(plan, corpus, true) + report.Records[0].Outcome = "edit_error" + report.Records[1].Outcome = "invalid_ack" + report.Records[2].Outcome = "diagnostic_failure" // decoded response without receipt + report.Records[3].Outcome = "complete" + report.Records[3].Ack = measured(12, "ms") + report.Records[3].FirstCorrect["search_visible"] = measured(15, "ms") + report.Records[4].Outcome = "phase_censored" + report.Records[4].Ack = measured(20, "ms") // genuine ACK before phase cancellation + report.Records[5].Outcome = "product_deadline" + report.Records[5].Ack = measured(30, "ms") + phase := &Phase{EditProbe: report} + finishEditProbeMetrics(phase) + if len(report.Records) != 100 || phase.EditLatency.Count != 3 || phase.Visibility.Count != 1 || phase.NeverSeen != 1 || phase.Censored != 95 || len(phase.Edits) != 0 { + t.Fatalf("invalid aggregate denominator: ack=%d visible=%d never=%d censored=%d legacy=%d", phase.EditLatency.Count, phase.Visibility.Count, phase.NeverSeen, phase.Censored, len(phase.Edits)) + } +} + +func TestEditProbeRawCapIsExplicitFailure(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "cap-*.jsonl") + if err != nil { + t.Fatal(err) + } + defer file.Close() + state := &editProbeState{sink: file, rawAccepted: editProbeRawLimit} + state.startRaw() + defer state.finishRaw() + if err := state.emit(map[string]any{"required": "proof"}); !errors.Is(err, errEditProbeEvidenceIncomplete) || !strings.Contains(err.Error(), "evidence limit") { + t.Fatal("dropped evidence silently passed") + } + if err := state.emit(nil); !errors.Is(err, state.sinkErr) { + t.Fatal("evidence failure was cleared") + } +} + +func TestEditProbeEditsAndOwnReadbacksUseTheirFixedEditorConnection(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + const editors = 3 + var mu sync.Mutex + frames := map[int][]stubFrame{} + edited := make(chan int, 2*editors) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Method != "tools/call" { + return stubContent(map[string]any{}), nil + } + mu.Lock() + frames[frame.Connection] = append(frames[frame.Connection], frame) + mu.Unlock() + if frame.Params["name"] == "edit" { + edited <- frame.Connection + if frame.Connection == 1 { + // Editor 0's ACK stays blocked until both later editors sent. + select { + case <-release: + case <-ctx.Done(): + } + } + return stubContent(map[string]any{"mutation_receipt": "commit-test"}), nil + } + return stubContent(map[string]any{}), nil + }) + corpus := editProbeCorpus() + full, err := makeEditProbeTrafficPlan(corpus, 7, 100, editors) + if err != nil { + t.Fatal(err) + } + plan := full + plan.Steps = nil + for editor := range editors { + for _, step := range full.Steps { + if step.Editor == editor { + step.Ordinal = len(plan.Steps) + step.ActivateNS = int64(len(plan.Steps)) * int64(10*time.Millisecond) + step.DueNS = step.ActivateNS + int64(editProbePeriod) + plan.Steps = append(plan.Steps, step) + break + } + } + } + var sessions []*socketClient // editors on connections 1..3, probers on 4..6 + for range 2 * editors { + client, err := openMCP(ctx, socket, "") + if err != nil { + t.Fatal(err) + } + defer client.Close() + sessions = append(sessions, client) + } + phase := &Phase{Metrics: make(map[string]Metric), EditProbe: newEditProbeReport(plan, corpus, true)} + run := &editProbeRuntime{options: runOptions{Seed: 1, Deadline: time.Second}, p: &privateRun{Repos: []privateRepo{{Prefix: "sample", Corpus: corpus}}}, phase: phase, plan: plan, editors: sessions[:editors], origin: time.Now(), requireAccounting: true} + for i, step := range plan.Steps { + file, err := os.CreateTemp(t.TempDir(), "probe-*.jsonl") + if err != nil { + t.Fatal(err) + } + record := newEditProbeRecord(corpus, step) + record.Outcome, record.Errors = "not_started", nil + state := &editProbeState{step: step, record: record, client: sessions[editors+i], sink: file} + state.startRaw() + run.states = append(run.states, state) + } + done := make(chan error, 1) + go func() { done <- runEditProbeArrivals(ctx, plan, run.origin, run.activate, run.doEdit, waitContext) }() + seen := map[int]bool{} + for !seen[2] || !seen[3] { + select { + case connection := <-edited: + seen[connection] = true + case <-ctx.Done(): + t.Fatal("a blocked editor delayed another editor's offered arrival") + } + } + unblock() + if err := <-done; err != nil { + t.Fatal(err) + } + _ = run.finish() // stub receipts never turn terminal + mu.Lock() + defer mu.Unlock() + for i, step := range plan.Steps { + if record := run.phase.EditProbe.Records[i]; record.Editor != step.Editor || record.Receipt != "commit-test" || record.PhasesMS["derived"].Reason != "status exposes no phases_ms" || record.Readback.Value != nil || record.Derived.Reason != "status exposes no publication_phases" { + t.Fatalf("record lost its fixed editor, ACK or unavailable phases: %+v", record) + } + edits, reads := 0, 0 + for _, frame := range frames[step.Editor+1] { + args, _ := json.Marshal(frame.Params["arguments"]) + for _, other := range plan.Steps { + if other.Ordinal != step.Ordinal && strings.Contains(string(args), other.Marker) { + t.Fatalf("editor %d carried another ordinal's call: %s", step.Editor, args) + } + } + switch frame.Params["name"] { + case "edit": + edits++ + case "read": + reads++ + default: + t.Fatalf("editor session carried a probe/status call: %v", frame.Params["name"]) + } + } + if edits != 1 || reads < 2 { + t.Fatalf("editor %d: edits=%d immediate readbacks=%d", step.Editor, edits, reads) + } + for _, frame := range frames[editors+i+1] { + if frame.Params["name"] == "edit" { + t.Fatal("prober session carried an edit") + } + } + } + // Cleanup rewrites the shared file, so it waits for every editor's receipt. + for i := range run.phase.EditProbe.Records { + run.phase.EditProbe.Records[i].GraphTerminal = run.phase.EditProbe.Records[i].Editor != 1 + } + if err := run.restore(ctx); err == nil || !strings.Contains(err.Error(), "pending") { + t.Fatalf("restore raced another editor's pending receipt: %v", err) + } + if err := run.close(); err != nil { + t.Fatal(err) + } +} diff --git a/bench/load/edit_probe_signature.go b/bench/load/edit_probe_signature.go new file mode 100644 index 000000000..2457f0cb3 --- /dev/null +++ b/bench/load/edit_probe_signature.go @@ -0,0 +1,100 @@ +package main + +import ( + "path/filepath" + "strings" +) + +// Only this fixture's optional/default parameter is admitted. Existing +// zero-argument callers remain valid; unsupported signatures are not parsed by +// a new generic language framework. +func editProbeSignaturePlain(file, source string) string { + switch filepath.Ext(file) { + case ".go": + return strings.ReplaceAll(source, "(optional ...int)", "()") + case ".ts": + return strings.ReplaceAll(source, "(optional?: number)", "()") + case ".py": + return strings.ReplaceAll(source, "(optional=0)", "()") + } + return source +} +func editProbeBodyPlain(c Corpus, source string) string { + if c.EditKind == "change_signature" { + return editProbeSignaturePlain(c.File, source) + } + return source +} +func editProbeFollowingCaller(file, name, target string) string { + switch filepath.Ext(file) { + case ".go": + return "func " + name + "() int { return " + target + "() }" + case ".ts": + return "export function " + name + "(): number { return " + target + "(); }" + case ".py": + return "def " + name + "():\n return " + target + "()" + } + return "" +} +func editProbeBodySignature(rows []any, c Corpus, step editProbeStep, meta bool) bool { + name := step.TargetSymbol[strings.LastIndex(step.TargetSymbol, "::")+2:] + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if !editProbeSourceID(id, c.File, name) { + continue + } + if meta { + row, ok = row["meta"].(map[string]any) + if !ok { + return false + } + } + signature, _ := row["signature"].(string) + switch filepath.Ext(c.File) { + case ".go": + return strings.Contains(signature, "optional") + case ".ts": + return signature == "function "+name+"()" + case ".py": + return signature == "def "+name+"(...)" + } + return false + } + return false +} +func editProbeBodySpan(body map[string]any, c Corpus, step editProbeStep) bool { + rows, ok := body["nodes"].([]any) + if !ok { + return false + } + name := step.TargetSymbol[strings.LastIndex(step.TargetSymbol, "::")+2:] + for _, raw := range rows { + row, ok := raw.(map[string]any) + if !ok { + return false + } + id, _ := row["id"].(string) + if editProbeSourceID(id, c.File, name) { + start, se := strconvEditProbeUint(row["start_line"]) + end, ee := strconvEditProbeUint(row["end_line"]) + return se == nil && ee == nil && start > 0 && end >= start && end-start == uint64(step.ExpectedLineGap-2) + } + } + return false +} + +func editProbeSignatureHeaders(file, before, after, name string) bool { + switch filepath.Ext(file) { + case ".go": + return strings.HasPrefix(before, "func "+name+"()") && strings.HasPrefix(after, "func "+name+"(optional ...int)") + case ".ts": + return (strings.HasPrefix(before, "export function "+name+"()") && strings.HasPrefix(after, "export function "+name+"(optional?: number)")) || (strings.HasPrefix(before, "function "+name+"()") && strings.HasPrefix(after, "function "+name+"(optional?: number)")) + case ".py": + return strings.HasPrefix(before, "def "+name+"():") && strings.HasPrefix(after, "def "+name+"(optional=0):") + } + return false +} diff --git a/bench/load/edit_probe_signature_test.go b/bench/load/edit_probe_signature_test.go new file mode 100644 index 000000000..9a3619a6f --- /dev/null +++ b/bench/load/edit_probe_signature_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func editProbeSignatureCorpus(t *testing.T, language string, count int) Corpus { + t.Helper() + c := editProbeSymbolCorpus(t, language) + c.EditKind = "change_signature" + switch language { + case "go": + c.EditBeforeTemplate = "func {{marker}}() int { return Example() }" + c.EditTemplate = "func {{marker}}(optional ...int) int {\n return Example()\n}" + case "typescript": + c.EditBeforeTemplate = "export function {{marker}}(): number { return Example(); }" + c.EditTemplate = "export function {{marker}}(optional?: number): number {\n return Example();\n}" + case "python": + c.EditBeforeTemplate = "def {{marker}}():\n return Example()" + c.EditTemplate = "def {{marker}}(optional=0):\n\n return Example()" + } + for i := 0; i < count; i++ { + marker := fmt.Sprintf("GortexEditProbe%016xN%03d", uint64(42), i) + c.EditInitialContent += "\n\n" + strings.ReplaceAll(c.EditBeforeTemplate, "{{marker}}", marker+"Before") + "\n\n" + editProbeFollowingCaller(c.File, marker+"After", marker+"Before") + } + return c +} +func TestEditProbeSignaturePreservesActualZeroArgumentCaller(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, tool := range []string{"edit.file", "edit.symbol", "edit.write", "edit.batch"} { + t.Run(language+"/"+tool, func(t *testing.T) { + c := editProbeSignatureCorpus(t, language, 100) + c.EditTool = tool + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + target := step.Marker + "Before" + following := editProbeFollowingCaller(c.File, step.Marker+"After", target) + if strings.Count(c.EditInitialContent, following) != 1 || step.TargetSymbol != c.File+"::"+target { + t.Fatal("signature fixture lost actual unchanged caller") + } + before := strings.ReplaceAll(c.EditBeforeTemplate, "{{marker}}", target) + after := strings.ReplaceAll(c.EditTemplate, "{{marker}}", target) + if !editProbeSignatureHeaders(c.File, before, after, target) || !editProbeFixtureSource(c.File, editProbeSignaturePlain(c.File, after), target, plan.Callee) { + t.Fatal("signature fixture is not the supported optional parameter") + } + oracles := editProbeOracles(c, step) + if oracles[4].call.Arguments["query"] != "optional" { + t.Fatal("signature text probe could pass on old unchanged body") + } + } + }) + } + } +} +func TestEditProbeSignatureRequiresChangedSignatureAndOwnSpan(t *testing.T) { + c := editProbeSignatureCorpus(t, "go", 1) + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + target := map[string]any{"id": "owned/" + step.TargetSymbol, "start_line": json.Number("10"), "end_line": json.Number(fmt.Sprint(10 + step.ExpectedLineGap - 2)), "signature": "func target(optional ...int) int", "meta": map[string]any{"signature": "func target(optional ...int) int"}} + follower := map[string]any{"id": "owned/" + step.FollowingSymbol, "start_line": json.Number(fmt.Sprint(10 + step.ExpectedLineGap))} + for _, oracle := range []string{"summary", "editing_context", "search_visible"} { + key := map[string]string{"summary": "nodes", "editing_context": "defines", "search_visible": "results"}[oracle] + if !editProbeCorrect(oracle, map[string]any{key: []any{target, follower}}, c, step, plan.Callee) { + t.Fatal("valid changed signature/gap refused") + } + } + delete(target, "signature") + delete(target, "meta") + for _, oracle := range []string{"summary", "editing_context"} { + key := map[string]string{"summary": "nodes", "editing_context": "defines"}[oracle] + if editProbeCorrect(oracle, map[string]any{key: []any{target, follower}}, c, step, plan.Callee) { + t.Fatal("line movement borrowed as actual signature metadata") + } + } + edge := map[string]any{"from": "owned/" + step.TargetSymbol, "to": "owned/" + c.Symbol, "kind": "calls"} + body := map[string]any{"nodes": []any{target}, "edges": []any{edge}} + if !editProbeCorrect("call_chain", body, c, step, plan.Callee) { + t.Fatal("actual grown target range refused") + } + target["end_line"] = json.Number("10") + if editProbeCorrect("call_chain", body, c, step, plan.Callee) { + t.Fatal("preexisting call edge alone proved new signature") + } +} diff --git a/bench/load/edit_probe_spool_test.go b/bench/load/edit_probe_spool_test.go new file mode 100644 index 000000000..2b7e08f0d --- /dev/null +++ b/bench/load/edit_probe_spool_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "strings" + "sync" + "testing" + "time" +) + +type editProbeHeldSink struct { + started chan struct{} + release chan struct{} + once sync.Once + unblockOnce sync.Once + mu sync.Mutex + data bytes.Buffer + closed bool + writeErr error + short bool +} + +func newEditProbeHeldSink(t *testing.T) *editProbeHeldSink { + t.Helper() + sink := &editProbeHeldSink{started: make(chan struct{}), release: make(chan struct{})} + return sink +} + +func startHeldEditProbeSpool(t *testing.T, sink *editProbeHeldSink) *editProbeState { + t.Helper() + state := &editProbeState{sink: sink} + state.startRaw() + t.Cleanup(func() { + sink.unblock() + state.finishRaw() + _ = sink.Close() + }) + return state +} +func (s *editProbeHeldSink) unblock() { s.unblockOnce.Do(func() { close(s.release) }) } +func (s *editProbeHeldSink) Write(p []byte) (int, error) { + s.once.Do(func() { close(s.started) }) + <-s.release + s.mu.Lock() + defer s.mu.Unlock() + if s.writeErr != nil { + return 0, s.writeErr + } + if s.short { + n, _ := s.data.Write(p[:len(p)-1]) + return n, nil + } + return s.data.Write(p) +} +func (s *editProbeHeldSink) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} +func awaitEditProbeSpool(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("owned spool boundary did not complete") + } +} + +func TestEditProbeSpoolAcceptsMoreThan32RowsWithoutWriterBackpressure(t *testing.T) { + sink := newEditProbeHeldSink(t) + state := startHeldEditProbeSpool(t, sink) + if err := state.emit(map[string]int{"ordinal": -1}); err != nil { + t.Fatal(err) + } + awaitEditProbeSpool(t, sink.started) + emitted := make(chan struct{}) + var emitErr error + go func() { + defer close(emitted) + for i := 0; i < 1024; i++ { + if err := state.emit(map[string]int{"ordinal": i}); err != nil { + emitErr = err + return + } + } + }() + awaitEditProbeSpool(t, emitted) // The actual writer remains held throughout. + if emitErr != nil { + t.Fatal(emitErr) + } + sink.unblock() + state.finishRaw() + if state.sinkErr != nil || state.rawBytes != state.rawAccepted || state.rawBytes >= editProbeRawLimit { + t.Fatalf("lost accepted rows: bytes%d accepted%d err%v", state.rawBytes, state.rawAccepted, state.sinkErr) + } + decoder := json.NewDecoder(bytes.NewReader(sink.data.Bytes())) + for i := -1; i < 1024; i++ { + var row struct { + Ordinal int `json:"ordinal"` + } + if err := decoder.Decode(&row); err != nil || row.Ordinal != i { + t.Fatalf("row%d: %+v %v", i, row, err) + } + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + t.Fatal("extra or malformed retained row") + } + if err := state.emit(nil); !errors.Is(err, errEditProbeEvidenceIncomplete) { + t.Fatal("post-close emit accepted") + } +} + +func TestEditProbeSpoolByteCapCountsInFlightBytesAndStillDrainsAcceptedData(t *testing.T) { + sink := newEditProbeHeldSink(t) + state := startHeldEditProbeSpool(t, sink) + if err := state.emit(strings.Repeat("x", editProbeRawLimit-3)); err != nil { + t.Fatal(err) + } + awaitEditProbeSpool(t, sink.started) + if err := state.emit(nil); !errors.Is(err, errEditProbeEvidenceIncomplete) || !strings.Contains(err.Error(), "evidence limit") { + t.Fatal("in-flight bytes escaped total cap") + } + sink.unblock() + state.finishRaw() + if state.rawAccepted != editProbeRawLimit || state.rawBytes != editProbeRawLimit || !errors.Is(state.sinkErr, errEditProbeEvidenceIncomplete) { + t.Fatalf("cap/drain: accepted%d bytes%d err%v", state.rawAccepted, state.rawBytes, state.sinkErr) + } +} + +func TestEditProbeSpoolRetainsWriteAndShortWriteFailures(t *testing.T) { + for _, short := range []bool{false, true} { + t.Run(map[bool]string{false: "write_error", true: "short_write"}[short], func(t *testing.T) { + sink := newEditProbeHeldSink(t) + sink.short = short + if !short { + sink.writeErr = errors.New("owned sink failure") + } + state := startHeldEditProbeSpool(t, sink) + if err := state.emit(map[string]bool{"required": true}); err != nil { + t.Fatal(err) + } + awaitEditProbeSpool(t, sink.started) + sink.unblock() + state.finishRaw() + if !errors.Is(state.sinkErr, errEditProbeEvidenceIncomplete) || state.rawBytes >= state.rawAccepted { + t.Fatalf("failed bytes credited: %d/%d %v", state.rawBytes, state.rawAccepted, state.sinkErr) + } + if err := state.emit(nil); !errors.Is(err, errEditProbeEvidenceIncomplete) { + t.Fatal("write failure cleared") + } + }) + } +} + +func TestEditProbeSpoolCancellationJoinsProducerThenDrainsBeforeClose(t *testing.T) { + sink := newEditProbeHeldSink(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + state := startHeldEditProbeSpool(t, sink) + state.cancel = cancel + if err := state.emit(map[string]string{"event": "before_cancel"}); err != nil { + t.Fatal(err) + } + awaitEditProbeSpool(t, sink.started) + producerDone := make(chan struct{}) + var emitErr error + state.probes.Go(func() { + defer close(producerDone) + <-ctx.Done() + emitErr = state.emit(map[string]string{"event": "phase_censored"}) + }) + runtime := &editProbeRuntime{finalized: true, states: []*editProbeState{state}} + closed := make(chan struct{}) + var closeErr error + go func() { defer close(closed); closeErr = runtime.close() }() + awaitEditProbeSpool(t, producerDone) + if emitErr != nil { + t.Fatal(emitErr) + } + sink.unblock() + awaitEditProbeSpool(t, closed) + if closeErr != nil || !sink.closed || state.rawBytes != state.rawAccepted || !bytes.Contains(sink.data.Bytes(), []byte("phase_censored")) { + t.Fatalf("cancel/close lost final evidence: close%v bytes%d/%d", closeErr, state.rawBytes, state.rawAccepted) + } + state.finishRaw() // Repeated shutdown joins the same completed writer. +} diff --git a/bench/load/edit_probe_summary.go b/bench/load/edit_probe_summary.go new file mode 100644 index 000000000..ff35e188a --- /dev/null +++ b/bench/load/edit_probe_summary.go @@ -0,0 +1,225 @@ +package main + +import ( + "fmt" + "hash/fnv" + "maps" + "math" + "math/rand/v2" + "slices" + "strconv" +) + +// Every oracle reports into exactly one visibility group: the edited file's +// own graph, its call relations, or search. +var editProbeOracleGroups = map[string]string{ + "source": "file_graph", "summary": "file_graph", "editing_context": "file_graph", "old_source_absent": "file_graph", + "callers": "relations", "call_chain": "relations", + "search_visible": "search", "text": "search", +} + +// A group is visible when its slowest member first answered correctly; a +// member that never did leaves the group unavailable rather than faster. +func editProbeGroupTime(firstCorrect map[string]Metric, group string) Metric { + var latest *float64 + members := 0 + for _, name := range slices.Sorted(maps.Keys(firstCorrect)) { + if editProbeOracleGroups[name] != group { + continue + } + members++ + value := firstCorrect[name].Value + if value == nil { + return unavailable("ms", name+" never answered correctly") + } + if latest == nil || *value > *latest { + latest = value + } + } + if members == 0 { + return unavailable("ms", "cell has no "+group+" oracle") + } + return measured(*latest, "ms") +} + +// The derived step runs after every client-visible answer can already be +// correct, so no probe observes it. It is timed on the daemon's own clock, +// from the receipt origin to patch_derived_finished in this record's own +// publication timeline, and is not additive with client-side times. A +// coalesced receipt's derived step ran in the serving publication's timeline. +func editProbeDerived(status map[string]any) Metric { + if _, coalesced := status["coalesced_publication"]; coalesced { + return unavailable("ms", "coalesced receipt: derived ran in the serving publication's timeline") + } + timeline, ok := status["publication_phases"].(map[string]any) + if !ok { + return unavailable("ms", "status exposes no publication_phases") + } + if timeline["clock"] != "monotonic" || timeline["terminal"] != true { + return unavailable("ms", "publication timeline is not terminal on a monotonic clock") + } + phases, _ := timeline["phases"].([]any) + offsets := map[string]int64{} + for _, entry := range phases { + row, _ := entry.(map[string]any) + name, _ := row["phase"].(string) + if name != "received" && name != "patch_derived_finished" { + continue + } + ns, err := strconv.ParseInt(fmt.Sprint(row["offset_ns"]), 10, 64) + if _, repeated := offsets[name]; err != nil || repeated { + return unavailable("ms", "invalid or repeated "+name+" offset") + } + offsets[name] = ns + } + origin, received := offsets["received"] + finished, derived := offsets["patch_derived_finished"] + switch { + case !received || !derived: + return unavailable("ms", "publication timeline has no received/derived pair") + case finished < origin: + return unavailable("ms", "derived step precedes receipt origin") + } + return measured(float64(finished-origin)/1e6, "ms") +} + +const editProbeBootstrapResamples = 2000 + +type editProbeInterval struct { + Low Metric `json:"low"` + High Metric `json:"high"` +} + +// Nearest-rank quantiles (the same rule as Distribution), each with a seeded +// percentile-bootstrap 95% interval. A censored record ended (deadline, run +// end or failure) before its value was observed: it ranks above every +// observed value, so a quantile whose rank falls among censored records is +// unavailable rather than understated. A not-applicable record cannot carry +// the value. Neither is ever sampled as zero. +type editProbeQuantiles struct { + Count int `json:"count"` + Censored int `json:"censored"` + NotApplicable int `json:"not_applicable"` + P50 Metric `json:"p50"` + P95 Metric `json:"p95"` + P99 Metric `json:"p99"` + Max Metric `json:"max"` + P50CI editProbeInterval `json:"p50_ci95"` + P95CI editProbeInterval `json:"p95_ci95"` + P99CI editProbeInterval `json:"p99_ci95"` + Resamples int `json:"bootstrap_resamples"` +} + +type editProbeSummary struct { + Offered int `json:"offered"` + Outcomes map[string]int `json:"outcomes"` + Measures map[string]editProbeQuantiles `json:"measures"` +} + +func editProbeRank(q float64, n int) int { return max(0, int(math.Ceil(q*float64(n)))-1) } + +// The resampling stream depends only on the plan seed and the measure name, so +// a report is reproducible and one measure's interval cannot shift another's. +// Censored records join the ranking as +Inf. A quantile needs at least +// ceil(1/(1-q)) ranked records before its nearest rank falls below the sample +// maximum; with fewer its interval is unavailable. +func editProbeQuantilesOf(values []float64, censored, notApplicable int, unit string, seed int64, measure string) editProbeQuantiles { + ranked := append(slices.Clone(values), slices.Repeat([]float64{math.Inf(1)}, censored)...) + beyond := unavailable(unit, fmt.Sprintf("nearest rank falls among %d censored records", censored)) + bounded := func(m Metric) Metric { + if m.Value != nil && math.IsInf(*m.Value, 1) { + return beyond + } + return m + } + d := distribution(ranked, unit) + out := editProbeQuantiles{Count: len(values), Censored: censored, NotApplicable: notApplicable, P50: bounded(d.P50), P95: bounded(d.P95), P99: bounded(d.P99), Max: bounded(d.Max)} + quantiles := []float64{.50, .95, .99} + minimum := []int{2, 20, 100} // ceil(1/(1-q)) + intervals := []*editProbeInterval{&out.P50CI, &out.P95CI, &out.P99CI} + estimates := make([][]float64, len(quantiles)) + for i := range quantiles { + if len(ranked) < minimum[i] { + reason := fmt.Sprintf("bootstrap interval needs at least %d observations", minimum[i]) + *intervals[i] = editProbeInterval{Low: unavailable(unit, reason), High: unavailable(unit, reason)} + continue + } + estimates[i] = make([]float64, 0, editProbeBootstrapResamples) + } + if estimates[0] == nil { + return out + } + hash := fnv.New64a() + _, _ = hash.Write([]byte(measure)) + rng := rand.New(rand.NewPCG(uint64(seed), hash.Sum64())) + sample := make([]float64, len(ranked)) + for range editProbeBootstrapResamples { + for i := range sample { + sample[i] = ranked[rng.IntN(len(ranked))] + } + slices.Sort(sample) + for i, q := range quantiles { + if estimates[i] != nil { + estimates[i] = append(estimates[i], sample[editProbeRank(q, len(sample))]) + } + } + } + for i := range quantiles { + if estimates[i] == nil { + continue + } + slices.Sort(estimates[i]) + *intervals[i] = editProbeInterval{Low: bounded(measured(estimates[i][editProbeRank(.025, len(estimates[i]))], unit)), High: bounded(measured(estimates[i][editProbeRank(.975, len(estimates[i]))], unit))} + } + out.Resamples = editProbeBootstrapResamples + return out +} + +// One summary over the complete offered denominator. A completed record lacks +// only values that do not apply to it (a native write's ACK, a coalesced +// receipt's derived time); any other record ended before its missing values +// were observed, so they are censored. A measure a record does not carry at +// all is not applicable to it. +func summarizeEditProbe(records []editProbeRecord, seed int64) *editProbeSummary { + out := &editProbeSummary{Offered: len(records), Outcomes: map[string]int{}, Measures: map[string]editProbeQuantiles{}} + type measure struct { + observed []float64 + censored int + } + values := map[string]*measure{} + for _, record := range records { + out.Outcomes[record.Outcome]++ + add := func(name string, metric Metric) { + m := values[name] + if m == nil { + m = &measure{} + values[name] = m + } + switch { + case metric.Value != nil: + m.observed = append(m.observed, *metric.Value) + case record.Outcome != "complete": + m.censored++ + } + } + add("t_ack_ms", record.Ack) + add("t_readback_ms", record.Readback) + add("t_file_graph_ms", record.FileGraph) + add("t_relations_ms", record.Relations) + add("t_search_ms", record.Search) + add("t_derived_ms", record.Derived) + for _, readback := range record.Readbacks { + add("readback_ms."+readback.Kind, readback.Latency) + } + for name, metric := range record.PhasesMS { + add("phases_ms."+name, metric) + } + for name, metric := range record.FirstCorrect { + add("first_correct_ms."+name, metric) + } + } + for name, m := range values { + out.Measures[name] = editProbeQuantilesOf(m.observed, m.censored, len(records)-len(m.observed)-m.censored, "ms", seed, name) + } + return out +} diff --git a/bench/load/edit_probe_summary_test.go b/bench/load/edit_probe_summary_test.go new file mode 100644 index 000000000..3762ab614 --- /dev/null +++ b/bench/load/edit_probe_summary_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "reflect" + "testing" +) + +func TestEditProbeEveryOracleNameHasExactlyOneGroup(t *testing.T) { + c := editProbeCorpus() + marker := "GortexEditProbe0000000000000001N000" + steps := map[string]editProbeStep{ + "default": {Marker: marker}, + "rename": {Marker: marker, BeforeSymbol: c.File + "::" + marker + "Before"}, + "body": {Marker: marker, BeforeSymbol: c.File + "::" + marker + "Before", TargetSymbol: c.File + "::" + marker + "Before", FollowingSymbol: c.File + "::" + marker + "After"}, + "delete": {Marker: marker, BeforeSymbol: c.File + "::" + marker + "Before", Deleted: true}, + "move": {Marker: marker, BeforeSymbol: c.File + "::" + marker + "Before", TargetSymbol: "target.go::" + marker + "Before", Relocated: true, DestinationFile: "target.go"}, + } + for name, step := range steps { + groups := map[string]int{} + for _, oracle := range editProbeOracles(c, step) { + group, ok := editProbeOracleGroups[oracle.name] + if !ok { + t.Fatalf("%s: oracle %s has no visibility group", name, oracle.name) + } + groups[group]++ + } + if groups["file_graph"] == 0 || groups["relations"] == 0 || groups["search"] == 0 { + t.Fatalf("%s: a visibility group has no member: %v", name, groups) + } + } +} + +func TestEditProbeGroupTimesUseLatestMemberAndNeverInventZero(t *testing.T) { + first := map[string]Metric{ + "source": measured(10, "ms"), "summary": measured(30, "ms"), "editing_context": measured(20, "ms"), + "callers": unavailable("ms", "not observed"), "call_chain": measured(5, "ms"), + "search_visible": measured(7, "ms"), "text": measured(9, "ms"), + } + record := editProbeRecord{FirstCorrect: first} + record.groupTimes() + if record.FileGraph.Value == nil || *record.FileGraph.Value != 30 || record.Search.Value == nil || *record.Search.Value != 9 { + t.Fatalf("group time is not its slowest member: %+v %+v", record.FileGraph, record.Search) + } + if record.Relations.Value != nil || record.Relations.Reason != "callers never answered correctly" { + t.Fatalf("missing member made relations faster: %+v", record.Relations) + } + if got := editProbeGroupTime(map[string]Metric{"source": measured(1, "ms")}, "relations"); got.Value != nil || got.Reason != "cell has no relations oracle" { + t.Fatalf("empty group invented a time: %+v", got) + } + unsent := newEditProbeRecord(editProbeCorpus(), editProbeStep{Marker: "M"}) + for name, metric := range map[string]Metric{"readback": unsent.Readback, "file_graph": unsent.FileGraph, "relations": unsent.Relations, "search": unsent.Search, "derived": unsent.Derived} { + if metric.Value != nil || metric.Reason == "" || metric.Unit != "ms" { + t.Fatalf("unsent edit %s is not explicitly unavailable: %+v", name, metric) + } + } +} + +func TestEditProbeDerivedTimeRequiresOwnedTimeline(t *testing.T) { + timeline := func(terminal bool, phases ...any) map[string]any { + return map[string]any{"publication_phases": map[string]any{"clock": "monotonic", "terminal": terminal, "phases": phases}} + } + row := func(phase string, ns int64) any { return map[string]any{"phase": phase, "offset_ns": ns} } + owned := timeline(true, row("received", 0), row("receipt_committed", 4_000_000), row("patch_derived_finished", 12_500_000)) + if got := editProbeDerived(owned); got.Value == nil || *got.Value != 12.5 { + t.Fatalf("owned derived time: %+v", got) + } + coalesced := timeline(true, row("received", 0), row("patch_derived_finished", 1)) + coalesced["coalesced_publication"] = map[string]any{} + for name, status := range map[string]map[string]any{ + "no_timeline": {"graph_status": "fresh"}, + "pending": timeline(false, row("received", 0), row("patch_derived_finished", 1)), + "no_derived": timeline(true, row("received", 0)), + "repeated": timeline(true, row("received", 0), row("patch_derived_finished", 1), row("patch_derived_finished", 2)), + "reversed": timeline(true, row("received", 5), row("patch_derived_finished", 1)), + "coalesced": coalesced, + } { + if got := editProbeDerived(status); got.Value != nil || got.Reason == "" { + t.Fatalf("%s: derived time without an owned terminal timeline: %+v", name, got) + } + } +} + +func TestEditProbeQuantilesUseNearestRankWithSeededBootstrap(t *testing.T) { + values := make([]float64, 100) + for i := range values { + values[i] = float64((i*37)%100 + 1) // 1..100, shuffled + } + got := editProbeQuantilesOf(values, 0, 3, "ms", 42, "t_ack_ms") + want := distribution(values, "ms") + if got.Count != 100 || got.Censored != 0 || got.NotApplicable != 3 || *got.P50.Value != *want.P50.Value || *got.P95.Value != *want.P95.Value || *got.P99.Value != *want.P99.Value || *got.Max.Value != 100 { + t.Fatalf("not the nearest-rank distribution: %+v", got) + } + if got.Resamples != editProbeBootstrapResamples { + t.Fatalf("resamples %d", got.Resamples) + } + for name, pair := range map[string]struct { + estimate Metric + ci editProbeInterval + }{"p50": {got.P50, got.P50CI}, "p95": {got.P95, got.P95CI}, "p99": {got.P99, got.P99CI}} { + if pair.ci.Low.Value == nil || pair.ci.High.Value == nil || *pair.ci.Low.Value > *pair.estimate.Value || *pair.ci.High.Value < *pair.estimate.Value { + t.Fatalf("%s interval does not bracket its estimate: %+v %+v", name, pair.ci, pair.estimate) + } + } + if again := editProbeQuantilesOf(values, 0, 3, "ms", 42, "t_ack_ms"); !reflect.DeepEqual(again, got) { + t.Fatal("same seed and measure changed the interval") + } + if other := editProbeQuantilesOf(values, 0, 3, "ms", 43, "t_ack_ms"); reflect.DeepEqual(other.P50CI, got.P50CI) && reflect.DeepEqual(other.P95CI, got.P95CI) { + t.Fatal("seed does not drive the resampling stream") + } + few := editProbeQuantilesOf(values[:19], 0, 0, "ms", 42, "t_ack_ms") + if few.P50CI.Low.Value == nil || few.P95CI.Low.Value != nil || few.P95CI.High.Reason != "bootstrap interval needs at least 20 observations" || few.P99CI.Low.Value != nil { + t.Fatalf("too few observations for p95/p99 intervals: %+v", few) + } + none := editProbeQuantilesOf(nil, 0, 5, "ms", 42, "t_ack_ms") + if none.Count != 0 || none.NotApplicable != 5 || none.P50.Value != nil || none.P50CI.Low.Value != nil || none.Resamples != 0 { + t.Fatalf("no observations invented a value: %+v", none) + } +} + +// Records cut off before a value was observed rank above every observed +// value: dropping them would make the tail look faster than it was. +func TestEditProbeQuantilesRankCensoredRecordsAboveObservedValues(t *testing.T) { + observed := make([]float64, 95) + for i := range observed { + observed[i] = float64(i + 1) + } + got := editProbeQuantilesOf(observed, 5, 2, "ms", 42, "t_search_ms") + if got.Count != 95 || got.Censored != 5 || got.NotApplicable != 2 || *got.P50.Value != 50 || *got.P95.Value != 95 { + t.Fatalf("censored records shifted the observed ranks: %+v", got) + } + reason := "nearest rank falls among 5 censored records" + if got.P99.Value != nil || got.P99.Reason != reason || got.Max.Value != nil || got.Max.Reason != reason || got.P99CI.High.Value != nil { + t.Fatalf("a quantile among censored records was reported: %+v", got) + } + if dropped := distribution(observed, "ms"); *dropped.P99.Value != 95 { + t.Fatalf("observed-only p99 = %v", *dropped.P99.Value) + } + if all := editProbeQuantilesOf(nil, 3, 0, "ms", 42, "t_search_ms"); all.P50.Value != nil || all.P50.Reason != "nearest rank falls among 3 censored records" || all.P50CI.Low.Value != nil { + t.Fatalf("only censored records invented a value: %+v", all) + } +} + +func TestEditProbeSummaryCountsCensoredAndNotApplicableWithoutZero(t *testing.T) { + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 100) + if err != nil { + t.Fatal(err) + } + report := newEditProbeReport(plan, corpus, true) + // 95 complete records (five without a derived time, which then does not + // apply), two that hit the deadline after their ACK before callers was + // correct, and three never sent. + for i := range 97 { + record := &report.Records[i] + record.Outcome = "complete" + record.Ack = measured(float64(100+i), "ms") + if i >= 95 { + record.Outcome = "product_deadline" + continue + } + record.Readbacks = []editProbeReadback{{Kind: "file", Latency: measured(2, "ms")}} + record.FirstCorrect["callers"] = measured(float64(200+i), "ms") + if i < 90 { + record.Derived = measured(float64(i), "ms") + } + } + summary := summarizeEditProbe(report.Records, plan.Seed) + if summary.Offered != 100 || summary.Outcomes["complete"] != 95 || summary.Outcomes["product_deadline"] != 2 || summary.Outcomes["phase_censored"] != 3 { + t.Fatalf("outcome denominator: %+v", summary.Outcomes) + } + ack := summary.Measures["t_ack_ms"] + if ack.Count != 97 || ack.Censored != 3 || ack.NotApplicable != 0 || *ack.P50.Value != 149 || *ack.P95.Value != 194 || ack.P99.Value != nil { + t.Fatalf("unsent edits were dropped from the ACK ranking: %+v", ack) + } + callers := summary.Measures["first_correct_ms.callers"] + if callers.Count != 95 || callers.Censored != 5 || *callers.P95.Value != 294 || callers.P99.Value != nil { + t.Fatalf("deadline-censored callers made the tail faster: %+v", callers) + } + if derived := summary.Measures["t_derived_ms"]; derived.Count != 90 || derived.NotApplicable != 5 || derived.Censored != 5 { + t.Fatalf("derived denominator: %+v", derived) + } + if file := summary.Measures["readback_ms.file"]; file.Count != 95 || file.NotApplicable != 5 || file.Censored != 0 { + t.Fatalf("readback denominator: %+v", file) + } + for name, measure := range summary.Measures { + if measure.Count+measure.Censored+measure.NotApplicable != 100 { + t.Fatalf("%s does not cover the offered denominator: %+v", name, measure) + } + } +} diff --git a/bench/load/edit_probe_symbol.go b/bench/load/edit_probe_symbol.go new file mode 100644 index 000000000..c5e8a21b3 --- /dev/null +++ b/bench/load/edit_probe_symbol.go @@ -0,0 +1,51 @@ +package main + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "strings" +) + +// The first symbol cell replaces an actual stable declaration with itself and +// the new marker sibling. The product already supports this text splice; the +// fixture does not invent a symbol-append operation or reuse marker presence. +func editProbeSymbolRequest(c Corpus, marker string) request { + return request{Tool: "edit", Operation: "symbol", Arguments: map[string]any{"operation": "symbol", "target": map[string]any{"symbol": c.Symbol}, "match": c.EditAnchor, "replacement": c.EditAnchor + strings.ReplaceAll(c.EditTemplate, "{{marker}}", marker)}} +} +func editProbeSymbolAnchor(c Corpus) error { + parts := strings.Split(c.Symbol, "::") + if len(parts) != 2 || parts[0] != c.File || !token.IsIdentifier(parts[1]) { + return errors.New("edit.symbol fixture requires an exact same-file top-level declaration") + } + name := parts[1] + switch filepath.Ext(c.File) { + case ".go": + set := token.NewFileSet() + file, err := parser.ParseFile(set, c.File, c.EditInitialContent, 0) + if err != nil { + return err + } + for _, decl := range file.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Recv == nil && fn.Name.Name == name { + source := c.EditInitialContent[set.Position(fn.Pos()).Offset:set.Position(fn.End()).Offset] + if strings.TrimSpace(source) == strings.TrimSpace(c.EditAnchor) { + return nil + } + } + } + case ".ts": + valid, correct := editProbeTypeScriptFixture(c.EditAnchor, name, "") + if valid && correct && strings.Count(c.EditAnchor, "function ") == 1 { + return nil + } + case ".py": + valid, correct := editProbePythonFixture(c.EditAnchor, name, "") + if valid && correct && len(strings.Split(strings.TrimSpace(c.EditAnchor), "\n")) == 2 { + return nil + } + } + return errors.New("edit.symbol anchor is not the exact selected fixture declaration") +} diff --git a/bench/load/edit_probe_symbol_test.go b/bench/load/edit_probe_symbol_test.go new file mode 100644 index 000000000..f209118ab --- /dev/null +++ b/bench/load/edit_probe_symbol_test.go @@ -0,0 +1,106 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func editProbeSymbolCorpus(t *testing.T, language string) Corpus { + t.Helper() + c := editProbeWriteCorpus(t, language) + c.EditTool = "edit.symbol" + switch language { + case "go": + c.EditAnchor = "func Example() int { return 1 }" + case "typescript": + c.EditAnchor = "export function Example(): number { return 1; }" + case "python": + c.EditAnchor = "def Example():\n return 1" + } + return c +} +func TestEditProbeSymbolUsesActualDeclarationAndSameOfferedSchedule(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeSymbolCorpus(t, language) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + ordinary := c + ordinary.EditTool = "edit.file" + ordinary.EditInitialContent = "" + reference, err := makeEditProbePlan(ordinary, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Tool != "edit.symbol" || len(plan.Steps) != 100 { + t.Fatal("tool/denominator mismatch") + } + for i, step := range plan.Steps { + if step.DueNS != reference.Steps[i].DueNS || step.ActivateNS != reference.Steps[i].ActivateNS { + t.Fatal("changed offered schedule") + } + req := step.Request + if req.name() != "edit.symbol" || req.Arguments["target"].(map[string]any)["symbol"] != c.Symbol || req.Arguments["match"] != c.EditAnchor { + t.Fatal("wrong public scoped splice", req) + } + offered := req.Arguments["replacement"].(string) + if !strings.HasPrefix(offered, c.EditAnchor) || !editProbeFixtureSource(c.File, offered, step.Marker, plan.Callee) { + t.Fatal("replacement does not preserve selected declaration and add real call") + } + args := dispatchArgs(req, privateRepo{Prefix: "owned"}) + if args["target"].(map[string]any)["symbol"] != "owned/"+c.Symbol || req.Arguments["target"].(map[string]any)["symbol"] != c.Symbol { + t.Fatal("wrong source binding or mutated plan") + } + } + }) + } +} +func TestEditProbeSymbolRefusesForeignCommentOnlyAndPartialAnchors(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + c := editProbeSymbolCorpus(t, language) + for _, name := range []string{"wrong_symbol", "wrong_file", "comment_anchor", "partial_body", "missing_initial"} { + t.Run(language+"/"+name, func(t *testing.T) { + invalid := c + switch name { + case "wrong_symbol": + invalid.Symbol = strings.Replace(c.Symbol, "Example", "Foreign", 1) + case "wrong_file": + invalid.Symbol = "wrong/" + c.Symbol + case "comment_anchor": + invalid.EditAnchor = "// pretend declaration" + invalid.EditInitialContent += "\n" + invalid.EditAnchor + case "partial_body": + invalid.EditAnchor = "return 1" + case "missing_initial": + invalid.EditInitialContent = "" + } + if _, err := makeEditProbePlan(invalid, 42, 1); err == nil { + t.Fatal("accepted unbound symbol fixture", name) + } + }) + } + } +} + +func TestEditSymbolAccountingRequiresOfferedToolAndExplicitParseApplicability(t *testing.T) { + status := editProbeAccountingFixture() + status["tool"] = "edit_symbol" + ack := status["acknowledgement"].(map[string]any) + ack["parse_scope"] = "not_applicable_text_splice" + ack["phase_states"].(map[string]any)["parse"] = "not_applicable" + ack["observed_execution_ms"].(map[string]any)["parse"] = nil + ack["caller_response_ready_ms"] = json.Number("7") + if got := checkEditProbeAccountingForTool(status, "receipt-a", "edit.symbol"); got.State != "observed_single_window_checks_pass" { + t.Fatal("matching actual symbol splice refused", got) + } + if got := checkEditProbeAccountingForTool(status, "receipt-a", "edit.file"); got.State == "observed_single_window_checks_pass" { + t.Fatal("borrowed symbol applicability for a file request") + } + status["tool"] = "edit_file" + if got := checkEditProbeAccountingForTool(status, "receipt-a", "edit.symbol"); got.State == "observed_single_window_checks_pass" { + t.Fatal("borrowed foreign producer tool") + } +} diff --git a/bench/load/edit_probe_typescript.go b/bench/load/edit_probe_typescript.go new file mode 100644 index 000000000..40e9511ce --- /dev/null +++ b/bench/load/edit_probe_typescript.go @@ -0,0 +1,88 @@ +package main + +import ( + "path/filepath" + "strings" + "text/scanner" +) + +// This cell uses explicit numeric, zero-argument function fixtures. It is not +// an arbitrary TypeScript parser: comments/strings cannot supply a declaration +// or call, and unsupported syntax fails coverage instead of becoming proof. +func editProbeTypeScriptFixture(source, marker, callee string) (valid, correct bool) { + var scan scanner.Scanner + scan.Init(strings.NewReader(source)) + invalid := false + scan.Error = func(*scanner.Scanner, string) { invalid = true } + kind, text := scan.Scan(), scan.TokenText() + advance := func() { kind = scan.Scan(); text = scan.TokenText() } + take := func(want string) bool { + if text != want { + return false + } + advance() + return true + } + functions, matches := 0, 0 + for kind != scanner.EOF { + if text == "export" { + advance() + } + if !take("function") || kind != scanner.Ident { + return false, false + } + name := text + advance() + for _, token := range []string{"(", ")", ":", "number", "{", "return"} { + if !take(token) { + return false, false + } + } + called := "" + switch kind { + case scanner.Int: + for _, digit := range text { + if digit < '0' || digit > '9' { + return false, false + } + } + advance() + case scanner.Ident: + called = text + advance() + if !take("(") || !take(")") { + return false, false + } + default: + return false, false + } + if !take(";") || !take("}") { + return false, false + } + functions++ + if name == marker { + matches++ + correct = called == callee + } + } + return !invalid && functions > 0, !invalid && matches == 1 && correct +} + +func editProbeFixtureSource(file, source, marker, callee string) bool { + switch filepath.Ext(file) { + case ".go": + return editProbeSourceCall(source, marker, callee) + case ".ts": + valid, correct := editProbeTypeScriptFixture(source, marker, callee) + return valid && correct + case ".py": + valid, correct := editProbePythonFixture(source, marker, callee) + return valid && correct + default: + return false + } +} + +func editProbeFixturePath(path, file string) bool { + return path == file || strings.HasSuffix(path, "/"+file) +} diff --git a/bench/load/edit_probe_typescript_test.go b/bench/load/edit_probe_typescript_test.go new file mode 100644 index 000000000..1e67e2efc --- /dev/null +++ b/bench/load/edit_probe_typescript_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" +) + +func editProbeTypeScriptCorpus() Corpus { + c := sampleCorpus() + c.File, c.Symbol = "main.ts", "main.ts::Example" + c.EditAnchor = "export function Caller(): number { return Example(); }" + c.EditTemplate = "\nexport function {{marker}}(): number { return Example(); }\n" + return c +} + +func TestEditProbeTypeScriptExplicitCellAndFixture(t *testing.T) { + c := editProbeTypeScriptCorpus() + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Language != "typescript" || plan.Kind != "add_intrafile_call" || plan.Tool != "edit.file" || plan.Mode != "json" || len(plan.Steps) != 100 { + t.Fatalf("wrong cell: %+v", plan) + } + goPlan, err := makeEditProbePlan(editProbeCorpus(), 42, 100) + if err != nil { + t.Fatal(err) + } + for i := range plan.Steps { + if plan.Steps[i].DueNS != goPlan.Steps[i].DueNS || plan.Steps[i].ActivateNS != goPlan.Steps[i].ActivateNS { + t.Fatal("language changed offered traffic") + } + } + fixture, err := os.ReadFile("testdata/edit-probe/typescript/main.ts") + if err != nil { + t.Fatal(err) + } + valid, _ := editProbeTypeScriptFixture(string(fixture), "", "") + if !valid || strings.Count(string(fixture), "export function ") != 10 || strings.Count(string(fixture), c.EditAnchor) != 1 { + t.Fatal("fixture does not meet its explicit syntax/initial symbol gate") + } + step := plan.Steps[0] + source := strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + for oracle, body := range map[string]map[string]any{ + "source": {"id": "sample/main.ts::" + step.Marker, "name": step.Marker, "file_path": "sample/main.ts", "source": source}, + "file": {"path": "sample/main.ts", "content": string(fixture) + source}, + "text": {"matches": []any{map[string]any{"path": "sample/main.ts", "line": json.Number("11"), "text": strings.TrimSpace(source)}}}, + } { + if !editProbeCorrect(oracle, body, c, step, plan.Callee) { + t.Fatalf("actual %s fixture rejected", oracle) + } + } + for _, oracle := range []string{"callers", "call_chain"} { + body := map[string]any{"edges": []any{map[string]any{"from": "sample/main.ts::" + step.Marker, "to": "sample/main.ts::Example", "kind": "calls"}}} + if !editProbeCorrect(oracle, body, c, step, plan.Callee) { + t.Fatalf("actual %s edge rejected", oracle) + } + } +} + +func TestEditProbeTypeScriptRejectsFalseSourceAndIdentity(t *testing.T) { + c := editProbeTypeScriptCorpus() + plan, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := plan.Steps[0] + source := strings.ReplaceAll(c.EditTemplate, "{{marker}}", step.Marker) + for name, changed := range map[string]string{ + "wrong_callee": strings.ReplaceAll(source, "Example()", "Other()"), + "comment_only": "/* " + source + " */", + "line_comment": "// " + strings.TrimSpace(source), + "string_only": "const quoted = `" + source + "`;", + "stale_body": strings.ReplaceAll(source, "Example()", "1"), + "duplicate": source + source, + "comment_call": strings.ReplaceAll(source, "return Example();", "return 1; /* Example() */"), + } { + t.Run(name, func(t *testing.T) { + body := map[string]any{"id": "sample/main.ts::" + step.Marker, "name": step.Marker, "file_path": "sample/main.ts", "source": changed, "fresh": true} + if editProbeCorrect("source", body, c, step, plan.Callee) { + t.Fatal("false source became actual-call proof") + } + file := map[string]any{"path": "sample/main.ts", "content": changed, "fresh": true} + if editProbeCorrect("file", file, c, step, plan.Callee) { + t.Fatal("false file became actual-call proof") + } + text := map[string]any{"matches": []any{map[string]any{"path": "sample/main.ts", "line": json.Number("1"), "text": changed}}} + if editProbeCorrect("text", text, c, step, plan.Callee) { + t.Fatal("comment/string/stale call became text proof") + } + }) + } + for _, field := range []string{"id", "file_path"} { + body := map[string]any{"id": "sample/main.ts::" + step.Marker, "name": step.Marker, "file_path": "sample/main.ts", "source": source} + body[field] = "sample/foreign.ts::" + step.Marker + if editProbeCorrect("source", body, c, step, plan.Callee) { + t.Fatalf("wrong %s accepted", field) + } + } + if editProbeCorrect("file", map[string]any{"path": "sample/foreign.ts", "content": source}, c, step, plan.Callee) { + t.Fatal("wrong file accepted") + } +} + +func TestEditProbeTypeScriptRejectsInvalidTemplates(t *testing.T) { + base := editProbeTypeScriptCorpus() + for _, template := range []string{ + strings.ReplaceAll(base.EditTemplate, "Example()", "Other()"), + "/* " + base.EditTemplate + " */", + strings.ReplaceAll(base.EditTemplate, "return Example();", "return \"Example()\";"), + strings.ReplaceAll(base.EditTemplate, "return Example();", "return 1; /* Example() */"), + } { + c := base + c.EditTemplate = template + if _, err := makeEditProbePlan(c, 42, 1); err == nil { + t.Fatal("invalid TypeScript fixture template admitted") + } + } +} diff --git a/bench/load/edit_probe_write.go b/bench/load/edit_probe_write.go new file mode 100644 index 000000000..0715d8341 --- /dev/null +++ b/bench/load/edit_probe_write.go @@ -0,0 +1,98 @@ +package main + +import ( + "bytes" + "errors" + "go/parser" + "go/token" + "path/filepath" + "strings" + "unicode/utf8" +) + +func editProbeTool(c Corpus) (string, error) { + tool := c.EditTool + if tool == "" { + tool = "edit.file" + } + switch tool { + case "edit.file": + if c.EditKind == "rename" || c.EditKind == "delete" || c.EditKind == "modify_body" || c.EditKind == "modify_caller_body" || c.EditKind == "change_signature" || c.EditKind == "add_crossfile_call" || c.EditSender == editProbeCreateDelete { + if err := editProbeInitialFixture(c); err != nil { + return "", err + } + } else if c.EditInitialContent != "" { + return "", errors.New("edit_initial_content requires explicit edit.write, edit.batch, edit.symbol or rename/delete fixture") + } + case "edit.write", "edit.batch", "edit.symbol": + if err := editProbeInitialFixture(c); err != nil { + return "", err + } + case "move_symbol", "refactor.move": + if filepath.Ext(c.File) != ".go" { + if err := editProbeRelocationRefusalFixture(c); err != nil { + return "", err + } + return "refactor.move", nil + } + if c.EditKind != "relocate" { + return "", errors.New("move_symbol applies to relocation, not append/body/signature/rename/delete fixtures") + } + if c.EditExpect != "" { + return "", errors.New("edit_expect refusal applies only to TypeScript/Python relocation: Go relocation is supported and measured as a move") + } + if err := editProbeMoveFixture(c); err != nil { + return "", err + } + return "refactor.move", nil + default: + return "", errors.New("edit-and-probe supports edit.file, explicit edit.write, edit.symbol or one-file edit.batch") + } + if tool == "edit.symbol" { + if err := editProbeSymbolAnchor(c); err != nil { + return "", err + } + } + return tool, nil +} + +func editProbeInitialFixture(c Corpus) error { + if c.EditInitialContent == "" || !utf8.ValidString(c.EditInitialContent) || c.EditAnchor == "" || strings.Count(c.EditInitialContent, c.EditAnchor) != 1 { + return errors.New("explicit edit fixture requires UTF-8 initial content with one stable anchor") + } + switch filepath.Ext(c.File) { + case ".go": + if _, err := parser.ParseFile(token.NewFileSet(), "fixture.go", c.EditInitialContent, 0); err != nil { + return errors.New("initial Go content must parse") + } + case ".ts": + if valid, _ := editProbeTypeScriptFixture(editProbeCrossFilePlain(c, c.EditInitialContent), "", ""); !valid { + return errors.New("initial TypeScript content must match explicit fixture") + } + case ".py": + if valid, _ := editProbePythonFixture(editProbeCrossFilePlain(c, c.EditInitialContent), "", ""); !valid { + return errors.New("initial Python content must match explicit fixture") + } + default: + return errors.New("unsupported edit fixture language") + } + return nil +} + +// Content is immutable offered client intent. Each request contains the original +// bytes plus the offered prefix of additions, not a reread of current disk state. +// Concurrent/out-of-order writes may therefore overwrite later intent; they are +// never serialized, regenerated or silently retried with different content. +func editProbeWriteRequest(c Corpus, content, marker string) (request, string) { + content = strings.Replace(content, c.EditAnchor, c.EditAnchor+strings.ReplaceAll(c.EditTemplate, "{{marker}}", marker), 1) + return request{Tool: "edit", Operation: "write", Arguments: map[string]any{"operation": "write", "target": map[string]any{"file": c.File}, "content": content}}, content +} + +// Check the immutable plan against the actual pinned clone before opening any +// per-edit probe sessions or dispatching edits. This does not refresh intent. +func editProbeWriteAdmission(c Corpus, original []byte) error { + if (c.EditTool == "edit.write" || c.EditTool == "edit.batch" || c.EditTool == "edit.symbol" || c.EditKind == "rename" || c.EditKind == "delete" || c.EditKind == "modify_body" || c.EditKind == "modify_caller_body" || c.EditKind == "change_signature" || c.EditKind == "add_crossfile_call" || c.EditKind == "relocate" || c.EditSender == editProbeCreateDelete) && !bytes.Equal(original, []byte(c.EditInitialContent)) { + return errors.New("initial content differs from the pinned edit fixture bytes") + } + return nil +} diff --git a/bench/load/edit_probe_write_test.go b/bench/load/edit_probe_write_test.go new file mode 100644 index 000000000..2a9a45ff0 --- /dev/null +++ b/bench/load/edit_probe_write_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +func editProbeWriteCorpus(t *testing.T, language string) Corpus { + t.Helper() + c := editProbeCorpus() + switch language { + case "go": + c.EditInitialContent = "package sample\nfunc Example() int { return 1 }\n" + c.EditAnchor + "\n" + case "typescript": + c = editProbeTypeScriptCorpus() + data, err := os.ReadFile("testdata/edit-probe/typescript/main.ts") + if err != nil { + t.Fatal(err) + } + c.EditInitialContent = string(data) + case "python": + c = editProbePythonCorpus() + data, err := os.ReadFile("testdata/edit-probe/python/main.py") + if err != nil { + t.Fatal(err) + } + c.EditInitialContent = string(data) + } + c.EditTool = "edit.write" + return c +} + +func TestEditProbeWriteHasImmutableOfferedPrefixAndSameSchedule(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeWriteCorpus(t, language) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + file := c + file.EditTool = "" + file.EditInitialContent = "" + reference, err := makeEditProbePlan(file, 42, 100) + if err != nil { + t.Fatal(err) + } + if plan.Tool != "edit.write" || len(plan.Steps) != 100 { + t.Fatal("wrong tool/denominator") + } + for i, step := range plan.Steps { + req := step.Request + content, ok := req.Arguments["content"].(string) + if !ok || req.Tool != "edit" || req.Operation != "write" || req.Arguments["operation"] != "write" || req.Arguments["target"].(map[string]any)["file"] != c.File || req.Arguments["options"].(map[string]any)["mutation_id"] != step.MutationID { + t.Fatal("wrong public write arguments", req) + } + if step.DueNS != reference.Steps[i].DueNS || step.ActivateNS != reference.Steps[i].ActivateNS || step.Marker != reference.Steps[i].Marker { + t.Fatal("tool changed offered schedule") + } + if !editProbeFixtureSource(c.File, content, step.Marker, plan.Callee) { + t.Fatal("write missing actual intrafile call") + } + for j, other := range plan.Steps { + if strings.Contains(content, other.Marker) != (j <= i) { + t.Fatal("content is not exact offered prefix", i, j) + } + } + hash, err := hashJSON(req) + if err != nil || hash != step.RequestHash { + t.Fatal("write intent hash mismatch") + } + } + if err := editProbeWriteAdmission(c, []byte(c.EditInitialContent)); err != nil { + t.Fatal(err) + } + if err := editProbeWriteAdmission(c, []byte(c.EditInitialContent+"\n")); err == nil { + t.Fatal("drifted initial bytes admitted") + } + // Apply later intent first, then earlier intent. The oracle must reject the + // genuinely overwritten later marker; neither request is regenerated. + earlier := plan.Steps[0].Request.Arguments["content"].(string) + if editProbeFixtureSource(c.File, earlier, plan.Steps[1].Marker, plan.Callee) { + t.Fatal("lost later write became correct") + } + if got := plan.Steps[1].Request.Arguments["content"].(string); !strings.Contains(got, plan.Steps[0].Marker) { + t.Fatal("intent was refreshed after reordering") + } + }) + } +} + +func TestEditProbeWriteRefusesUnknownOrUnpinnedIntent(t *testing.T) { + c := editProbeWriteCorpus(t, "go") + for _, bad := range []Corpus{ + func() Corpus { x := c; x.EditTool = "edit.symbol"; return x }(), + func() Corpus { x := c; x.EditInitialContent = ""; return x }(), + func() Corpus { x := c; x.EditInitialContent += c.EditAnchor; return x }(), + func() Corpus { x := c; x.EditInitialContent = "not Go " + c.EditAnchor; return x }(), + func() Corpus { x := c; x.EditInitialContent += string([]byte{0xff}); return x }(), + func() Corpus { x := c; x.EditTool = "edit.file"; return x }(), + } { + if _, err := makeEditProbePlan(bad, 42, 1); err == nil { + t.Fatal("invalid write plan admitted", bad.EditTool) + } + } +} + +func TestEditProbeWriteOraclesRejectForeignStaleAndCommentOnlyBytes(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + c := editProbeWriteCorpus(t, language) + p, err := makeEditProbePlan(c, 42, 1) + if err != nil { + t.Fatal(err) + } + step := p.Steps[0] + content := step.Request.Arguments["content"].(string) + body := map[string]any{"path": c.File, "content": content} + if !editProbeCorrect("file", body, c, step, p.Callee) { + t.Fatal("actual write source refused") + } + for _, bad := range []map[string]any{ + {"path": "foreign/other.go", "content": content}, + {"path": c.File, "content": c.EditInitialContent}, + {"path": c.File, "content": strings.ReplaceAll(content, "Example()", "Other()")}, + {"path": c.File, "content": "/*" + content + "*/"}, + } { + if editProbeCorrect("file", bad, c, step, p.Callee) { + t.Fatal("wrong file/callee/stale/comment evidence accepted") + } + } + source := map[string]any{"id": c.File + "::" + step.Marker, "file_path": c.File, "name": step.Marker, "source": content} + if !editProbeCorrect("source", source, c, step, p.Callee) { + t.Fatal("real source identity refused") + } + source["id"] = "foreign.go::" + step.Marker + if editProbeCorrect("source", source, c, step, p.Callee) { + t.Fatal("foreign producer identity accepted") + } + }) + } +} diff --git a/bench/load/paired_command.go b/bench/load/paired_command.go index 544c4f5f5..4125b3926 100644 --- a/bench/load/paired_command.go +++ b/bench/load/paired_command.go @@ -58,6 +58,16 @@ func parsePaired(args []string) (runOptions, []buildSpec, error) { } switch o.Scenario { case "read", "edit", "many", "many-edit", "worktrees", "sweep": + case editProbeScenario: + if len(builds) != 2 { + return o, nil, errors.New("paired edit-and-probe requires one baseline and one candidate") + } + if o.Editors != 1 || o.Agents != 0 { + return o, nil, errors.New("paired edit-and-probe is the one-editor, no-reader no-regression cell") + } + if o.EditProbeRole != "" { + return o, nil, errors.New("paired edit-and-probe assigns baseline and candidate roles by build order") + } default: return o, nil, errors.New("paired supports workload scenarios; cold/restart/all use standalone runs") } @@ -111,6 +121,7 @@ func pairedCommand(ctx context.Context, args []string) error { options := o options.Binary = build.binary options.pairedDriver = coordinator.driver(i) + options.editProbeBaseline = o.Scenario == editProbeScenario && i == 0 dir := filepath.Join(root, build.label) if err := os.MkdirAll(dir, 0o700); err != nil { coordinator.abort(err) diff --git a/bench/load/paired_core.go b/bench/load/paired_core.go index a3e4e4714..2b051e09d 100644 --- a/bench/load/paired_core.go +++ b/bench/load/paired_core.go @@ -20,23 +20,24 @@ type pairedState struct { } type pairedCoordinator struct { - ctx context.Context - cancel context.CancelFunc - options runOptions - labels []string - group string - mu sync.Mutex - states []*pairedState - registered int - started bool - finished bool - err error - firstReady chan struct{} - done chan struct{} - orders map[string]uint64 - records []pairedRecord - window measurementWindow - metrics map[string]Metric + ctx context.Context + cancel context.CancelFunc + options runOptions + labels []string + group string + mu sync.Mutex + states []*pairedState + registered int + started bool + finished bool + err error + firstReady chan struct{} + done chan struct{} + orders map[string]uint64 + records []pairedRecord + window measurementWindow + metrics map[string]Metric + editContinuations sync.WaitGroup // owned own-ACK readback/status continuations } func newPairedCoordinator(ctx context.Context, o runOptions, labels []string, group string) (*pairedCoordinator, error) { @@ -128,6 +129,21 @@ func (c *pairedCoordinator) execute() { } } } + if base.phase.Name == editProbeScenario { + if result = c.preparePairedEditProbe(); result != nil { + return + } + } + budget := c.options.Duration + switch base.phase.Name { + case "sweep": + budget = c.options.SweepBudget + case editProbeScenario: + budget, result = base.phase.editProbeRuntime.plan.budget(c.options.Deadline, len(c.states)) + if result != nil { + return + } + } samplers := make([]processSampler, len(c.states)) monitors := make([]*phaseMonitor, len(c.states)) for i, state := range c.states { @@ -141,15 +157,14 @@ func (c *pairedCoordinator) execute() { groupBefore = cgroupMetrics(c.group, os.Getpagesize()) } c.window = measurementWindow{Started: time.Now(), Method: "coupled native back-to-back requests; process counters bracket the common group window; per-build sample endpoints retained"} - budget := c.options.Duration - if base.phase.Name == "sweep" { - budget = c.options.SweepBudget - } ctx, cancel := context.WithTimeout(c.ctx, budget) defer cancel() - if base.phase.Name == "sweep" { + switch base.phase.Name { + case "sweep": result = c.sweep(ctx) - } else { + case editProbeScenario: + result = c.editProbes(ctx) + default: result = c.agents(ctx) } c.window.Ended = time.Now() @@ -185,14 +200,18 @@ func (c *pairedCoordinator) execute() { } else { state.phase.Gate["paired_measurement_window"] = data } - finishEditMetrics(state.phase) + if state.phase.EditProbe == nil { + finishEditMetrics(state.phase) + } } } type pairedCallResult struct { - data json.RawMessage - err error - sample pairedSample + data json.RawMessage + err error + sample pairedSample + identity wireCallIdentity + pending bool } func (c *pairedCoordinator) callAcross(ctx context.Context, r request, repoIndex, agent, step, slot int, role string, timeout time.Duration, sweepIndex int) map[string]pairedCallResult { @@ -203,6 +222,9 @@ func (c *pairedCoordinator) callAcrossWithEnds(ctx context.Context, r request, r baseRepo := c.states[0].p.Repos[repoIndex] hash, hashErr := hashJSON(r) tuple := requestTuple{Tool: r.name(), Role: role, Repo: baseRepo.Corpus.Label, Commit: baseRepo.Corpus.Commit, Agent: agent, Step: step, Slot: slot, RequestHash: hash} + if r.ExpectedAbsentSymbol != "" { + tuple.ExpectedOutcome = "symbol_absent" + } c.mu.Lock() category := role + "." + r.name() tuple.Ordinal = c.orders[category] @@ -222,7 +244,17 @@ func (c *pairedCoordinator) callAcrossWithEnds(ctx context.Context, r request, r sample := pairedSample{Repo: repo.Corpus.Label, Commit: repo.Corpus.Commit, RequestHash: hash, Started: time.Now(), Latency: unavailable("ms", "request was not dispatched"), Outcome: "phase_censored"} var data json.RawMessage var rows []Attempt + var identity wireCallIdentity + var editCall pairedEditProbeCall err := hashErr + if runtime := state.phase.editProbeRuntime; runtime != nil { + var prepareErr error + editCall, prepareErr = runtime.preparePairedCall(ctx, role, step, slot) + if prepareErr != nil { + err = errors.Join(err, prepareErr) + c.abort(prepareErr) + } + } budget := timeout if end, ok := ends[label]; ok { budget = time.Until(end) @@ -238,28 +270,49 @@ func (c *pairedCoordinator) callAcrossWithEnds(ctx context.Context, r request, r } sample.Latency = unavailable("ms", "visibility deadline elapsed before probe dispatch") } else if err == nil && ctx.Err() == nil { - client := state.clients[agent] - if sweepIndex >= 0 { - client = state.sweepClients[sweepIndex] + if editCall.runtime != nil { + data, identity, rows, err = editCall.perform(r, role, slot) + } else { + client := state.clients[agent] + if sweepIndex >= 0 { + client = state.sweepClients[sweepIndex] + } + data, rows, err = callTool(ctx, client, r, dispatchArgs(r, repo), role, agent, step, slot, budget, waitContext) } - data, rows, err = callTool(ctx, client, r, dispatchArgs(r, repo), role, agent, step, slot, budget, waitContext) if len(rows) > 0 { last := rows[len(rows)-1] sample.Outcome, sample.Code, sample.Error = last.Outcome, last.Code, last.Error + sample.ExpectedOutcome = last.ExpectedOutcome sample.Latency = measured(last.LogicalMS, "ms") } } else if err == nil { err = ctx.Err() } + if editCall.runtime != nil && err != nil && sample.Outcome == "success" { + // Logical evidence persistence may fail after a genuine RPC reply. + // Retain its endpoint, but never use incomplete evidence in ratios. + sample.Outcome = "error" + if errors.Is(err, errEditProbeEvidenceIncomplete) { + sample.Outcome = "harness_incomplete" + } + } if err != nil && sample.Error == "" { sample.Error = err.Error() } sample.Ended = time.Now() + if !identity.Finished.IsZero() { + sample.Ended = identity.Finished + } + // Start this build's own-ACK continuation before the loop awaits the + // partner response. Its readbacks/status remain explicitly unpaired. + editCall.continueAfterACK(c, data, identity, err) record.Samples[label] = sample - results[label] = pairedCallResult{data: data, err: err, sample: sample} - c.mu.Lock() - state.phase.Attempts = append(state.phase.Attempts, rows...) - c.mu.Unlock() + results[label] = pairedCallResult{data: data, err: err, sample: sample, identity: identity, pending: editCall.pending} + if editCall.runtime == nil { // editProbe.perform already owns its attempt append. + c.mu.Lock() + state.phase.Attempts = append(state.phase.Attempts, rows...) + c.mu.Unlock() + } } c.mu.Lock() c.records = append(c.records, record) diff --git a/bench/load/paired_edit_probe.go b/bench/load/paired_edit_probe.go new file mode 100644 index 000000000..a2ad5d72f --- /dev/null +++ b/bench/load/paired_edit_probe.go @@ -0,0 +1,207 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "sync" + "time" +) + +// Callback-local state for the existing paired dispatcher, not another queue, +// scheduler or request registry. Only edits and the seven oracles are paired. +type pairedEditProbeCall struct { + runtime *editProbeRuntime + state *editProbeState + ctx context.Context + phaseContext context.Context + cancel context.CancelFunc + started time.Time + purpose string + editing, pending bool +} + +// The existing two completion callbacks own this one per-edit cancellation. +// The first completed build cannot cancel its partner's outstanding probes. +type pairedEditProbeLifetime struct { + mu sync.Mutex + finished int + cancel context.CancelFunc + evidenceOnce sync.Once +} + +func (r *editProbeRuntime) preparePairedCall(ctx context.Context, role string, ordinal, slot int) (pairedEditProbeCall, error) { + if ordinal < 0 || ordinal >= len(r.states) { + return pairedEditProbeCall{}, errors.New("paired edit ordinal outside offered plan") + } + out := pairedEditProbeCall{runtime: r, state: r.states[ordinal], ctx: ctx} + out.state.mu.Lock() + if out.state.pairedContext != nil { + ctx = out.state.pairedContext + } + out.state.mu.Unlock() + out.ctx, out.phaseContext = ctx, ctx + switch { + case role == "workload" && slot == 0: + out.editing, out.purpose = true, "edit" + out.state, out.ctx, out.cancel, out.started = r.beginEdit(ctx, r.plan.Steps[ordinal]) + case role == "probe" && slot >= 10 && slot < 10+len(editProbeOracles(r.p.Repos[0].Corpus, r.plan.Steps[ordinal])): + oracle := editProbeOracles(r.p.Repos[0].Corpus, r.plan.Steps[ordinal])[slot-10] + out.purpose = "oracle:" + oracle.name + out.pending = out.state.noteProbeStart() + default: + return pairedEditProbeCall{}, errors.New("unsupported paired edit-probe operation") + } + return out, nil +} +func (call pairedEditProbeCall) perform(req request, role string, stream int) (json.RawMessage, wireCallIdentity, []Attempt, error) { + client := call.state.client + if call.editing { + client = call.runtime.editorFor(call.state.step) + } + return call.runtime.performObserved(call.ctx, call.state, client, req, call.purpose, role, stream) +} +func (call pairedEditProbeCall) continueAfterACK(c *pairedCoordinator, data json.RawMessage, identity wireCallIdentity, err error) { + if call.runtime == nil || !call.editing { + return + } + c.editContinuations.Go(func() { + defer call.state.stopProbes() + defer call.cancel() + call.runtime.finishEditAfterACK(call.phaseContext, call.ctx, call.state, call.state.step, call.started, data, identity, err) + }) +} + +func (c *pairedCoordinator) preparePairedEditProbe() error { + if len(c.states) != 2 { + return errors.New("paired edit-and-probe needs baseline and candidate") + } + base := c.states[0].phase.editProbeRuntime + if base == nil { + return errors.New("paired baseline edit runtime unavailable") + } + if base.plan.Mode == "cli" { + return errors.New("paired edit-and-probe pairs MCP probe calls; CLI probes run unpaired") + } + if base.plan.Sender != "" { + return errors.New("paired edit-and-probe pairs plain MCP edit cells; native, hot-file and restart cells run unpaired") + } + for i, state := range c.states { + r := state.phase.editProbeRuntime + if r == nil || !reflect.DeepEqual(r.plan, base.plan) || r.options.Deadline != c.options.Deadline || r.requireAccounting != (i != 0) { + return errors.New("paired edit plan/budget/accounting roles differ") + } + if r.p.Root == "" || r.p.Store == "" || r.p.Socket == "" || !inside(r.p.Root, r.p.Store) || + i > 0 && (r.p == base.p || r.p.Root == base.p.Root || r.p.Store == base.p.Store || r.p.Socket == base.p.Socket) { + return errors.New("paired edit builds do not own independent roots/stores/sockets") + } + before, err := r.stats(c.ctx) + state.phase.EditProbe.StatsBefore = before + if err != nil { + return err + } + state.phase.EditProbe.Limitations = append(state.phase.EditProbe.Limitations, + "paired first-correct times are coupled probe observations, not independent 25ms freshness acceptance", + "immediate own-ACK readbacks and receipt/status controls are unpaired and excluded from matched latency ratios", + "absent baseline phases remain unavailable; candidate accounting stays required") + } + return nil +} + +func (c *pairedCoordinator) editProbes(ctx context.Context) error { + base := c.states[0].phase.editProbeRuntime + for _, state := range c.states { + r := state.phase.editProbeRuntime + r.origin = c.window.Started + state.phase.Started = r.origin + } + err := runEditProbeArrivals(ctx, base.plan, c.window.Started, c.activatePairedEditProbe, func(ctx context.Context, step editProbeStep) { + c.callAcross(ctx, step.Request, 0, 0, step.Ordinal, 0, "workload", c.options.Deadline, -1) + }, waitContext) + c.editContinuations.Wait() // Each continuation joins its owned paired probes. + var failures []error + failures = append(failures, err) + for _, state := range c.states { + r := state.phase.editProbeRuntime + failures = append(failures, r.finish()) + after, statsErr := r.stats(ctx) + state.phase.EditProbe.StatsAfter = after + failures = append(failures, statsErr) + } + return errors.Join(failures...) +} + +// A shared oracle pair lives until both own-ACK continuations are done. Fast +// builds keep matching probes, but their first-correct endpoints never change. +// Activation only starts tasks; it never waits for a network response. +func (c *pairedCoordinator) activatePairedEditProbe(ctx context.Context, step editProbeStep) { + lifetime := &pairedEditProbeLifetime{} + var probeCtx context.Context + probeCtx, lifetime.cancel = context.WithCancel(ctx) + for _, state := range c.states { + var once sync.Once + done := func() { + once.Do(func() { + lifetime.mu.Lock() + lifetime.finished++ + all := lifetime.finished == len(c.states) + lifetime.mu.Unlock() + if all { + lifetime.cancel() + } + }) + } + s := state.phase.editProbeRuntime.activateProbeState(step, done) + s.mu.Lock() + s.pairedContext = probeCtx + s.mu.Unlock() + } + oracles := editProbeOracles(c.states[0].p.Repos[0].Corpus, step) + for index, oracle := range oracles { + for _, state := range c.states { + state.phase.editProbeRuntime.states[step.Ordinal].probes.Add(1) + } + go func() { + defer func() { + for _, state := range c.states { + state.phase.editProbeRuntime.states[step.Ordinal].probes.Done() + } + }() + ticker := time.NewTicker(editProbePeriod) + defer ticker.Stop() + for probeCtx.Err() == nil { + results := c.callAcross(probeCtx, oracle.call, 0, 0, step.Ordinal, 10+index, "probe", c.options.Deadline, -1) + evidenceFailed := false + for i, label := range c.labels { + result := results[label] + r := c.states[i].phase.editProbeRuntime + if err := r.observeProbe(r.states[step.Ordinal], oracle, result.sample.Started, result.data, result.identity, result.err, result.pending); err != nil { + evidenceFailed = true + } + } + if evidenceFailed { + lifetime.evidenceOnce.Do(func() { + c.cancelPairedEditEvidence(step.Ordinal, lifetime.cancel) + }) + return + } + select { + case <-probeCtx.Done(): + return + case <-ticker.C: + } + } + }() + } +} + +func (c *pairedCoordinator) cancelPairedEditEvidence(ordinal int, cancel context.CancelFunc) { + for _, state := range c.states { + s := state.phase.editProbeRuntime.states[ordinal] + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "paired probes canceled after partner evidence failure") + s.mu.Unlock() + } + cancel() +} diff --git a/bench/load/paired_edit_probe_test.go b/bench/load/paired_edit_probe_test.go new file mode 100644 index 000000000..512307b71 --- /dev/null +++ b/bench/load/paired_edit_probe_test.go @@ -0,0 +1,317 @@ +package main + +import ( + "context" + "errors" + "path/filepath" + "sync" + "testing" + "time" +) + +func pairedEditProbeFixture(t *testing.T, ctx context.Context, count int, handle func(string, stubFrame) (any, *rpcError)) *pairedCoordinator { + t.Helper() + options := runOptions{Seed: 1, Deadline: time.Second} + c, err := newPairedCoordinator(ctx, options, []string{"baseline", "candidate"}, "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(c.cancel) + corpus := editProbeCorpus() + plan, err := makeEditProbePlan(corpus, 1, 100) + if err != nil { + t.Fatal(err) + } + plan.Steps = plan.Steps[:count] + plan.Count = count + for i, label := range c.labels { + socket := startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Method == "resources/read" && frame.Params["uri"] == "gortex://stats" { + return map[string]any{"contents": []any{map[string]any{"type": "text", "text": `{}`}}}, nil + } + return handle(label, frame) + }) + editor, err := openMCP(ctx, socket, "/owned/"+label) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = editor.Close() }) + root := t.TempDir() + p := &privateRun{Root: root, Store: filepath.Join(root, "store.db"), Socket: socket, Repos: []privateRepo{{Corpus: corpus, Prefix: "sample"}}} + phase := &Phase{Metrics: make(map[string]Metric), EditProbe: newEditProbeReport(plan, corpus, i != 0)} + r := &editProbeRuntime{options: options, p: p, phase: phase, plan: plan, editors: []*socketClient{editor}, origin: time.Now(), requireAccounting: i != 0} + phase.editProbeRuntime = r + clients := []*socketClient{editor} + for _, step := range plan.Steps { + prober, err := openMCP(ctx, socket, "/owned/"+label) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = prober.Close() }) + sink := newEditProbeHeldSink(t) + sink.unblock() + state := startHeldEditProbeSpool(t, sink) + state.step = step + state.record = newEditProbeRecord(corpus, step) + state.record.Errors = nil + state.client = prober + r.states = append(r.states, state) + clients = append(clients, prober) + } + c.states[i] = &pairedState{p: p, clients: clients, phase: phase} + } + return c +} + +func TestPairedEditOwnACKStartsReadbackBeforePartnerACK(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + readback, partner := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + var releaseOnce, readOnce, partnerOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + c := pairedEditProbeFixture(t, ctx, 1, func(label string, frame stubFrame) (any, *rpcError) { + if frame.Params["name"] == "edit" { + if label == "candidate" { + partnerOnce.Do(func() { close(partner) }) + <-release + } + return stubContent(map[string]any{"mutation_receipt": label + "-receipt"}), nil + } + if label == "baseline" && frame.Params["name"] == "read" { + readOnce.Do(func() { close(readback) }) + } + return stubContent(map[string]any{}), nil + }) + done := make(chan struct{}) + t.Cleanup(func() { unblock(); cancel(); <-done; c.editContinuations.Wait() }) + go func() { + defer close(done) + step := c.states[0].phase.editProbeRuntime.plan.Steps[0] + c.callAcross(ctx, step.Request, 0, 0, 0, 0, "workload", time.Second, -1) + }() + awaitEditProbeSpool(t, partner) + awaitEditProbeSpool(t, readback) // Partner edit still cannot return its ACK. + unblock() + cancel() + awaitEditProbeSpool(t, done) + c.editContinuations.Wait() + first := c.states[0].phase.editProbeRuntime.states[0].record + if first.Receipt != "baseline-receipt" || len(first.Readbacks) == 0 { + t.Fatalf("own ACK/readback lost: %+v", first) + } + if len(c.records) != 1 || len(c.records[0].Samples) != 2 { + t.Fatal("partner failure/cancel erased pair denominator") + } +} + +func TestPairedEditBlockedProbeCannotDelayOfferedEdits(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + probe, edited := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + var releaseOnce, probeOnce sync.Once + var mu sync.Mutex + editCount := 0 + unblock := func() { releaseOnce.Do(func() { close(release) }) } + c := pairedEditProbeFixture(t, ctx, 1, func(label string, frame stubFrame) (any, *rpcError) { + if frame.Connection > 1 { + probeOnce.Do(func() { close(probe) }) + <-release + } + if frame.Params["name"] == "edit" { + mu.Lock() + editCount++ + if editCount == 2 { + close(edited) + } + mu.Unlock() + return stubContent(map[string]any{"mutation_receipt": label + "-receipt"}), nil + } + return stubContent(map[string]any{}), nil + }) + origin := time.Now() + plan := c.states[0].phase.editProbeRuntime.plan + plan.Steps[0].ActivateNS = 0 + plan.Steps[0].DueNS = int64(25 * time.Millisecond) + for _, state := range c.states { + state.phase.editProbeRuntime.origin = origin + state.phase.editProbeRuntime.plan = plan + } + done := make(chan struct{}) + t.Cleanup(func() { unblock(); cancel(); <-done; c.editContinuations.Wait() }) + go func() { + defer close(done) + _ = runEditProbeArrivals(ctx, plan, origin, c.activatePairedEditProbe, func(ctx context.Context, step editProbeStep) { + c.callAcross(ctx, step.Request, 0, 0, step.Ordinal, 0, "workload", time.Second, -1) + }, waitContext) + }() + awaitEditProbeSpool(t, probe) + awaitEditProbeSpool(t, edited) + cancel() + unblock() + awaitEditProbeSpool(t, done) + c.editContinuations.Wait() +} + +func TestPairedEditBlockedACKCannotShiftNextOfferedPair(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + held, next := make(chan struct{}), make(chan struct{}) + release := make(chan struct{}) + var releaseOnce, heldOnce, nextOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + c := pairedEditProbeFixture(t, ctx, 2, func(label string, frame stubFrame) (any, *rpcError) { + if frame.Params["name"] == "edit" { + args, _ := frame.Params["arguments"].(map[string]any) + opts, _ := args["options"].(map[string]any) + if opts["mutation_id"] == "load-edit-0000000000000001-000" && label == "baseline" { + heldOnce.Do(func() { close(held) }) + <-release + } + if opts["mutation_id"] == "load-edit-0000000000000001-001" { + nextOnce.Do(func() { close(next) }) + } + return stubContent(map[string]any{"mutation_receipt": label + "-receipt"}), nil + } + return stubContent(map[string]any{}), nil + }) + origin := time.Now() + plan := c.states[0].phase.editProbeRuntime.plan + for i := range plan.Steps { + plan.Steps[i].ActivateNS = 0 + plan.Steps[i].DueNS = int64(time.Duration(i+1) * 25 * time.Millisecond) + } + for _, state := range c.states { + state.phase.editProbeRuntime.origin = origin + state.phase.editProbeRuntime.plan = plan + } + done := make(chan struct{}) + t.Cleanup(func() { unblock(); cancel(); <-done; c.editContinuations.Wait() }) + go func() { + defer close(done) + _ = runEditProbeArrivals(ctx, plan, origin, func(context.Context, editProbeStep) {}, func(ctx context.Context, step editProbeStep) { + c.callAcross(ctx, step.Request, 0, 0, step.Ordinal, 0, "workload", time.Second, -1) + }, waitContext) + }() + awaitEditProbeSpool(t, held) + awaitEditProbeSpool(t, next) + cancel() + unblock() + awaitEditProbeSpool(t, done) + c.editContinuations.Wait() +} + +func TestPairedEditPolicyKeepsBaselinePhasesExplicitAndPlanExact(t *testing.T) { + args := []string{"--build", "main=/private/main", "--build", "candidate=/private/candidate", "--manifest", "/private/manifest.json", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", "1", "--worktrees", "0", "--edit-count", "100"} + options, builds, err := parsePaired(args) + if err != nil || len(builds) != 2 || options.EditCount != 100 { + t.Fatalf("initial paired policy: %v %v", builds, err) + } + if _, _, err := parsePaired(append(args, "--build", "third=/private/third")); err == nil { + t.Fatal("third build silently widened initial paired cell") + } + // Baseline missing instrumentation is a qualification, not a measured zero. + baseline := newEditProbeReport(editProbePlan{}, Corpus{}, false) + candidate := newEditProbeReport(editProbePlan{}, Corpus{}, true) + if baseline.AccountingRequired || !candidate.AccountingRequired { + t.Fatal("baseline/candidate accounting roles lost") + } + missing := checkEditProbeAccounting(map[string]any{"receipt": "r"}, "r") + if missing.State == "observed_single_window_checks_pass" || missing.A != nil || missing.B != nil { + t.Fatal("absent baseline phases became zero or pass") + } +} + +func TestPairedEditPreparationRejectsForeignPlanSharedStoreAndWrongAccountingRole(t *testing.T) { + for _, name := range []string{"valid", "foreign_plan", "shared_store", "wrong_role", "cli_probes", "native_sender"} { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c := pairedEditProbeFixture(t, ctx, 1, func(string, stubFrame) (any, *rpcError) { + return stubContent(map[string]any{}), nil + }) + candidate := c.states[1].phase.editProbeRuntime + switch name { + case "foreign_plan": + candidate.plan.Steps = append([]editProbeStep(nil), candidate.plan.Steps...) + candidate.plan.Steps[0].MutationID = "foreign" + case "shared_store": + candidate.p.Store = c.states[0].p.Store + case "wrong_role": + candidate.requireAccounting = false + case "cli_probes": + c.states[0].phase.editProbeRuntime.plan.Mode, candidate.plan.Mode = "cli", "cli" + case "native_sender": + c.states[0].phase.editProbeRuntime.plan.Sender, candidate.plan.Sender = editProbeShellAfterTool, editProbeShellAfterTool + } + if err := c.preparePairedEditProbe(); (err != nil) != (name != "valid") { + t.Fatalf("prepared %s: %v", name, err) + } + }) + } +} + +func TestPairedEvidenceFailureCancelsPartnerAndRefusesEarlierCompleteResult(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c := pairedEditProbeFixture(t, ctx, 1, func(string, stubFrame) (any, *rpcError) { + return stubContent(map[string]any{}), nil + }) + baseline := c.states[0].phase.editProbeRuntime + candidate := c.states[1].phase.editProbeRuntime + baseline.states[0].record.Outcome = "complete" + state := candidate.states[0] + sink := state.sink.(*editProbeHeldSink) + sink.mu.Lock() + sink.writeErr = errors.New("owned evidence disk failure") + sink.mu.Unlock() + if err := state.emit(map[string]any{"event": "held-probe"}); err != nil { + t.Fatal(err) + } + state.finishRaw() // Actual failing write is joined before its next observation. + err := state.emit(map[string]any{"event": "next-probe"}) + if !errors.Is(err, errEditProbeEvidenceIncomplete) { + t.Fatalf("actual evidence failure was not visible: %v", err) + } + shared, stop := context.WithCancel(ctx) + defer stop() + c.cancelPairedEditEvidence(0, stop) + if shared.Err() == nil { + t.Fatal("partner continued under a product deadline after coupled evidence loss") + } + for _, r := range []*editProbeRuntime{baseline, candidate} { + if err := r.finish(); err == nil { + t.Fatal("coupled evidence loss became success") + } + if len(r.phase.EditProbe.Records) != 1 || len(r.phase.EditProbe.Records[0].Errors) == 0 { + t.Fatal("evidence loss erased the edit denominator/cause") + } + } + if baseline.phase.EditProbe.Records[0].Outcome != "diagnostic_failure" || candidate.phase.EditProbe.Records[0].Outcome != "harness_evidence_incomplete" { + t.Fatal("late partner failure left an earlier complete result or hid failed evidence") + } +} + +func TestPairedCanceledPlanRetainsEveryUnstartedEdit(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + c := pairedEditProbeFixture(t, ctx, 2, func(string, stubFrame) (any, *rpcError) { + return stubContent(map[string]any{}), nil + }) + c.window.Started = time.Now() + cancel() + if err := c.editProbes(ctx); err == nil { + t.Fatal("canceled offered plan became success") + } + for _, state := range c.states { + if len(state.phase.EditProbe.Records) != 2 { + t.Fatal("cancellation dropped planned entries") + } + for _, record := range state.phase.EditProbe.Records { + if record.Outcome != "phase_censored" || record.Ack.Value != nil || record.StartedNS != nil { + t.Fatal("unstarted edit acquired dispatch/ACK evidence or lost censoring") + } + } + } +} diff --git a/bench/load/paired_types.go b/bench/load/paired_types.go index 38ab7132b..34ccc023f 100644 --- a/bench/load/paired_types.go +++ b/bench/load/paired_types.go @@ -29,15 +29,16 @@ type measurementWindow struct { } type requestTuple struct { - Repo string `json:"repo"` - Commit string `json:"commit"` - Tool string `json:"tool"` - Role string `json:"role"` - Agent int `json:"agent"` - Step int `json:"step"` - Slot int `json:"slot"` - RequestHash string `json:"request_sha256"` - Ordinal uint64 `json:"category_ordinal"` + Repo string `json:"repo"` + Commit string `json:"commit"` + Tool string `json:"tool"` + Role string `json:"role"` + Agent int `json:"agent"` + Step int `json:"step"` + Slot int `json:"slot"` + RequestHash string `json:"request_sha256"` + Ordinal uint64 `json:"category_ordinal"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` } type pairedRecord struct { @@ -48,15 +49,16 @@ type pairedRecord struct { } type pairedSample struct { - Repo string `json:"repo"` - Commit string `json:"commit"` - RequestHash string `json:"request_sha256"` - Started time.Time `json:"started"` - Ended time.Time `json:"ended"` - Latency Metric `json:"logical_latency_ms"` - Outcome string `json:"outcome"` - Code int `json:"code"` - Error string `json:"error"` + Repo string `json:"repo"` + Commit string `json:"commit"` + RequestHash string `json:"request_sha256"` + Started time.Time `json:"started"` + Ended time.Time `json:"ended"` + Latency Metric `json:"logical_latency_ms"` + Outcome string `json:"outcome"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` + Code int `json:"code"` + Error string `json:"error"` } type pairedRatio struct { @@ -99,6 +101,9 @@ func matchedLatencyRatios(labels []string, records []pairedRecord) map[string]ma values := make(map[string][]float64) for _, record := range records { category := record.Tuple.Role + "." + record.Tuple.Tool + if record.Tuple.ExpectedOutcome != "" { + category += "." + record.Tuple.ExpectedOutcome + } row := rows[category] if row.FirstBuildCounts == nil { row.FirstBuildCounts = make(map[string]int) @@ -108,7 +113,11 @@ func matchedLatencyRatios(labels []string, records []pairedRecord) map[string]ma } a, aOK := record.Samples[labels[0]] b, bOK := record.Samples[label] - if !aOK || !bOK || a.Outcome != "success" || b.Outcome != "success" || a.RequestHash != record.Tuple.RequestHash || b.RequestHash != record.Tuple.RequestHash || a.Repo != record.Tuple.Repo || b.Repo != record.Tuple.Repo || a.Commit != record.Tuple.Commit || b.Commit != record.Tuple.Commit || a.Latency.Value == nil || b.Latency.Value == nil || *a.Latency.Value <= 0 || *b.Latency.Value <= 0 { + validOutcomes := a.Outcome == "success" && b.Outcome == "success" && record.Tuple.ExpectedOutcome == "" && a.ExpectedOutcome == "" && b.ExpectedOutcome == "" + if record.Tuple.ExpectedOutcome == "symbol_absent" { + validOutcomes = a.Outcome == "expected_refusal" && b.Outcome == "expected_refusal" && a.ExpectedOutcome == "symbol_absent" && b.ExpectedOutcome == "symbol_absent" + } + if !aOK || !bOK || !validOutcomes || a.RequestHash != record.Tuple.RequestHash || b.RequestHash != record.Tuple.RequestHash || a.Repo != record.Tuple.Repo || b.Repo != record.Tuple.Repo || a.Commit != record.Tuple.Commit || b.Commit != record.Tuple.Commit || a.Latency.Value == nil || b.Latency.Value == nil || *a.Latency.Value <= 0 || *b.Latency.Value <= 0 { row.Excluded++ } else { row.Matched++ diff --git a/bench/load/paired_types_test.go b/bench/load/paired_types_test.go index d72ba8f4a..2510ee870 100644 --- a/bench/load/paired_types_test.go +++ b/bench/load/paired_types_test.go @@ -107,3 +107,31 @@ func TestMatchedRatiosExcludeZeroClockSamplesOnEitherBuild(t *testing.T) { } } } + +func TestMatchedAbsenceRatiosRemainDistinctFromSuccessAndGenericErrors(t *testing.T) { + var records []pairedRecord + for _, variant := range []string{"valid", "generic_error", "ordinary_success", "missing_expectation", "censored"} { + a := pairedSample{Outcome: "expected_refusal", ExpectedOutcome: "symbol_absent", RequestHash: "same", Latency: measured(10, "ms")} + b := a + b.Latency = measured(11, "ms") + switch variant { + case "generic_error": + b.Outcome = "error" + case "ordinary_success": + b.Outcome = "success" + case "missing_expectation": + b.ExpectedOutcome = "" + case "censored": + b.Outcome = "phase_censored" + } + records = append(records, pairedRecord{Tuple: requestTuple{Tool: "read.source", Role: "probe", RequestHash: "same", ExpectedOutcome: "symbol_absent"}, Order: []string{"main", "candidate"}, Samples: map[string]pairedSample{"main": a, "candidate": b}}) + } + rows := matchedLatencyRatios([]string{"main", "candidate"}, records)["candidate/main"] + got := rows["probe.read.source.symbol_absent"] + if got.Matched != 1 || got.Excluded != 4 || got.Median.Value == nil || *got.Median.Value != 1.1 { + t.Fatalf("absence ratio lost explicit outcomes: %+v", got) + } + if _, ok := rows["probe.read.source"]; ok { + t.Fatal("expected absence was folded into ordinary successful RPC ratios") + } +} diff --git a/bench/load/plan.go b/bench/load/plan.go index dc1f31f1d..d17afa478 100644 --- a/bench/load/plan.go +++ b/bench/load/plan.go @@ -54,31 +54,33 @@ func busyBackoff(retry int) time.Duration { } type request struct { - Tool string `json:"tool"` - Operation string `json:"operation"` - Arguments map[string]any `json:"arguments"` + Tool string `json:"tool"` + Operation string `json:"operation"` + Arguments map[string]any `json:"arguments"` + ExpectedAbsentSymbol string `json:"expected_absent_symbol,omitempty"` } // Workload is the complete normalized offered load. Binary paths, private // clone paths and revisions belong to the run identity, never the plan hash. type Workload struct { - PRSmokeDrain *PRSmokeDrain `json:"pr_smoke_drain,omitempty"` - Scenario string `json:"scenario"` - Agents int `json:"agents"` - EditAgents int `json:"edit_agents"` - LinkedWorktrees int `json:"linked_worktrees"` - DurationSec float64 `json:"duration_seconds"` - DeadlineSec float64 `json:"product_deadline_seconds"` - Seed int64 `json:"seed"` - Profile string `json:"profile"` - Operations []request `json:"operations"` - CPUProfileSec float64 `json:"cpu_profile_seconds"` - WarmTimeoutSec float64 `json:"warm_timeout_seconds"` - StopTimeoutSec float64 `json:"stop_timeout_seconds"` - KeepState bool `json:"keep_state"` - SessionModel string `json:"session_model"` - SweepRepetitions int `json:"sweep_repetitions"` - SweepBudgetSec float64 `json:"sweep_budget_seconds"` + PRSmokeDrain *PRSmokeDrain `json:"pr_smoke_drain,omitempty"` + Scenario string `json:"scenario"` + Agents int `json:"agents"` + EditAgents int `json:"edit_agents"` + LinkedWorktrees int `json:"linked_worktrees"` + DurationSec float64 `json:"duration_seconds"` + DeadlineSec float64 `json:"product_deadline_seconds"` + Seed int64 `json:"seed"` + Profile string `json:"profile"` + Operations []request `json:"operations"` + CPUProfileSec float64 `json:"cpu_profile_seconds"` + WarmTimeoutSec float64 `json:"warm_timeout_seconds"` + StopTimeoutSec float64 `json:"stop_timeout_seconds"` + KeepState bool `json:"keep_state"` + SessionModel string `json:"session_model"` + SweepRepetitions int `json:"sweep_repetitions"` + SweepBudgetSec float64 `json:"sweep_budget_seconds"` + EditProbe *editProbePlan `json:"edit_probe,omitempty"` } func (r request) name() string { @@ -186,6 +188,10 @@ func validateAgentCounts(scenario string, readers, editors int) error { if readers != 0 || editors != 0 { return errors.New("idle-stop requires exactly zero reading/editing agents") } + case editProbeScenario: + if readers > editProbeMaxReaders || editors < 1 || editors > editProbeMaxEditors { + return fmt.Errorf("edit-and-probe has 1..%d fixed editors and 0..%d pure readers", editProbeMaxEditors, editProbeMaxReaders) + } case "edit", "many-edit": if editors < 1 { return errors.New("edit scenarios require at least one combined edit/readback agent") @@ -198,6 +204,7 @@ func validateAgentCounts(scenario string, readers, editors int) error { if readers < 1 { return errors.New("zero pure readers are supported only for edit/many-edit") } + case startupScenario: // probes only; reader and editor counts are unused default: return fmt.Errorf("unknown scenario %q", scenario) } diff --git a/bench/load/report.go b/bench/load/report.go index a095497c7..a02c5f307 100644 --- a/bench/load/report.go +++ b/bench/load/report.go @@ -48,15 +48,25 @@ func distribution(values []float64, unit string) Distribution { } type Corpus struct { - Label string `json:"label"` - Source string `json:"source"` - Commit string `json:"commit"` - File string `json:"file"` - Symbol string `json:"symbol"` - Query string `json:"query"` - EditTemplate string `json:"edit_template"` - EditAnchor string `json:"edit_anchor"` - ProposedSignature string `json:"proposed_signature"` + Label string `json:"label"` + Source string `json:"source"` + Commit string `json:"commit"` + File string `json:"file"` + Symbol string `json:"symbol"` + Query string `json:"query"` + EditTemplate string `json:"edit_template"` + EditAnchor string `json:"edit_anchor"` + EditTool string `json:"edit_tool,omitempty"` + EditOutput string `json:"edit_output,omitempty"` + EditExpect string `json:"edit_expect,omitempty"` + EditSender string `json:"edit_sender,omitempty"` + EditKind string `json:"edit_kind,omitempty"` + EditCallee string `json:"edit_callee,omitempty"` + EditBeforeTemplate string `json:"edit_before_template,omitempty"` + EditDestination string `json:"edit_destination,omitempty"` + EditDestinationContent string `json:"edit_destination_content,omitempty"` + EditInitialContent string `json:"edit_initial_content,omitempty"` + ProposedSignature string `json:"proposed_signature"` } type Host struct { @@ -73,29 +83,33 @@ type Host struct { // Every attempt is retained, including busy refusals and phase-censored calls. // Workload and readiness/visibility probes have distinct Role denominators. type Attempt struct { - Tool string `json:"tool"` - Role string `json:"role"` - Request string `json:"request_sha256"` - Agent int `json:"agent"` - Step int `json:"step"` - Slot int `json:"slot"` - Retry int `json:"retry"` - Started time.Time `json:"started"` - DurationMS float64 `json:"duration_ms"` - LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time - Code int `json:"code"` - Error string `json:"error"` - Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored + Tool string `json:"tool"` + Role string `json:"role"` + Request string `json:"request_sha256"` + Agent int `json:"agent"` + Step int `json:"step"` + Slot int `json:"slot"` + Retry int `json:"retry"` + Started time.Time `json:"started"` + DurationMS float64 `json:"duration_ms"` + LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time + Code int `json:"code"` + Error string `json:"error"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` + Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored } type ToolMetrics struct { - Latency Distribution `json:"latency_ms"` - SuccessPerSec Metric `json:"successful_calls_per_second"` - Successes int `json:"successes"` - BusyRefusals int `json:"busy_refusals"` - Errors map[string]int `json:"errors"` - PhaseCensored int `json:"phase_censored"` - ProductExpired int `json:"product_expired"` + Latency Distribution `json:"latency_ms"` + SuccessPerSec Metric `json:"successful_calls_per_second"` + Successes int `json:"successes"` + ExpectedRefusals int `json:"expected_refusals,omitempty"` + BusyRefusals int `json:"busy_refusals"` + RetryableRefusals int `json:"retryable_refusals,omitempty"` + HarnessIncomplete int `json:"harness_incomplete,omitempty"` + Errors map[string]int `json:"errors"` + PhaseCensored int `json:"phase_censored"` + ProductExpired int `json:"product_expired"` } type SweepEntry struct { @@ -116,24 +130,26 @@ type EditObservation struct { } type Phase struct { - SessionModel string `json:"session_model"` - Operations []request `json:"operations"` - Name string `json:"name"` - Started time.Time `json:"started"` - DurationSec float64 `json:"duration_seconds"` - Gate map[string]json.RawMessage `json:"gate"` - Tools map[string]ToolMetrics `json:"tools"` - Probes map[string]ToolMetrics `json:"probes"` - Attempts []Attempt `json:"attempts"` - Edits []EditObservation `json:"edits"` - EditLatency Distribution `json:"edit_latency_ms"` - Visibility Distribution `json:"visible_latency_ms"` - NeverSeen int `json:"never_visible"` - Censored int `json:"visibility_censored"` - Metrics map[string]Metric `json:"metrics"` - Status []json.RawMessage `json:"daemon_status"` - Stats []json.RawMessage `json:"graph_stats"` - Errors []string `json:"errors"` + EditProbe *editProbeReport `json:"edit_probe,omitempty"` + editProbeRuntime *editProbeRuntime + SessionModel string `json:"session_model"` + Operations []request `json:"operations"` + Name string `json:"name"` + Started time.Time `json:"started"` + DurationSec float64 `json:"duration_seconds"` + Gate map[string]json.RawMessage `json:"gate"` + Tools map[string]ToolMetrics `json:"tools"` + Probes map[string]ToolMetrics `json:"probes"` + Attempts []Attempt `json:"attempts"` + Edits []EditObservation `json:"edits"` + EditLatency Distribution `json:"edit_latency_ms"` + Visibility Distribution `json:"visible_latency_ms"` + NeverSeen int `json:"never_visible"` + Censored int `json:"visibility_censored"` + Metrics map[string]Metric `json:"metrics"` + Status []json.RawMessage `json:"daemon_status"` + Stats []json.RawMessage `json:"graph_stats"` + Errors []string `json:"errors"` } type Report struct { @@ -181,8 +197,14 @@ func summarize(attempts []Attempt, seconds float64, role string) map[string]Tool case "success": m.Successes++ latencies[a.Tool] = append(latencies[a.Tool], a.LogicalMS) + case "expected_refusal": + m.ExpectedRefusals++ case "busy": m.BusyRefusals++ + case "retryable": + m.RetryableRefusals++ + case "harness_incomplete": + m.HarnessIncomplete++ case "phase_censored": m.PhaseCensored++ case "product_deadline": diff --git a/bench/load/run.go b/bench/load/run.go index bf3bc8ba6..43d7262aa 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -14,30 +14,34 @@ import ( ) type runOptions struct { - Binary string - Manifest string - Scratch string - Output string - GitSHA string - Profile string - Host string - Scenario string - Agents int - Editors int - Worktrees int - Seed int64 - Duration time.Duration - PRSmokeDrain time.Duration - Deadline time.Duration - WarmTimeout time.Duration - StopTimeout time.Duration - CPUProfile time.Duration - KeepState bool - SweepRepetitions int - SweepBudget time.Duration - pairedDriver func(context.Context, *privateRun, []*socketClient, []*socketClient, *Phase) error - smokeState *privateRun // only the fixed smoke entry lends its freshly owned store - smokeMarkers []string + Binary string + Manifest string + Scratch string + Output string + GitSHA string + Profile string + Host string + Scenario string + Agents int + Editors int + Worktrees int + Seed int64 + Duration time.Duration + PRSmokeDrain time.Duration + Deadline time.Duration + WarmTimeout time.Duration + StopTimeout time.Duration + CPUProfile time.Duration + KeepState bool + SweepRepetitions int + SweepBudget time.Duration + EditCount int + EditProbeRole string + editProbeBaseline bool + Startup startupOptions + pairedDriver func(context.Context, *privateRun, []*socketClient, []*socketClient, *Phase) error + smokeState *privateRun // only the fixed smoke entry lends its freshly owned store + smokeMarkers []string } func parseRun(args []string) (runOptions, error) { @@ -50,6 +54,7 @@ func parseRun(args []string) (runOptions, error) { if f.NArg() != 0 { return o, errors.New("unexpected positional run arguments") } + o.editProbeBaseline = o.EditProbeRole == "baseline" return o, validateRunOptions(o) } @@ -61,8 +66,8 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.StringVar(&o.GitSHA, "git-sha", "", "full binary revision if absent from Go build metadata") f.StringVar(&o.Profile, "profile", "agent", "agent (default) or explicit saturation") f.StringVar(&o.Host, "host", "native", "native, reference, or floor (validated, never provisioned)") - f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, cold, restart, sweep, idle-stop, all") - f.IntVar(&o.Agents, "agents", 4, "pure reading agents (may be 0 only for edit/many-edit)") + f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, cold, restart, sweep, idle-stop, edit-and-probe, all") + f.IntVar(&o.Agents, "agents", 4, "pure reading agents (may be 0 for edit, many-edit, or edit-and-probe; idle-stop requires 0)") f.IntVar(&o.Editors, "edit-agents", 4, "combined editing/readback/reading agents in edit scenarios, in addition to pure readers") f.IntVar(&o.Worktrees, "worktrees", 4, "linked worktrees in addition to primary") f.Int64Var(&o.Seed, "seed", 1, "reproducible per-agent request/think plan seed") @@ -75,6 +80,9 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.BoolVar(&o.KeepState, "keep-state", false, "retain private corpus/store for native diagnostics only (default removes completed scenario state)") f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") + bindStartupFlags(f, &o.Startup) + f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") + f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } func validateRunOptions(o runOptions) error { @@ -87,12 +95,21 @@ func validateRunOptions(o runOptions) error { if err := validateAgentCounts(o.Scenario, o.Agents, o.Editors); err != nil { return err } + if err := o.Startup.validate(o.Scenario); err != nil { + return err + } if o.Scenario == "idle-stop" && (o.Worktrees != 0 || o.CPUProfile != 0 || o.Profile != "agent") { return errors.New("idle-stop requires an agent-profile standalone run with no worktrees or CPU profiling") } if _, err := parseProfile(o.Profile); err != nil { return err } + if o.Scenario == editProbeScenario && (o.Profile != "agent" || o.Agents > editProbeMaxReaders || o.Editors < 1 || o.Editors > editProbeMaxEditors || o.Worktrees != 0 || o.CPUProfile != 0 || o.EditCount != 1 && o.EditCount != 100 || o.EditCount == 1 && o.Editors != 1) { + return fmt.Errorf("edit-and-probe requires --profile agent --agents 0..%d --edit-agents 1..%d on one repository --worktrees 0, no CPU profile, and --edit-count 100 (or an explicit 1-edit, 1-editor diagnostic)", editProbeMaxReaders, editProbeMaxEditors) + } + if o.EditProbeRole != "" && (o.Scenario != editProbeScenario || o.EditProbeRole != "candidate" && o.EditProbeRole != "baseline") { + return errors.New("--edit-probe-role is candidate or baseline, for edit-and-probe only") + } if o.SweepRepetitions < 1 || o.SweepBudget <= 0 { return errors.New("sweep repetitions and duration must be positive") } @@ -103,7 +120,7 @@ func validateRunOptions(o runOptions) error { return errors.New("--cpu-profile requires a whole number of seconds") } activeAgents := o.Agents - if o.Scenario == "edit" || o.Scenario == "many-edit" || o.Scenario == "all" { + if o.Scenario == "edit" || o.Scenario == "many-edit" || o.Scenario == "all" || o.Scenario == editProbeScenario { activeAgents += o.Editors } activeWorktrees := 1 @@ -124,6 +141,12 @@ func selectedScenarios(o runOptions) ([]string, error) { return []string{"idle-stop"}, nil } all := []string{"read", "edit", "many", "many-edit", "worktrees", "cold", "restart", "sweep"} + if o.Scenario == startupScenario { + return []string{startupScenario}, nil + } + if o.Scenario == editProbeScenario { + return []string{editProbeScenario}, nil + } if o.Scenario == "all" { if o.Host == "floor" { return []string{"read", "edit", "worktrees", "cold", "restart", "sweep"}, nil @@ -235,6 +258,15 @@ func newRunReport(o runOptions, corpora []Corpus, sha string, host Host, outputR } report.Workload.Operations = append(report.Workload.Operations, ops...) } + if o.Scenario == editProbeScenario { + plan, err := planEditProbe(corpora[0], o) + if err != nil { + return report, err + } + report.Workload.EditProbe = &plan + report.Workload.Operations = editProbeOperations(plan, corpora[0]) + report.Workload.SessionModel = plan.SessionModel + } report.Workload.Scenario, report.Workload.Agents, report.Workload.EditAgents, report.Workload.LinkedWorktrees = o.Scenario, o.Agents, o.Editors, o.Worktrees report.Workload.DurationSec, report.Workload.DeadlineSec, report.Workload.Seed, report.Workload.Profile = o.Duration.Seconds(), o.Deadline.Seconds(), o.Seed, o.Profile if o.PRSmokeDrain > 0 { @@ -249,6 +281,18 @@ func newRunReport(o runOptions, corpora []Corpus, sha string, host Host, outputR if o.Scenario == "all" { report.Workload.SessionModel = "per_agent_load_and_per_operation_isolated_sweep" } + if report.Workload.EditProbe != nil { + report.Workload.SessionModel = report.Workload.EditProbe.SessionModel + builds := 1 + if o.pairedDriver != nil { + builds = 2 + } + budget, err := report.Workload.EditProbe.budget(o.Deadline, builds) + if err != nil { + return report, err + } + report.Workload.DurationSec = budget.Seconds() + } var err error report.RequestPlan, err = hashJSON(report.Workload) return report, err diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 1820b1bbb..eedaa9616 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -14,6 +14,9 @@ import ( ) func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corpus, group, outputRoot string) (phase Phase, sweep []SweepEntry, artifacts map[string]string, crashes map[string]int, version string, resultErr error) { + if scenario == startupScenario { + return runStartupPhase(ctx, o, corpora, outputRoot) + } phase = Phase{Name: scenario, SessionModel: "per_agent", Started: time.Now().UTC(), Gate: make(map[string]json.RawMessage), Metrics: make(map[string]Metric), EditLatency: distribution(nil, "ms"), Visibility: distribution(nil, "ms")} artifacts, crashes = make(map[string]string), make(map[string]int) var scenarioGroupStart map[string]Metric @@ -47,7 +50,8 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp if phase.NeverSeen > 0 { phase.Errors = append(phase.Errors, fmt.Sprintf("%d acknowledged edits never visible before product deadline", phase.NeverSeen)) } - if scenario != "cold" && scenario != "restart" && scenario != "idle-stop" && len(phase.Tools) == 0 { + nativeWork := scenario == editProbeScenario && editProbeNativeWorkObserved(phase.EditProbe) + if scenario != "cold" && scenario != "restart" && scenario != "idle-stop" && len(phase.Tools) == 0 && !nativeWork { phase.Errors = append(phase.Errors, "no workload calls observed before phase end") } if resultErr != nil { @@ -64,7 +68,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } selected = corpora } - editing := scenario == "edit" || scenario == "many-edit" + editing := scenario == "edit" || scenario == "many-edit" || scenario == editProbeScenario for _, corpus := range selected { operations, planErr := readingPlan(corpus) if planErr != nil { @@ -82,6 +86,15 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } } } + if scenario == editProbeScenario { + plan, err := planEditProbe(selected[0], o) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + phase.Operations = editProbeOperations(plan, selected[0]) + phase.EditProbe = newEditProbeReport(plan, selected[0], !o.editProbeBaseline) + phase.SessionModel = plan.SessionModel + } p := o.smokeState var err error if p == nil { @@ -306,6 +319,18 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp phase.Metrics["sweep_operation_sessions"] = measured(float64(len(sweepClients)), "sessions") } phase.Metrics["mcp_sessions"] = measured(float64(len(clients)), "sessions") + if scenario == editProbeScenario { + runtime, probers, prepareErr := prepareEditProbe(warmCtx, o, p, clients[:o.Agents], clients[o.Agents:o.Agents+o.Editors], &phase) + clients = append(clients, probers...) + if runtime != nil { + defer func() { resultErr = errors.Join(resultErr, runtime.close()) }() + } + if prepareErr != nil { + return phase, sweep, artifacts, crashes, version, prepareErr + } + phase.Metrics["mcp_sessions"] = measured(float64(len(clients)), "sessions") + artifacts["edit_probe_raw"] = filepath.Join(p.Root, "edit-probe") + } captureReceipt(ctx, p, clients[0], &phase) sampler := newProcessSampler() if scenario == "idle-stop" { @@ -318,11 +343,20 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp if err != nil { resultErr = errors.Join(resultErr, err) } + if phase.editProbeRuntime != nil { + if ctx.Err() != nil { + resultErr = errors.Join(resultErr, errors.New("restore batch unavailable after canceled workload")) + } else { + restoreCtx, cancel := context.WithTimeout(ctx, o.Deadline) + resultErr = errors.Join(resultErr, phase.editProbeRuntime.restore(restoreCtx)) + cancel() + } + } if scenario == "sweep" { finalizeSweepInventory(sweep, phase.Attempts, o.SweepRepetitions, &phase) } if scenario != "idle-stop" { - captureReceipt(ctx, p, clients[0], &phase) + captureReceipt(ctx, p, editProbeReceiptClient(&phase, clients), &phase) } for i, client := range clients { client.mu.Lock() @@ -361,7 +395,9 @@ func measureStandalone(ctx context.Context, o runOptions, p *privateRun, clients } } var err error - if phase.Name == "sweep" { + if phase.Name == editProbeScenario { + err = phase.editProbeRuntime.run(ctx) + } else if phase.Name == "sweep" { sweepCtx, cancelSweep := context.WithTimeout(ctx, o.SweepBudget) defer cancelSweep() err = runSweep(sweepCtx, o, sweepClients, p.Repos[0], sweepRequests, sweep, phase, waitContext) @@ -378,7 +414,7 @@ func measureStandalone(ctx context.Context, o runOptions, p *privateRun, clients func dispatchArgs(r request, repo privateRepo) map[string]any { args := copyJSON(r.Arguments) - args["output"] = map[string]any{"format": "json", "max_bytes": 0} + args["output"] = editProbeDispatchOutput(r) if target, ok := args["target"].(map[string]any); ok { for _, key := range []string{"file", "symbol"} { if value, ok := target[key].(string); ok { @@ -393,6 +429,23 @@ func dispatchArgs(r request, repo privateRepo) map[string]any { } } } + if r.name() == "refactor.move" { + if destination, ok := args["destination"].(string); ok { + args["destination"] = repo.Prefix + "/" + destination + } + } + if r.name() == "edit.batch" { + if changes, ok := args["changes"].([]any); ok { + for _, item := range changes { + if change, ok := item.(map[string]any); ok && change["op"] == "edit_file" { + if path, ok := change["path"].(string); ok { + change["path"] = repo.Prefix + "/" + path + } + } + } + } + } + if r.name() == "change.verify" { if source, ok := args["source"].(map[string]any); ok { if changes, ok := source["changes"].([]any); ok { diff --git a/bench/load/schema.go b/bench/load/schema.go index 62ee77043..5ef28f239 100644 --- a/bench/load/schema.go +++ b/bench/load/schema.go @@ -45,10 +45,11 @@ func schemaFor(t reflect.Type) map[string]any { for i := range t.NumField() { field := t.Field(i) name := strings.Split(field.Tag.Get("json"), ",")[0] + if field.PkgPath != "" || name == "-" { + continue + } properties[name] = schemaFor(field.Type) - // Only this new optional Workload extension is omitted. Historical - // required lists, including existing omitempty fields, stay intact. - if t != reflect.TypeFor[Workload]() || field.Name != "PRSmokeDrain" { + if !strings.Contains(field.Tag.Get("json"), ",omitempty") { required = append(required, name) } } diff --git a/bench/load/selftest_test.go b/bench/load/selftest_test.go index 9ebf11310..b7e0154a4 100644 --- a/bench/load/selftest_test.go +++ b/bench/load/selftest_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "os" + "os/exec" "path/filepath" "strings" "sync" @@ -28,6 +29,17 @@ func fixtureDaemon(mode string) int { fmt.Fprintln(os.Stderr, "panic: synthetic daemon crash") return 2 } + // The startup probe runs the CLI and the hook through the same binary; + // answer them as against a ready daemon instead of starting another one. + for _, arg := range os.Args[1:] { + switch arg { + case "call": + fmt.Println(`{"results":[]}`) + return 0 + case "hook": + return 0 + } + } data, err := os.ReadFile(filepath.Join(os.Getenv("XDG_CONFIG_HOME"), "gortex", "config.yaml")) if err != nil { return 3 @@ -41,6 +53,14 @@ func fixtureDaemon(mode string) int { if json.Unmarshal(data, &config) != nil || len(config.Repos) == 0 { return 4 } + var nativeCorpus Corpus + var nativeReplies func(stubFrame) (any, *rpcError) + if mode == "native" { + if json.Unmarshal([]byte(os.Getenv("LOAD_NATIVE_CORPUS")), &nativeCorpus) != nil { + return 6 + } + nativeReplies = editProbeDiskReplies(nativeCorpus, config.Repos[0].Path, func(err error) { panic(err) }) + } listener, err := net.Listen("unix", os.Getenv("GORTEX_DAEMON_SOCKET")) if err != nil { return 5 @@ -80,7 +100,7 @@ func fixtureDaemon(mode string) int { } result = map[string]any{"ready": true, "enriched": true, "tracked_repos": repos, "loaded_roots": roots} case "status": - result = map[string]any{"ready": true, "enriched": true, "views": map[string]any{}, "storage": map[string]any{"wal_pending_frames": 0}} + result = map[string]any{"ready": true, "enriched": true, "views": map[string]any{"coordinators": 1}, "storage": map[string]any{"wal_pending_frames": 0}} case "shutdown": result = map[string]any{"stopping": true} default: @@ -109,6 +129,16 @@ func fixtureDaemon(mode string) int { case "tools/call": if frame.Params["name"] == "analyze" { result = stubContent(map[string]any{"hotspots": []any{}, "total": 0}) + } else if frame.Params["name"] == "session" { + result = stubContent(map[string]any{}) + } else if nativeReplies != nil { + args, _ := frame.Params["arguments"].(map[string]any) + target, _ := args["target"].(map[string]any) + if frame.Params["name"] == "read" && args["operation"] == "source" && target["symbol"] == "fixture/"+nativeCorpus.Symbol { + result = stubContent(map[string]any{"id": "fixture/" + nativeCorpus.Symbol, "source": "func Example() int { return 1 }"}) + } else { + result, _ = nativeReplies(frame) + } } else { result = stubContent(map[string]any{"id": "sample/sample.go::Example", "source": "func Example() {}"}) } @@ -176,3 +206,77 @@ func TestFailedPrivateDaemonStillCollectsCrashEvidence(t *testing.T) { t.Fatal(err) } } + +// Exercise runPhase's final validation after a real native save and socket +// probes. The paired driver only skips the Poisson delay in this diagnostic. +func TestNativeEditProbePhaseRequiresWriteAndProbeEvidence(t *testing.T) { + for _, mode := range []string{"complete", "empty", "missing_write", "missing_probes"} { + t.Run(mode, func(t *testing.T) { + c := editProbeEditorSaveCorpus(t, 1) + pin := gitFixture(t) + c.Source, c.Label = pin.Source, "fixture" + if err := os.WriteFile(filepath.Join(c.Source, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", c.File}, {"commit", "--quiet", "-m", "native fixture"}, {"rev-parse", "HEAD"}} { + cmd := exec.Command("git", append([]string{"-C", c.Source, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "core.hooksPath=" + os.DevNull}, args...)...) + data, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v %s", args, err, data) + } + if args[0] == "rev-parse" { + c.Commit = strings.TrimSpace(string(data)) + } + } + corpusJSON, err := json.Marshal(c) + if err != nil { + t.Fatal(err) + } + t.Setenv("LOAD_DAEMON_FIXTURE", "native") + t.Setenv("LOAD_NATIVE_CORPUS", string(corpusJSON)) + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + o := runOptions{Binary: binary, Profile: "agent", Host: "native", Editors: 1, EditCount: 1, Seed: 42, Duration: time.Second, Deadline: 2 * time.Second, WarmTimeout: 10 * time.Second, StopTimeout: 5 * time.Second} + o.pairedDriver = func(ctx context.Context, p *privateRun, clients, sweepClients []*socketClient, phase *Phase) error { + r := phase.editProbeRuntime + if mode == "empty" { + r.finalized = true + return nil + } + r.origin = time.Now() + step := r.plan.Steps[0] + r.activate(ctx, step) + awaitEditProbeFirstProbe(t, r.states[0]) + r.doEdit(ctx, step) + if err := r.finish(); err != nil { + return err + } + if mode == "missing_write" { + phase.EditProbe.Records[0].NativeWrite.EndNS = nil + } + if mode == "missing_probes" { + phase.EditProbe.Records[0].FirstCorrect["source"] = unavailable("ms", "not observed") + } + return nil + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + phase, _, artifacts, _, _, err := runPhase(ctx, o, editProbeScenario, []Corpus{c}, "", t.TempDir()) + if data, _ := os.ReadFile(artifacts["daemon-1.stderr.log"]); len(data) != 0 { + t.Logf("native fixture stderr: %s", data) + } + if len(phase.Tools) != 0 { + t.Fatalf("native save invented MCP workload calls: %+v", phase.Tools) + } + if mode == "complete" { + if err != nil { + t.Fatalf("native save failed final phase validation: %v", err) + } + } else if err == nil || !strings.Contains(strings.Join(phase.Errors, "; "), "no workload calls observed before phase end") { + t.Fatalf("%s native evidence accepted: %v", mode, err) + } + }) + } +} diff --git a/bench/load/startup_probe.go b/bench/load/startup_probe.go new file mode 100644 index 000000000..37bc3f4fc --- /dev/null +++ b/bench/load/startup_probe.go @@ -0,0 +1,345 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +// The startup scenario observes repository admission from before the +// daemon's socket exists until every configured repository is ready and +// enriched, on an initial boot and on each optional restart. The probes keep +// sampling across the gap between boots, so a restart is seen exactly as an +// agent sees it. A seeded store makes the initial boot an upgrade: its +// migration runs before the socket listens. + +const ( + startupScenario = "startup" + startupRosterWorkspace = "probe-roster" + startupWatchInterval = 100 * time.Millisecond + startupCoordinatorWindow = 2 * time.Second +) + +type startupOptions struct { + Restarts int + SeedStore string +} + +func bindStartupFlags(f *flag.FlagSet, o *startupOptions) { + f.IntVar(&o.Restarts, "restarts", 0, "startup: warm restarts after the initial boot, each probed from before its socket exists") + f.StringVar(&o.SeedStore, "seed-store", "", "startup: store file copied into the private run before the initial boot, e.g. one written by an older schema; the source is only read") +} + +func (o startupOptions) validate(scenario string) error { + if o.Restarts < 0 { + return errors.New("--restarts must be nonnegative") + } + if scenario != startupScenario && (o.Restarts != 0 || o.SeedStore != "") { + return errors.New("--restarts and --seed-store apply to the startup scenario only") + } + return nil +} + +// bootTiming holds what the readiness watcher observed for one daemon start. +// Every instant is an observed upper bound at the watcher's poll interval. +type bootTiming struct { + Boot int `json:"boot"` + Started time.Time `json:"started"` + Stopping time.Time `json:"stopping"` + SocketAt time.Time `json:"socket_at"` + ReadyAt map[string]time.Time `json:"ready_at"` + EnrichedAt time.Time `json:"enriched_at"` + // Coordinators is the largest checkout coordinator count the status + // census read in the 2 s after the primary registered. + Coordinators *int `json:"coordinators_within_2s"` + CensusReads int `json:"census_reads"` + CensusErrors []string `json:"census_errors"` + Views json.RawMessage `json:"views"` +} + +func runStartupPhase(ctx context.Context, o runOptions, corpora []Corpus, outputRoot string) (phase Phase, _ []SweepEntry, artifacts map[string]string, crashes map[string]int, version string, resultErr error) { + phase = Phase{Name: startupScenario, SessionModel: "fresh_session_per_probe_tick", Started: time.Now().UTC(), Gate: make(map[string]json.RawMessage), Metrics: make(map[string]Metric), EditLatency: distribution(nil, "ms"), Visibility: distribution(nil, "ms")} + artifacts, crashes = make(map[string]string), make(map[string]int) + phase.Operations = startupOperations(corpora[0]) + defer func() { + phase.DurationSec = time.Since(phase.Started).Seconds() + if resultErr != nil { + phase.Errors = append(phase.Errors, resultErr.Error()) + } else if len(phase.Errors) > 0 { + resultErr = errors.New(strings.Join(phase.Errors, "; ")) + } + }() + p, err := newPrivateRun(ctx, outputRoot, o.Binary, corpora) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + artifacts["private_root"] = p.Root + defer func() { + stopCtx, cancel := context.WithTimeout(context.Background(), o.StopTimeout) + _, forced, err := p.stop(stopCtx) + cancel() + if forced { + err = errors.Join(err, errors.New("final stop was forced")) + } + logArtifacts, logCrashes, logErr := collectLogs(p.Root) + for name, path := range logArtifacts { + artifacts[name] = path + } + for name, count := range logCrashes { + crashes[name] += count + } + resultErr = errors.Join(resultErr, err, logErr, os.RemoveAll(p.SocketDir)) + if !o.KeepState { + resultErr = errors.Join(resultErr, p.removeState()) + } + }() + linked, err := addLinkedCheckouts(ctx, p, o.Worktrees) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + routerDir, err := writeRouterRoster(p) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + configured, err := p.writeBootRepos(false) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + if o.Startup.SeedStore != "" { + if err := copySeedStore(o.Startup.SeedStore, p.Store); err != nil { + return phase, nil, artifacts, crashes, "", err + } + } + cwds := []probeCWD{{Role: "primary", Dir: p.Repos[0].Path, File: p.Repos[0].File}} + cwds = append(cwds, linked[:min(1, len(linked))]...) + cwds = append(cwds, probeCWD{Role: "router", Dir: routerDir, File: p.Repos[0].File}) + logPath := filepath.Join(p.Root, "startup-probe.jsonl") + probe, err := newStartupProbe(p, cwds, corpora[0].Query, logPath) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + artifacts["startup_probe_log"] = logPath + probeCtx, stopProbe := context.WithCancel(ctx) + probe.start(probeCtx) + var boots []bootTiming + for index := 0; index <= o.Startup.Restarts; index++ { + if index > 0 { + probe.stopping() + boots[index-1].Stopping = time.Now() + stopCtx, cancel := context.WithTimeout(ctx, o.StopTimeout) + _, forced, err := p.stop(stopCtx) + cancel() + if err != nil || forced { + resultErr = fmt.Errorf("restart %d: stop failed or was forced: %v", index, err) + break + } + } + timing, err := runStartupBoot(ctx, o, p, probe, configured, index) + boots = append(boots, timing) + if err != nil { + resultErr = err + break + } + } + if resultErr == nil { + receiptCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + client, err := openMCP(receiptCtx, p.Socket, p.Repos[0].Path) + if err == nil { + captureReceipt(receiptCtx, p, client, &phase) + err = client.Close() + } + cancel() + resultErr = err + } + if len(boots) > 0 && boots[len(boots)-1].Stopping.IsZero() { + boots[len(boots)-1].Stopping = time.Now() + } + stopProbe() + samples, probeErr := probe.finish() + resultErr = errors.Join(resultErr, probeErr) + summary := summarizeStartup(samples, boots, p.Repos[0].Corpus.Label, len(linked)) + summary.CLITTY = probe.ttyStatus + for name, metric := range summary.metrics() { + phase.Metrics[name] = metric + } + if data, err := json.Marshal(summary); err == nil { + phase.Gate["startup_probe"] = data + } else { + resultErr = errors.Join(resultErr, err) + } + return phase, nil, artifacts, crashes, probe.daemonVersion(), resultErr +} + +// runStartupBoot starts the daemon and follows it until every configured +// repository is ready, the daemon reports enrichment, and the measured tail +// (--duration) has passed, so the next restart begins from a warm store. +func runStartupBoot(ctx context.Context, o runOptions, p *privateRun, probe *startupProbe, configured []privateRepo, index int) (bootTiming, error) { + timing := bootTiming{Boot: index, ReadyAt: make(map[string]time.Time)} + timing.Started = time.Now() + probe.beginBoot(index) + if err := p.start(); err != nil { + return timing, err + } + primary := configured[0].Corpus.Label + for { + probeCtx, cancel := context.WithTimeout(ctx, time.Second) + data, err := control(probeCtx, p.Socket, "probe", nil) + cancel() + now := time.Now() // after the answer: an upper bound, never early + if err == nil { + if timing.SocketAt.IsZero() { + timing.SocketAt = now + probe.socketUp(index) + } + var state readiness + if err := json.Unmarshal(data, &state); err != nil { + return timing, err + } + if err := observeReadiness(&timing, state, configured, now); err != nil { + return timing, err + } + } + ready, registered := timing.ReadyAt[primary] + if registered && now.Sub(ready) <= startupCoordinatorWindow { + readCoordinators(ctx, p, &timing) + } + if !timing.EnrichedAt.IsZero() && now.Sub(timing.EnrichedAt) >= o.Duration && now.Sub(ready) > startupCoordinatorWindow { + return timing, nil + } + select { + case <-p.done: + return timing, fmt.Errorf("boot %d: daemon exited: %v", index, p.exitErr) + default: + } + if now.Sub(timing.Started) > o.WarmTimeout { + return timing, fmt.Errorf("boot %d: configured repositories not ready and enriched within %s", index, o.WarmTimeout) + } + if err := waitContext(ctx, startupWatchInterval); err != nil { + return timing, err + } + } +} + +// observeReadiness stamps each configured repository the first time the probe +// lists it as ready. Daemons that predate per-repository state list a +// registered repository with its prefix and no state. +func observeReadiness(timing *bootTiming, state readiness, configured []privateRepo, now time.Time) error { + for _, repo := range configured { + if _, seen := timing.ReadyAt[repo.Corpus.Label]; seen { + continue + } + for _, tracked := range state.TrackedRepos { + if filepath.Clean(tracked.Path) == filepath.Clean(repo.Path) && tracked.Prefix != "" && (tracked.State == "ready" || tracked.State == "") { + timing.ReadyAt[repo.Corpus.Label] = now + } + } + } + covered, err := readyCoverage(state, configured) + if err == nil && covered && state.Ready && state.Enriched && timing.EnrichedAt.IsZero() && len(timing.ReadyAt) == len(configured) { + timing.EnrichedAt = now + } + return err +} + +func readCoordinators(ctx context.Context, p *privateRun, timing *bootTiming) { + statusCtx, cancel := context.WithTimeout(ctx, time.Second) + data, err := control(statusCtx, p.Socket, "status", nil) + cancel() + timing.CensusReads++ + var status struct { + Views json.RawMessage `json:"views"` + } + var views struct { + Coordinators *int `json:"coordinators"` + } + if err == nil { + err = json.Unmarshal(data, &status) + } + if err == nil && len(status.Views) > 0 { + err = json.Unmarshal(status.Views, &views) + } + if err == nil && views.Coordinators == nil { + err = errors.New("status has no checkout coordinator census") + } + if err != nil { + if len(timing.CensusErrors) < 5 { + timing.CensusErrors = append(timing.CensusErrors, err.Error()) + } + return + } + timing.Views = status.Views + if timing.Coordinators == nil || *views.Coordinators > *timing.Coordinators { + timing.Coordinators = views.Coordinators + } +} + +// addLinkedCheckouts adds linked worktrees of the first manifest repository. +// They are not configured: the daemon discovers them through the primary. +func addLinkedCheckouts(ctx context.Context, p *privateRun, count int) ([]probeCWD, error) { + primary := p.Repos[0] + var linked []probeCWD + for i := range count { + path := filepath.Join(p.Root, "repos", fmt.Sprintf("linked-%d", i+1)) + if _, err := p.git(ctx, primary.Path, "worktree", "add", "--detach", path, primary.Corpus.Commit); err != nil { + return nil, err + } + file, err := ownedFile(path, primary.Corpus.File) + if err != nil { + return nil, err + } + linked = append(linked, probeCWD{Role: "worktree", Dir: path, File: file}) + } + return linked, nil +} + +// writeRouterRoster gives one cwd inside the primary a workspace that a +// servers.toml roster claims. The roster's only server is a socket that never +// exists, so a routed call fails to dial and the daemon answers locally; the +// primary and worktree cwds keep their ordinary path. +func writeRouterRoster(p *privateRun) (string, error) { + dir := filepath.Join(p.Repos[0].Path, ".gortex-probe-router") + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(dir, ".gortex.yaml"), []byte("workspace: "+startupRosterWorkspace+"\n"), 0o600); err != nil { + return "", err + } + configDir := filepath.Join(p.Root, "config", "gortex") + if err := os.MkdirAll(configDir, 0o700); err != nil { + return "", err + } + remote := "unix://" + filepath.ToSlash(filepath.Join(p.Root, "absent-remote.sock")) + roster := fmt.Sprintf("[[server]]\nslug = \"probe-remote\"\nurl = %q\nworkspaces = [%q]\n", remote, startupRosterWorkspace) + return dir, os.WriteFile(filepath.Join(configDir, "servers.toml"), []byte(roster), 0o600) +} + +// copySeedStore copies a quiescent store, and its write-ahead log when there +// is one, into the private run. The source is opened read-only. +func copySeedStore(source, dest string) error { + for _, suffix := range []string{"", "-wal"} { + in, err := os.Open(source + suffix) + if suffix != "" && errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return fmt.Errorf("seed store: %w", err) + } + out, err := os.OpenFile(dest+suffix, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + _ = in.Close() + return fmt.Errorf("seed store: %w", err) + } + _, err = io.Copy(out, in) + if err = errors.Join(err, in.Close(), out.Close()); err != nil { + return fmt.Errorf("seed store: %w", err) + } + } + return nil +} diff --git a/bench/load/startup_probe_sample.go b/bench/load/startup_probe_sample.go new file mode 100644 index 000000000..0b6b01eb0 --- /dev/null +++ b/bench/load/startup_probe_sample.go @@ -0,0 +1,610 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "os" + "os/exec" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +const ( + startupProbeInterval = time.Second + startupCallTimeout = 10 * time.Second + // An interactive CLI call waits up to 30 s for a loading repository. + startupCLITimeout = 45 * time.Second + startupTextLimit = 240 +) + +// probeCWD is one directory an agent could start in, and an indexed file the +// native-Read hook probe reads from there. +type probeCWD struct { + Role string // primary, worktree or router + Dir string + File string +} + +// probeSample is one answer as the per-second log records it. Outcome fields +// come from the public wire only: JSON-RPC error data, tool results, CLI exit +// codes and output, and the hook's decision. +type probeSample struct { + CWD string `json:"cwd"` + Surface string `json:"surface"` + Started time.Time `json:"started"` + LatencyMS float64 `json:"latency_ms"` + OK bool `json:"ok"` + Code string `json:"error_code,omitempty"` + Retryable *bool `json:"retryable,omitempty"` + Phase string `json:"phase,omitempty"` + Percent *int `json:"percent,omitempty"` + Status string `json:"status,omitempty"` + Variant string `json:"variant,omitempty"` + ExitCode *int `json:"exit_code,omitempty"` + TrackAdvice bool `json:"track_advice,omitempty"` + Text string `json:"text,omitempty"` +} + +func (s probeSample) completed() time.Time { + return s.Started.Add(time.Duration(s.LatencyMS * float64(time.Millisecond))) +} + +// startupOperations is the offered MCP probe plan. The same values build the +// frames, so the plan hash describes exactly what was sent. +func startupOperations(c Corpus) []request { + return []request{ + {Tool: "initialize"}, + {Tool: "search_symbols", Arguments: map[string]any{"query": c.Query, "limit": 1}}, + {Tool: "resources/read", Operation: "index-health", Arguments: map[string]any{"uri": "gortex://index-health"}}, + {Tool: "workspace", Operation: "index", Arguments: map[string]any{"operation": "index"}}, + } +} + +type startupProbe struct { + socket, binary, query string + env []string + cwds []probeCWD + plan []request + ttyStatus string + workers sync.WaitGroup + + mu sync.Mutex + samples []probeSample + log *os.File + encoder *json.Encoder + logErr error + boot int + socketOn bool + version string +} + +func newStartupProbe(p *privateRun, cwds []probeCWD, query, logPath string) (*startupProbe, error) { + log, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return nil, err + } + s := &startupProbe{socket: p.Socket, binary: p.Binary, query: query, env: p.Env, cwds: cwds, plan: startupOperations(Corpus{Query: query}), log: log, encoder: json.NewEncoder(log), boot: -1} + s.ttyStatus = "script(1) pty" + if _, err := ttyCommand(context.Background(), "true", nil); err != nil { + s.ttyStatus = "unavailable: " + err.Error() + } else if _, err := exec.LookPath("script"); err != nil { + s.ttyStatus = "unavailable: " + err.Error() + } + return s, nil +} + +// start launches every sampler. They run until ctx ends, across boots. +func (s *startupProbe) start(ctx context.Context) { + for _, cwd := range s.cwds { + s.every(ctx, func(ctx context.Context) { s.mcpTick(ctx, cwd) }) + s.every(ctx, func(ctx context.Context) { s.recordLive(ctx, s.hookRead(ctx, cwd)) }) + s.cliLoop(ctx, cwd, false) + if strings.HasPrefix(s.ttyStatus, "script") { + s.cliLoop(ctx, cwd, true) + } + } +} + +func (s *startupProbe) every(ctx context.Context, tick func(context.Context)) { + s.workers.Go(func() { + ticker := time.NewTicker(startupProbeInterval) + defer ticker.Stop() + for ctx.Err() == nil { + tick(ctx) + select { + case <-ctx.Done(): + case <-ticker.C: + } + } + }) +} + +// cliLoop runs one CLI variant every second until it first succeeds in the +// current boot. The pipe variant also runs before the socket exists; the pty +// variant starts once it does, because an interactive call waits through the +// load and its time to exit is the measurement. +func (s *startupProbe) cliLoop(ctx context.Context, cwd probeCWD, tty bool) { + done := -1 + s.every(ctx, func(ctx context.Context) { + boot, socketOn := s.bootState() + if boot < 0 || boot == done || (tty && !socketOn) { + return + } + sample := s.cliCall(ctx, cwd, tty) + if s.recordLive(ctx, sample) && sample.OK { + done = boot + } + }) +} + +func (s *startupProbe) beginBoot(index int) { + s.mu.Lock() + s.boot, s.socketOn = index, false + s.mu.Unlock() +} + +func (s *startupProbe) socketUp(index int) { + s.mu.Lock() + if s.boot == index { + s.socketOn = true + } + s.mu.Unlock() +} + +func (s *startupProbe) stopping() { + s.mu.Lock() + s.socketOn = false + s.mu.Unlock() +} + +func (s *startupProbe) bootState() (int, bool) { + s.mu.Lock() + defer s.mu.Unlock() + return s.boot, s.socketOn +} + +func (s *startupProbe) daemonVersion() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.version +} + +// recordLive keeps a sample unless the probe was stopping when it ended: a +// call cut short by the end of the run is not an answer. +func (s *startupProbe) recordLive(ctx context.Context, sample probeSample) bool { + if ctx.Err() != nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + s.samples = append(s.samples, sample) + if s.logErr == nil { + s.logErr = s.encoder.Encode(sample) + } + return true +} + +func (s *startupProbe) finish() ([]probeSample, error) { + s.workers.Wait() + s.mu.Lock() + defer s.mu.Unlock() + return s.samples, errors.Join(s.logErr, s.log.Close()) +} + +// mcpTick opens fresh sessions from cwd, as a newly started agent would: the +// legacy surface for search_symbols and the index-health resource, and the +// compact surface for workspace(operation:"index"). Each session records the +// instructions variant its initialize returned. +func (s *startupProbe) mcpTick(ctx context.Context, cwd probeCWD) { + sessions := []struct { + tools, mode, initialize string + steps []request + }{ + {"core", "defer", "initialize", s.plan[1:3]}, + {"facade-v1", "hide", "initialize_facade", s.plan[3:]}, + } + for _, session := range sessions { + if !s.mcpSession(ctx, cwd, session.tools, session.mode, session.initialize, session.steps) { + return + } + } +} + +func (s *startupProbe) mcpSession(ctx context.Context, cwd probeCWD, tools, mode, initialize string, steps []request) bool { + started := time.Now() + dialCtx, cancel := context.WithTimeout(ctx, startupCallTimeout) + conn, reader, version, err := dialSocketSurface(dialCtx, s.socket, cwd.Dir, "mcp", tools, mode) + cancel() + if err != nil { + code := "handshake_error" + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "dial" { + code = "no_socket" + } + s.recordLive(ctx, probeSample{CWD: cwd.Role, Surface: "connect", Started: started, LatencyMS: elapsedMS(started), Code: code, Text: clip(err.Error())}) + return false + } + defer conn.Close() + s.mu.Lock() + s.version = version + s.mu.Unlock() + c := &probeConn{conn: conn, reader: reader} + initParams := map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "gortex-load", "version": "1"}} + calls := []probeCall{{initialize, "initialize", initParams}} + for _, step := range steps { + switch { + case step.Tool == "resources/read": + calls = append(calls, probeCall{"index_health_resource", step.Tool, step.Arguments}) + case step.Operation != "": + calls = append(calls, probeCall{step.Tool + "_" + step.Operation, "tools/call", map[string]any{"name": step.Tool, "arguments": step.Arguments}}) + default: + calls = append(calls, probeCall{step.Tool, "tools/call", map[string]any{"name": step.Tool, "arguments": step.Arguments}}) + } + } + for i, call := range calls { + start := time.Now() + callCtx, cancel := context.WithTimeout(ctx, startupCallTimeout) + frame, err := c.roundTrip(callCtx, call.method, call.params) + timedOut := callCtx.Err() == context.DeadlineExceeded + cancel() + var sample probeSample + switch { + case err == nil: + sample = decodeProbeFrame(frame) + case timedOut: + sample = probeSample{Code: "timeout", Text: clip(err.Error())} + default: + sample = probeSample{Code: "transport_error", Text: clip(err.Error())} + } + sample.CWD, sample.Surface, sample.Started, sample.LatencyMS = cwd.Role, call.surface, start, elapsedMS(start) + if !s.recordLive(ctx, sample) || err != nil { + return ctx.Err() == nil + } + if i == 0 && c.notify("notifications/initialized") != nil { + return true + } + } + return true +} + +type probeCall struct { + surface, method string + params any +} + +type probeConn struct { + conn net.Conn + reader *bufio.Reader + next int +} + +// roundTrip sends one request and returns its response frame, skipping +// notifications and server-initiated requests. +func (c *probeConn) roundTrip(ctx context.Context, method string, params any) ([]byte, error) { + c.next++ + id := strconv.Itoa(c.next) + data, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": c.next, "method": method, "params": params}) + if err != nil { + return nil, err + } + if deadline, ok := ctx.Deadline(); ok { + if err := c.conn.SetDeadline(deadline); err != nil { + return nil, err + } + } + stop := context.AfterFunc(ctx, func() { _ = c.conn.SetDeadline(time.Now()) }) + defer stop() + if _, err := c.conn.Write(append(data, '\n')); err != nil { + return nil, err + } + for { + line, err := readWireLine(c.reader) + if err != nil { + return nil, err + } + var peek struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + } + if err := json.Unmarshal(line, &peek); err != nil { + return nil, fmt.Errorf("invalid daemon JSON: %w", err) + } + if peek.Method == "" && string(peek.ID) == id { + return line, nil + } + } +} + +func (c *probeConn) notify(method string) error { + data, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "method": method}) + if err == nil { + _, err = c.conn.Write(append(data, '\n')) + } + return err +} + +// admissionFields are the machine-readable fields of a refusal's JSON-RPC +// error data, or of a tool's JSON payload. +type admissionFields struct { + ErrorCode string `json:"error_code"` + Retryable *bool `json:"retryable"` + Phase string `json:"phase"` + Percent *float64 `json:"percent"` + Status string `json:"status"` +} + +func (f admissionFields) apply(s *probeSample) { + if f.ErrorCode != "" { + s.Code = f.ErrorCode + } + s.Retryable, s.Phase, s.Status = f.Retryable, f.Phase, f.Status + if f.Percent != nil { + percent := int(*f.Percent) + s.Percent = &percent + } +} + +// decodeProbeFrame classifies one JSON-RPC response frame. +func decodeProbeFrame(frame []byte) probeSample { + var reply struct { + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Data json.RawMessage `json:"data"` + } `json:"error"` + Result *struct { + Instructions *string `json:"instructions"` + IsError bool `json:"isError"` + Content []struct { + Text string `json:"text"` + } `json:"content"` + Contents []struct { + Text string `json:"text"` + } `json:"contents"` + } `json:"result"` + } + if err := json.Unmarshal(frame, &reply); err != nil || (reply.Error == nil && reply.Result == nil) { + return probeSample{Code: "invalid_response", Text: clip(string(frame))} + } + // Successful graph answers can quote indexed text that mentions the + // commands, so only refusals and instructions are checked for advice. + var sample probeSample + if reply.Error != nil { + sample.TrackAdvice = adviceIn(string(frame)) + sample.Code = fmt.Sprintf("rpc_%d", reply.Error.Code) + var fields admissionFields + _ = json.Unmarshal(reply.Error.Data, &fields) + fields.apply(&sample) + sample.Text = clip(reply.Error.Message) + return sample + } + result := reply.Result + if result.Instructions != nil { + sample.OK, sample.Variant, sample.TrackAdvice = true, instructionsVariant(*result.Instructions), adviceIn(*result.Instructions) + if sample.Variant != "active" { + sample.Text = clip(*result.Instructions) + } + return sample + } + text := "" + for _, block := range append(result.Content, result.Contents...) { + if text = block.Text; text != "" { + break + } + } + var fields admissionFields + _ = json.Unmarshal([]byte(text), &fields) + fields.apply(&sample) + sample.OK = !result.IsError && fields.ErrorCode == "" + if !sample.OK { + if sample.Code == "" { + sample.Code = "tool_error" + } + sample.Text, sample.TrackAdvice = clip(text), adviceIn(string(frame)) + } + return sample +} + +// instructionsVariant names the initialize instructions an agent received. +// Anything without one of the inactive or loading preambles is active. +func instructionsVariant(text string) string { + for _, variant := range []struct{ prefix, name string }{ + {"Gortex is LOADING graph admission", "admission_pending"}, + {"Gortex is LOADING ", "loading"}, + {"Gortex is connected but could not load", "unavailable"}, + {"Gortex is connected but INACTIVE", "untracked"}, + {"Gortex is connected; checkout discovery is pending", "checkout_pending"}, + {"Gortex checkout discovery is unavailable", "checkout_admission_error"}, + } { + if strings.HasPrefix(text, variant.prefix) { + return variant.name + } + } + if strings.TrimSpace(text) == "" { + return "empty" + } + return "active" +} + +// trackAdvicePattern finds advice to run `gortex track` or `gortex repos +// reconcile`. A negated mention ("Do not run `gortex track`") is guidance +// against it and is captured in the first group so it can be ignored. +var trackAdvicePattern = regexp.MustCompile("(?i)(\\bnot\\s+(?:run\\s+)?|n't\\s+(?:run\\s+)?|\\bnever\\s+(?:run\\s+)?)?`?gortex\\s+(?:track|repos\\s+reconcile)\\b") + +func adviceIn(text string) bool { + for _, match := range trackAdvicePattern.FindAllStringSubmatch(text, -1) { + if match[1] == "" { + return true + } + } + return false +} + +// cliCall runs `gortex call search_symbols` from cwd. Without a pty the CLI +// must fail fast with exit code 75 while the repository loads; with one it +// waits for the load, so its time to exit includes that wait. +func (s *startupProbe) cliCall(ctx context.Context, cwd probeCWD, tty bool) probeSample { + surface := "cli_pipe" + if tty { + surface = "cli_tty" + } + args, _ := json.Marshal(map[string]any{"query": s.query, "limit": 1}) + callCtx, cancel := context.WithTimeout(ctx, startupCLITimeout) + defer cancel() + argv := []string{"call", "search_symbols", "--json", string(args)} + var cmd *exec.Cmd + var err error + if tty { + cmd, err = ttyCommand(callCtx, s.binary, argv) + } else { + cmd = exec.CommandContext(callCtx, s.binary, argv...) + } + started := time.Now() + if err != nil { + return probeSample{CWD: cwd.Role, Surface: surface, Started: started, Code: "unavailable", Text: clip(err.Error())} + } + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + exitCode, err := runOwned(cmd, cwd.Dir, s.env) + sample := probeSample{CWD: cwd.Role, Surface: surface, Started: started, LatencyMS: elapsedMS(started), ExitCode: &exitCode} + text := output.String() + sample.OK, sample.Code = classifyCLI(exitCode, text) + if callCtx.Err() == context.DeadlineExceeded { + sample.OK, sample.Code = false, "timeout" + } else if err != nil && exitCode < 0 { + sample.Code = "exec_error" + text += err.Error() + } + if !sample.OK { + sample.Text, sample.TrackAdvice = clip(text), adviceIn(text) + } + return sample +} + +func classifyCLI(exitCode int, text string) (bool, string) { + switch { + case exitCode == 0: + return true, "" + case exitCode == 75: + return false, "repo_loading" + case strings.Contains(text, "does not track"): + return false, "repo_not_tracked" + case strings.Contains(text, "no gortex daemon is running"): + return false, "no_daemon" + default: + return false, fmt.Sprintf("exit_%d", exitCode) + } +} + +// ttyCommand runs argv under script(1) so the CLI's stdout is a terminal. +func ttyCommand(ctx context.Context, binary string, argv []string) (*exec.Cmd, error) { + switch runtime.GOOS { + case "darwin", "freebsd": + return exec.CommandContext(ctx, "script", append([]string{"-q", "/dev/null", binary}, argv...)...), nil + case "linux": + quoted := make([]string, 0, len(argv)+1) + for _, arg := range append([]string{binary}, argv...) { + quoted = append(quoted, "'"+strings.ReplaceAll(arg, "'", `'\''`)+"'") + } + return exec.CommandContext(ctx, "script", "-q", "-e", "-c", strings.Join(quoted, " "), "/dev/null"), nil + default: + return nil, fmt.Errorf("no pty runner on %s", runtime.GOOS) + } +} + +// hookRead asks the PreToolUse hook about a native Read of an indexed file. +func (s *startupProbe) hookRead(ctx context.Context, cwd probeCWD) probeSample { + payload, _ := json.Marshal(map[string]any{"hook_event_name": "PreToolUse", "session_id": "gortex-load-startup", "tool_name": "Read", "tool_input": map[string]any{"file_path": cwd.File}, "cwd": cwd.Dir}) + callCtx, cancel := context.WithTimeout(ctx, startupCallTimeout) + defer cancel() + cmd := exec.CommandContext(callCtx, s.binary, "hook") + cmd.Stdin = bytes.NewReader(payload) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + started := time.Now() + exitCode, err := runOwned(cmd, cwd.Dir, s.env) + sample := probeSample{CWD: cwd.Role, Surface: "hook_read", Started: started, LatencyMS: elapsedMS(started), ExitCode: &exitCode} + decision, text := decodeHookOutput(stdout.Bytes()) + sample.Variant, sample.TrackAdvice, sample.Text = decision, adviceIn(text), clip(text) + sample.OK = err == nil + if err != nil { + sample.Variant, sample.Code = "", "hook_error" + if callCtx.Err() == context.DeadlineExceeded { + sample.Code = "timeout" + } + sample.Text = clip(err.Error() + " " + stderr.String()) + } + return sample +} + +// decodeHookOutput reads the default hook protocol's PreToolUse reply. No +// output, or context without a decision, lets the native tool run. +func decodeHookOutput(output []byte) (decision, text string) { + output = bytes.TrimSpace(output) + if len(output) == 0 { + return "allow", "" + } + var reply struct { + Decision string `json:"decision"` + Reason string `json:"reason"` + Specific *struct { + Decision string `json:"permissionDecision"` + Reason string `json:"permissionDecisionReason"` + Context string `json:"additionalContext"` + } `json:"hookSpecificOutput"` + } + if json.Unmarshal(output, &reply) != nil { + return "unparsed", string(output) + } + decision, text = "allow", reply.Reason + if reply.Specific != nil { + if reply.Specific.Decision != "" { + decision = reply.Specific.Decision + } + text = strings.TrimSpace(text + " " + reply.Specific.Reason + " " + reply.Specific.Context) + } + if reply.Decision == "block" { + decision = "deny" + } + return decision, text +} + +// runOwned runs a probe command in its own process group, so a call cut +// short by its deadline cannot leave children behind. +func runOwned(cmd *exec.Cmd, dir string, env []string) (int, error) { + configureOwnedProcess(cmd) + cmd.Dir, cmd.Env, cmd.WaitDelay = dir, env, 5*time.Second + cmd.Cancel = func() error { + if cmd.Process == nil { + return os.ErrProcessDone + } + return killOwnedProcess(cmd) + } + err := cmd.Run() + if err != nil && cmd.Process != nil { + _ = cleanupExitedOwnedProcess(cmd) + } + if cmd.ProcessState == nil { + return -1, err + } + return cmd.ProcessState.ExitCode(), err +} + +func elapsedMS(start time.Time) float64 { return durationMS(time.Since(start)) } + +func clip(text string) string { + text = strings.TrimSpace(text) + if len(text) <= startupTextLimit { + return text + } + return text[:startupTextLimit] + "..." +} diff --git a/bench/load/startup_probe_summary.go b/bench/load/startup_probe_summary.go new file mode 100644 index 000000000..495af080d --- /dev/null +++ b/bench/load/startup_probe_summary.go @@ -0,0 +1,247 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +// graphSurfaces answer from the repository graph. Their first success after +// the primary registers is how long an agent waits for a working tool. +var graphSurfaces = []string{"search_symbols", "index_health_resource", "cli_pipe", "cli_tty"} + +var probeRoles = []string{"primary", "worktree", "router"} + +type firstOKTiming struct { + FromStartMS Metric `json:"from_start_ms"` + FromReadyMS Metric `json:"from_ready_ms"` +} + +type bootSummary struct { + bootTiming + TimeToReadyMS Metric `json:"time_to_ready_ms"` + FirstOK map[string]firstOKTiming `json:"first_ok"` +} + +// startupSummary is the startup probe's gate receipt. Answers are keyed +// "surface/cwd" and split into windows by what the readiness watcher saw: +// no_socket before the socket answered, loading until the primary was ready, +// and ready after. +type startupSummary struct { + PrimaryRepo string `json:"primary_repo"` + LinkedWorktrees int `json:"linked_worktrees"` + WatchIntervalMS float64 `json:"watch_interval_ms"` + CLITTY string `json:"cli_tty"` + Boots []bootSummary `json:"boots"` + // Outcomes counts each answer as ok, ok:, an error code (with + // repo_loading(incomplete) for a loading refusal missing retryable=true, + // phase or percent), an initialize instructions variant, or a hook + // decision. + Outcomes map[string]map[string]map[string]int `json:"outcomes"` + TrackAdvice map[string]map[string]int `json:"track_advice"` + RestartReadyToFirstOK map[string]Distribution `json:"restart_ready_to_first_ok_ms"` + BetweenBoots int `json:"samples_between_boots"` +} + +// firstOK is when the earliest successful sample for surface and cwd that +// started in [from, until) completed; a zero until is open. Measured from a +// boot's start it is time to first OK; measured from readiness it is ready +// to first OK, for any warm or cold time-to-ready comparison. +func firstOK(samples []probeSample, surface, cwd string, from, until time.Time) (time.Time, bool) { + var first time.Time + for _, s := range samples { + if !s.OK || s.Surface != surface || s.CWD != cwd || s.Started.Before(from) || (!until.IsZero() && !s.Started.Before(until)) { + continue + } + if done := s.completed(); first.IsZero() || done.Before(first) { + first = done + } + } + return first, !first.IsZero() +} + +// bootIndex places a sample in the boot it observed. The probe starts just +// before the first daemon does, so earlier samples belong to the first boot; +// samples between a stop request and the next start belong to none. +func bootIndex(boots []bootTiming, at time.Time) int { + for i, boot := range boots { + if (i == 0 || !at.Before(boot.Started)) && (boot.Stopping.IsZero() || at.Before(boot.Stopping)) { + return i + } + } + return -1 +} + +func startupWindow(boot bootTiming, primary string, at time.Time) string { + if boot.SocketAt.IsZero() || at.Before(boot.SocketAt) { + return "no_socket" + } + if ready, ok := boot.ReadyAt[primary]; !ok || at.Before(ready) { + return "loading" + } + return "ready" +} + +func outcomeKey(s probeSample) string { + switch { + case s.Variant != "": + return s.Variant + case s.OK && s.Status != "": + return "ok:" + s.Status + case s.OK: + return "ok" + case s.Code == "repo_loading" && !strings.HasPrefix(s.Surface, "cli_") && (s.Retryable == nil || !*s.Retryable || s.Phase == "" || s.Percent == nil): + return "repo_loading(incomplete)" + default: + return s.Code + } +} + +func summarizeStartup(samples []probeSample, boots []bootTiming, primary string, linked int) startupSummary { + sum := startupSummary{PrimaryRepo: primary, LinkedWorktrees: linked, WatchIntervalMS: durationMS(startupWatchInterval), Outcomes: make(map[string]map[string]map[string]int), TrackAdvice: make(map[string]map[string]int), RestartReadyToFirstOK: make(map[string]Distribution)} + for _, s := range samples { + index := bootIndex(boots, s.Started) + if index < 0 { + sum.BetweenBoots++ + continue + } + key, window := s.Surface+"/"+s.CWD, startupWindow(boots[index], primary, s.Started) + if sum.Outcomes[key] == nil { + sum.Outcomes[key] = make(map[string]map[string]int) + } + if sum.Outcomes[key][window] == nil { + sum.Outcomes[key][window] = make(map[string]int) + } + sum.Outcomes[key][window][outcomeKey(s)]++ + if s.TrackAdvice { + if sum.TrackAdvice[key] == nil { + sum.TrackAdvice[key] = make(map[string]int) + } + sum.TrackAdvice[key][window]++ + } + } + restarts := make(map[string][]float64) + for i, boot := range boots { + row := bootSummary{bootTiming: boot, TimeToReadyMS: unavailable("ms", "primary repository never ready"), FirstOK: make(map[string]firstOKTiming)} + ready, registered := boot.ReadyAt[primary] + if registered { + row.TimeToReadyMS = measured(durationMS(ready.Sub(boot.Started)), "ms") + } + for _, surface := range graphSurfaces { + for _, cwd := range probeRoles { + first, ok := firstOK(samples, surface, cwd, boot.Started, boot.Stopping) + if !ok { + continue + } + key := surface + "/" + cwd + timing := firstOKTiming{FromStartMS: measured(durationMS(first.Sub(boot.Started)), "ms"), FromReadyMS: unavailable("ms", "primary repository never ready")} + if registered { + // The watcher stamps readiness up to one poll late, so a + // success can precede it; that wait is zero. + fromReady := max(0, durationMS(first.Sub(ready))) + timing.FromReadyMS = measured(fromReady, "ms") + if i > 0 { + restarts[key] = append(restarts[key], fromReady) + } + } + row.FirstOK[key] = timing + } + } + sum.Boots = append(sum.Boots, row) + } + for key, values := range restarts { + sum.RestartReadyToFirstOK[key] = distribution(values, "ms") + } + return sum +} + +// metrics are the headline counters. Counts cover every boot; restart +// figures cover boots after the first and are unavailable unless every +// restart produced the observation. +func (sum startupSummary) metrics() map[string]Metric { + count := func(match func(surface, window, outcome string) bool) Metric { + total := 0 + for key, windows := range sum.Outcomes { + surface, _, _ := strings.Cut(key, "/") + for window, outcomes := range windows { + for outcome, n := range outcomes { + if match(surface, window, outcome) { + total += n + } + } + } + } + return measured(float64(total), "count") + } + advice := func(window string) Metric { + total := 0 + for _, windows := range sum.TrackAdvice { + total += windows[window] + } + return measured(float64(total), "count") + } + m := map[string]Metric{ + "startup_boots": measured(float64(len(sum.Boots)), "count"), + "startup_linked_worktrees": measured(float64(sum.LinkedWorktrees), "count"), + // The untracked initialize variant is the same answer at handshake time. + "startup_repo_not_tracked_before_ready": count(func(_, window, outcome string) bool { + return window != "ready" && (outcome == "repo_not_tracked" || outcome == "untracked") + }), + "startup_nonconforming_loading_refusals": count(func(surface, window, outcome string) bool { + return window == "loading" && surface != "hook_read" && !strings.HasPrefix(surface, "initialize") && outcome != "repo_loading" && !strings.HasPrefix(outcome, "ok") + }), + "startup_hook_denies_while_loading": count(func(surface, window, outcome string) bool { + return surface == "hook_read" && window != "ready" && outcome == "deny" + }), + "startup_hook_denies_after_ready": count(func(surface, window, outcome string) bool { + return surface == "hook_read" && window == "ready" && outcome == "deny" + }), + "startup_track_advice_while_loading": advice("loading"), + "startup_track_advice_without_socket": advice("no_socket"), + "startup_initial_time_to_ready_ms": unavailable("ms", "no boot"), + "startup_initial_first_ok_ms": unavailable("ms", "no successful search_symbols from the primary"), + } + if len(sum.Boots) > 0 { + m["startup_initial_time_to_ready_ms"] = sum.Boots[0].TimeToReadyMS + if first, ok := sum.Boots[0].FirstOK["search_symbols/primary"]; ok { + m["startup_initial_first_ok_ms"] = first.FromStartMS + } + } + restarts := max(0, len(sum.Boots)-1) + var toReady []float64 + for _, boot := range sum.Boots[min(1, len(sum.Boots)):] { + if boot.TimeToReadyMS.Value != nil { + toReady = append(toReady, *boot.TimeToReadyMS.Value) + } + } + m["startup_restart_time_to_ready_p95_ms"] = restartP95(distribution(toReady, "ms"), restarts, "reached ready") + m["startup_restart_ready_to_first_ok_p95_ms"] = restartP95(sum.RestartReadyToFirstOK["search_symbols/primary"], restarts, "had a successful search_symbols from the primary") + // The census is per boot; with no restart the initial boot is all there is. + census := sum.Boots[min(1, len(sum.Boots)):] + if len(sum.Boots) == 1 { + census = sum.Boots + } + m["startup_coordinators_within_2s_min"] = unavailable("count", "no boot") + for i, boot := range census { + if boot.Coordinators == nil { + m["startup_coordinators_within_2s_min"] = unavailable("count", fmt.Sprintf("boot %d has no coordinator census", boot.Boot)) + break + } + if value := float64(*boot.Coordinators); i == 0 || value < *m["startup_coordinators_within_2s_min"].Value { + m["startup_coordinators_within_2s_min"] = measured(value, "count") + } + } + return m +} + +func restartP95(d Distribution, restarts int, observation string) Metric { + if restarts == 0 { + return unavailable("ms", "no restarts") + } + if d.Count < restarts { + return unavailable("ms", fmt.Sprintf("%d of %d restarts %s", d.Count, restarts, observation)) + } + return d.P95 +} + +func durationMS(d time.Duration) float64 { return float64(d) / float64(time.Millisecond) } diff --git a/bench/load/startup_probe_test.go b/bench/load/startup_probe_test.go new file mode 100644 index 000000000..dd55103ce --- /dev/null +++ b/bench/load/startup_probe_test.go @@ -0,0 +1,329 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDecodeProbeFrameReadsAdmissionAnswers(t *testing.T) { + active := "When Gortex can serve the step: MUST use Gortex MCP.\n\nWorktree and branch routing:\n\n- A linked worktree is an automatic overlay; do not `track` it. Graph calls return `view_building` until discovery completes; retry them, no reconnect is needed. Do not run `gortex track`." + for _, test := range []struct { + name, frame, outcome string + ok, advice bool + check func(t *testing.T, s probeSample) + }{ + {name: "loading refusal", outcome: "repo_loading", + frame: `{"jsonrpc":"2.0","id":2,"error":{"code":-32000,"message":"repository loading: app (resolving, 40%)","data":{"error_code":"repo_loading","retryable":true,"phase":"resolving","percent":40,"suggestion":"Gortex is still loading app after a restart; tools start working in place, no reconnect needed. Retry shortly. Do not change tracking configuration."}}}`, + check: func(t *testing.T, s probeSample) { + if s.Retryable == nil || !*s.Retryable || s.Phase != "resolving" || s.Percent == nil || *s.Percent != 40 { + t.Fatalf("loading fields lost: %+v", s) + } + }}, + {name: "loading refusal without progress", outcome: "repo_loading(incomplete)", + frame: `{"jsonrpc":"2.0","id":2,"error":{"code":-32000,"message":"repository loading","data":{"error_code":"repo_loading","retryable":true}}}`}, + {name: "untracked refusal advises tracking", outcome: "repo_not_tracked", advice: true, + frame: "{\"jsonrpc\":\"2.0\",\"id\":2,\"error\":{\"code\":-32000,\"message\":\"repository not tracked: /w\",\"data\":{\"error_code\":\"repo_not_tracked\",\"path\":\"/w\",\"suggestion\":\"Run `gortex track /w` to include this repo in the shared graph.\"}}}"}, + {name: "unavailable is final", outcome: "repo_unavailable", advice: true, + frame: "{\"jsonrpc\":\"2.0\",\"id\":2,\"error\":{\"code\":-32000,\"message\":\"repository unavailable\",\"data\":{\"error_code\":\"repo_unavailable\",\"retryable\":false,\"suggestion\":\"Gortex could not load /w after the restart. Run `gortex track /w` to index it again.\"}}}", + check: func(t *testing.T, s probeSample) { + if s.Retryable == nil || *s.Retryable { + t.Fatalf("retryable=false must be kept: %+v", s) + } + }}, + {name: "busy without data", outcome: "rpc_-32002", + frame: `{"jsonrpc":"2.0","id":2,"error":{"code":-32002,"message":"busy"}}`}, + {name: "loading index health answers", outcome: "ok:loading", ok: true, + frame: `{"jsonrpc":"2.0","id":3,"result":{"content":[{"type":"text","text":"{\"status\":\"loading\",\"phase\":\"starting\",\"percent\":0,\"repos\":[]}"}]}}`, + check: func(t *testing.T, s probeSample) { + if s.Phase != "starting" || s.Percent == nil || *s.Percent != 0 { + t.Fatalf("a zero percent is an observation, not a missing one: %+v", s) + } + }}, + {name: "resource read", outcome: "ok:ok", ok: true, + frame: `{"jsonrpc":"2.0","id":4,"result":{"contents":[{"uri":"gortex://index-health","text":"{\"status\":\"ok\"}"}]}}`}, + {name: "search results may quote the commands", outcome: "ok", ok: true, + frame: "{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"{\\\"results\\\":[{\\\"doc\\\":\\\"run `gortex track .` first\\\"}]}\"}]}}"}, + {name: "tool error payload", outcome: "view_building", + frame: `{"jsonrpc":"2.0","id":2,"result":{"isError":true,"content":[{"type":"text","text":"{\"error_code\":\"view_building\",\"message\":\"retry\"}"}]}}`}, + {name: "plain tool error", outcome: "tool_error", + frame: `{"jsonrpc":"2.0","id":2,"result":{"isError":true,"content":[{"type":"text","text":"tool not found"}]}}`}, + {name: "loading instructions", outcome: "loading", ok: true, + frame: mustInitialize(t, "Gortex is LOADING app after a restart (starting, 0%). Graph tools return repo_loading (retryable) until it is ready; Do not re-track it or change tracking configuration: the repository is already configured.\n\n"+active)}, + {name: "admission pending instructions", outcome: "admission_pending", ok: true, + frame: mustInitialize(t, "Gortex is LOADING graph admission after a restart (identity_resolution, 0%). Graph admission for this path is not yet established.")}, + {name: "unavailable instructions", outcome: "unavailable", ok: true, advice: true, + frame: mustInitialize(t, "Gortex is connected but could not load /w after the restart: boom.\n\nGraph tools return repo_unavailable for it. To repair, run `gortex track /w` to index it again.")}, + {name: "untracked instructions", outcome: "untracked", ok: true, advice: true, + frame: mustInitialize(t, "Gortex is connected but INACTIVE for this directory: \"/w\" is not covered by any tracked repository.\n\nTo activate: run `gortex track /w`, then retry graph calls; no reconnect is needed.")}, + {name: "active instructions forbid tracking", outcome: "active", ok: true, + frame: mustInitialize(t, active)}, + {name: "not JSON", outcome: "invalid_response", frame: `daemon: serving`}, + } { + t.Run(test.name, func(t *testing.T) { + s := decodeProbeFrame([]byte(test.frame)) + if got := outcomeKey(s); got != test.outcome || s.OK != test.ok || s.TrackAdvice != test.advice { + t.Fatalf("outcome %q ok=%v advice=%v, want %q ok=%v advice=%v: %+v", got, s.OK, s.TrackAdvice, test.outcome, test.ok, test.advice, s) + } + if test.check != nil { + test.check(t, s) + } + }) + } +} + +func mustInitialize(t *testing.T, instructions string) string { + t.Helper() + data, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "result": map[string]any{"protocolVersion": "2024-11-05", "instructions": instructions}}) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestTrackAdviceIgnoresGuidanceAgainstIt(t *testing.T) { + for text, want := range map[string]bool{ + "Run `gortex track /w` to include this repo": true, + "the gortex daemon does not track /w — add it with `gortex track /w`": true, + "run `gortex repos reconcile /w` and retry": true, + "start it, then track this repo with gortex track /w": true, + "Do not run `gortex track`.": false, + "Don't run gortex track here": false, + "never gortex repos reconcile a loading family": false, + "remove it with `gortex untrack /w`": false, + "gortex tracks hundreds of repositories": false, + "the repository is already tracked; retry shortly": false, + "Do not run `gortex track`. To activate: run `gortex track /w`.": true, + } { + if got := adviceIn(text); got != want { + t.Errorf("adviceIn(%q) = %v, want %v", text, got, want) + } + } +} + +func TestCLIAndHookAnswers(t *testing.T) { + for _, test := range []struct { + exit int + text string + ok bool + code string + }{ + {0, `{"results":[]}`, true, ""}, + {75, "the gortex daemon is still loading app after a restart (starting, 0%)", false, "repo_loading"}, + {1, "the gortex daemon does not track /w — add it with `gortex track /w`", false, "repo_not_tracked"}, + {1, "no gortex daemon is running — start it with `gortex daemon start --detach`", false, "no_daemon"}, + {2, "unknown flag", false, "exit_2"}, + } { + if ok, code := classifyCLI(test.exit, test.text); ok != test.ok || code != test.code { + t.Errorf("classifyCLI(%d, %q) = %v %q, want %v %q", test.exit, test.text, ok, code, test.ok, test.code) + } + } + for output, want := range map[string]string{ + "": "allow", + `{"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":"deny","permissionDecisionReason":"use read(operation:\"file\")"}}`: "deny", + `{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"app is still loading (repo_loading); the native Read is allowed"}}`: "allow", + `{"decision":"block","reason":"indexed"}`: "deny", + `not json`: "unparsed", + } { + if got, _ := decodeHookOutput([]byte(output)); got != want { + t.Errorf("decodeHookOutput(%q) = %q, want %q", output, got, want) + } + } +} + +// roundTrip must skip pushes the daemon interleaves before the response. +func TestProbeRoundTripSkipsNotifications(t *testing.T) { + client, server := net.Pipe() + defer client.Close() + go func() { + defer server.Close() + request, err := bufio.NewReader(server).ReadString('\n') + if err != nil || !strings.Contains(request, `"method":"tools/call"`) { + return + } + _, _ = server.Write([]byte(`{"jsonrpc":"2.0","method":"notifications/message","params":{"data":"loading"}}` + "\n" + + `{"jsonrpc":"2.0","id":7,"method":"roots/list"}` + "\n" + + `{"jsonrpc":"2.0","id":1,"result":{"content":[{"type":"text","text":"{}"}]}}` + "\n")) + }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + c := &probeConn{conn: client, reader: bufio.NewReader(client)} + frame, err := c.roundTrip(ctx, "tools/call", map[string]any{"name": "search_symbols"}) + if err != nil || !strings.Contains(string(frame), `"id":1`) { + t.Fatalf("roundTrip = %s, %v", frame, err) + } +} + +func TestFirstOKUsesEarliestCompletionInsideTheBoot(t *testing.T) { + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + samples := []probeSample{ + {CWD: "primary", Surface: "search_symbols", Started: t0, OK: true}, // previous boot + {CWD: "primary", Surface: "search_symbols", Started: t0.Add(11 * time.Second), LatencyMS: 3000, OK: true}, // slow + {CWD: "primary", Surface: "search_symbols", Started: t0.Add(12 * time.Second), LatencyMS: 100, OK: true}, // first to finish + {CWD: "worktree", Surface: "search_symbols", Started: t0.Add(11 * time.Second), LatencyMS: 10, OK: true}, // other cwd + {CWD: "primary", Surface: "search_symbols", Started: t0.Add(10500 * time.Millisecond), LatencyMS: 5, Code: "repo_loading"}, + {CWD: "primary", Surface: "search_symbols", Started: t0.Add(30 * time.Second), LatencyMS: 1, OK: true}, // next boot + } + first, ok := firstOK(samples, "search_symbols", "primary", t0.Add(10*time.Second), t0.Add(20*time.Second)) + if want := t0.Add(12100 * time.Millisecond); !ok || !first.Equal(want) { + t.Fatalf("firstOK = %v %v, want %v", first, ok, want) + } + if _, ok := firstOK(samples, "cli_pipe", "primary", t0, time.Time{}); ok { + t.Fatal("no sample of the surface is not a first OK") + } +} + +func TestSummarizeStartupCountsPerWindowAndRestart(t *testing.T) { + t0 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + at := func(ms int) time.Time { return t0.Add(time.Duration(ms) * time.Millisecond) } + coordinators := func(n int) *int { return &n } + boots := []bootTiming{ + {Boot: 0, Started: at(0), SocketAt: at(2000), ReadyAt: map[string]time.Time{"app": at(5000)}, Stopping: at(20000), Coordinators: coordinators(3)}, + {Boot: 1, Started: at(21000), SocketAt: at(21500), ReadyAt: map[string]time.Time{"app": at(22000)}, Stopping: at(30000), Coordinators: coordinators(4)}, + {Boot: 2, Started: at(31000), SocketAt: at(31200), ReadyAt: map[string]time.Time{"app": at(31600)}, Coordinators: coordinators(2)}, + } + yes, percent := true, 10 + samples := []probeSample{ + {CWD: "primary", Surface: "connect", Started: at(-500), Code: "no_socket"}, // before the first start + {CWD: "primary", Surface: "cli_pipe", Started: at(1000), Code: "no_daemon", TrackAdvice: true}, + {CWD: "primary", Surface: "initialize", Started: at(3000), OK: true, Variant: "loading"}, + {CWD: "router", Surface: "initialize", Started: at(3000), OK: true, Variant: "untracked", TrackAdvice: true}, + {CWD: "primary", Surface: "search_symbols", Started: at(3000), Code: "repo_loading", Retryable: &yes, Phase: "starting", Percent: &percent}, + {CWD: "primary", Surface: "search_symbols", Started: at(4000), Code: "repo_loading", Retryable: &yes}, + {CWD: "worktree", Surface: "search_symbols", Started: at(3000), Code: "repo_not_tracked", TrackAdvice: true}, + {CWD: "primary", Surface: "workspace_index", Started: at(3000), OK: true, Status: "loading"}, + {CWD: "primary", Surface: "hook_read", Started: at(3000), OK: true, Variant: "allow"}, + {CWD: "primary", Surface: "hook_read", Started: at(6000), OK: true, Variant: "deny"}, + {CWD: "primary", Surface: "search_symbols", Started: at(5500), LatencyMS: 100, OK: true}, + {CWD: "primary", Surface: "search_symbols", Started: at(20500), Code: "no_socket"}, // between boots + {CWD: "primary", Surface: "search_symbols", Started: at(23000), LatencyMS: 50, OK: true}, + {CWD: "primary", Surface: "search_symbols", Started: at(31500), LatencyMS: 50, OK: true}, // before the stamp + } + sum := summarizeStartup(samples, boots, "app", 24) + if got := sum.Outcomes["search_symbols/primary"]["loading"]; got["repo_loading"] != 1 || got["repo_loading(incomplete)"] != 1 || got["ok"] != 1 { + t.Fatalf("loading outcomes = %v", got) + } + if got := sum.Outcomes["search_symbols/primary"]["ready"]["ok"]; got != 2 { + t.Fatalf("ready successes = %d", got) + } + if sum.Outcomes["connect/primary"]["no_socket"]["no_socket"] != 1 || sum.BetweenBoots != 1 || sum.TrackAdvice["cli_pipe/primary"]["no_socket"] != 1 || sum.TrackAdvice["search_symbols/worktree"]["loading"] != 1 { + t.Fatalf("between=%d advice=%v", sum.BetweenBoots, sum.TrackAdvice) + } + if first := sum.Boots[0].FirstOK["search_symbols/primary"]; *first.FromStartMS.Value != 5600 || *first.FromReadyMS.Value != 600 { + t.Fatalf("initial first OK = %+v", first) + } + if d := sum.RestartReadyToFirstOK["search_symbols/primary"]; d.Count != 2 || *d.P95.Value != 1050 || *d.P50.Value != 0 { + t.Fatalf("restart ready to first OK = %+v", d) + } + metrics := sum.metrics() + for name, want := range map[string]float64{ + "startup_boots": 3, + "startup_linked_worktrees": 24, + "startup_repo_not_tracked_before_ready": 2, + "startup_nonconforming_loading_refusals": 2, + "startup_hook_denies_while_loading": 0, + "startup_hook_denies_after_ready": 1, + "startup_track_advice_while_loading": 2, + "startup_track_advice_without_socket": 1, + "startup_initial_time_to_ready_ms": 5000, + "startup_initial_first_ok_ms": 5600, + "startup_restart_time_to_ready_p95_ms": 1000, + "startup_restart_ready_to_first_ok_p95_ms": 1050, + "startup_coordinators_within_2s_min": 2, + } { + if m := metrics[name]; m.Value == nil || *m.Value != want { + t.Errorf("%s = %+v, want %v", name, m, want) + } + } + + // A restart that never answered must not vanish from the p95. + sum = summarizeStartup(samples[:len(samples)-1], boots, "app", 24) + if m := sum.metrics()["startup_restart_ready_to_first_ok_p95_ms"]; m.Value != nil || !strings.Contains(m.Reason, "1 of 2 restarts") { + t.Fatalf("missing restart observation = %+v", m) + } + boots[2].Coordinators = nil + if m := summarizeStartup(samples, boots, "app", 24).metrics()["startup_coordinators_within_2s_min"]; m.Value != nil { + t.Fatalf("a boot without a census cannot report a minimum: %+v", m) + } +} + +func TestStartupOptionsStayScoped(t *testing.T) { + base := []string{"--binary", "/b", "--manifest", "/m", "--scratch", "/s"} + o, err := parseRun(append(base, "--scenario", "startup", "--restarts", "10", "--seed-store", "/v21.db")) + if err != nil || o.Startup.Restarts != 10 || o.Startup.SeedStore != "/v21.db" { + t.Fatalf("startup options = %+v, %v", o.Startup, err) + } + if scenarios, _ := selectedScenarios(o); len(scenarios) != 1 || scenarios[0] != "startup" { + t.Fatalf("startup scenarios = %v", scenarios) + } + o, err = parseRun(append(base, "--scenario", "all")) + if scenarios, _ := selectedScenarios(o); err != nil || strings.Contains(strings.Join(scenarios, ","), "startup") { + t.Fatalf("all must not restart daemons on its own: %v %v", scenarios, err) + } + for _, args := range [][]string{{"--scenario", "read", "--restarts", "1"}, {"--scenario", "cold", "--seed-store", "/v21.db"}, {"--scenario", "startup", "--restarts", "-1"}} { + if _, err := parseRun(append(base, args...)); err == nil { + t.Errorf("parseRun(%v) accepted startup options outside their scope", args) + } + } +} + +// The whole scenario against the fixture daemon: probing starts before the +// first socket, keeps sampling through a restart, and the gate, metrics and +// per-second log come from that one run. +func TestStartupScenarioProbesAcrossARestart(t *testing.T) { + t.Setenv("LOAD_DAEMON_FIXTURE", "serve") + corpus := gitFixture(t) + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + options := runOptions{Binary: binary, Profile: "agent", Host: "native", Agents: 1, Worktrees: 1, Duration: time.Second, Deadline: 2 * time.Second, WarmTimeout: 20 * time.Second, StopTimeout: 5 * time.Second, KeepState: true, Startup: startupOptions{Restarts: 1}} + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + phase, _, artifacts, crashes, version, err := runPhase(ctx, options, startupScenario, []Corpus{corpus}, "", t.TempDir()) + if err != nil { + t.Fatalf("startup scenario: %v %+v", err, phase.Errors) + } + var summary startupSummary + if err := json.Unmarshal(phase.Gate["startup_probe"], &summary); err != nil { + t.Fatal(err) + } + if version != "fixture-v1" || len(summary.Boots) != 2 || len(phase.Status) != 1 || len(phase.Stats) != 1 || crashes["panic_lines"] != 0 { + t.Fatalf("version=%q boots=%d status=%d stats=%d crashes=%v", version, len(summary.Boots), len(phase.Status), len(phase.Stats), crashes) + } + for _, key := range []string{"initialize/primary", "initialize_facade/worktree", "search_symbols/primary", "search_symbols/worktree", "search_symbols/router", "index_health_resource/router", "workspace_index/primary", "hook_read/worktree", "cli_pipe/primary"} { + answered := 0 + for _, outcomes := range summary.Outcomes[key] { + answered += outcomes["ok"] + outcomes["allow"] + } + if answered == 0 { + t.Errorf("%s has no successful answer: %v", key, summary.Outcomes[key]) + } + } + for _, name := range []string{"startup_initial_first_ok_ms", "startup_restart_ready_to_first_ok_p95_ms", "startup_restart_time_to_ready_p95_ms"} { + if phase.Metrics[name].Value == nil { + t.Errorf("%s unavailable: %s", name, phase.Metrics[name].Reason) + } + } + if m := phase.Metrics["startup_coordinators_within_2s_min"]; m.Value == nil || *m.Value != 1 || *phase.Metrics["startup_linked_worktrees"].Value != 1 { + t.Errorf("coordinator census = %+v", m) + } + log, err := os.ReadFile(artifacts["startup_probe_log"]) + if err != nil || strings.Count(string(log), "\n") < 10 { + t.Fatalf("per-second log: %d bytes, %v", len(log), err) + } + roster, err := os.ReadFile(filepath.Join(artifacts["private_root"], "config", "gortex", "servers.toml")) + if err != nil || !strings.Contains(string(roster), startupRosterWorkspace) { + t.Fatalf("router roster: %s %v", roster, err) + } + claim, err := os.ReadFile(filepath.Join(artifacts["private_root"], "repos", corpus.Label, ".gortex-probe-router", ".gortex.yaml")) + if err != nil || string(claim) != "workspace: "+startupRosterWorkspace+"\n" { + t.Fatalf("router cwd claim: %q %v", claim, err) + } +} diff --git a/bench/load/testdata/edit-probe/python/main.py b/bench/load/testdata/edit-probe/python/main.py new file mode 100644 index 000000000..ded84c90a --- /dev/null +++ b/bench/load/testdata/edit-probe/python/main.py @@ -0,0 +1,29 @@ +def Example(): + return 1 + +def Caller(): + return Example() + +def FixtureOne(): + return 1 + +def FixtureTwo(): + return 2 + +def FixtureThree(): + return 3 + +def FixtureFour(): + return 4 + +def FixtureFive(): + return 5 + +def FixtureSix(): + return 6 + +def FixtureSeven(): + return 7 + +def FixtureEight(): + return 8 diff --git a/bench/load/testdata/edit-probe/typescript/main.ts b/bench/load/testdata/edit-probe/typescript/main.ts new file mode 100644 index 000000000..c85ee6e4c --- /dev/null +++ b/bench/load/testdata/edit-probe/typescript/main.ts @@ -0,0 +1,10 @@ +export function Example(): number { return 1; } +export function Caller(): number { return Example(); } +export function FixtureOne(): number { return 1; } +export function FixtureTwo(): number { return 2; } +export function FixtureThree(): number { return 3; } +export function FixtureFour(): number { return 4; } +export function FixtureFive(): number { return 5; } +export function FixtureSix(): number { return 6; } +export function FixtureSeven(): number { return 7; } +export function FixtureEight(): number { return 8; } diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index f7ca87411..56717f6fb 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -41,9 +41,39 @@ "edit_anchor": { "type": "string" }, + "edit_before_template": { + "type": "string" + }, + "edit_callee": { + "type": "string" + }, + "edit_destination": { + "type": "string" + }, + "edit_destination_content": { + "type": "string" + }, + "edit_expect": { + "type": "string" + }, + "edit_initial_content": { + "type": "string" + }, + "edit_kind": { + "type": "string" + }, + "edit_output": { + "type": "string" + }, + "edit_sender": { + "type": "string" + }, "edit_template": { "type": "string" }, + "edit_tool": { + "type": "string" + }, "file": { "type": "string" }, @@ -234,6 +264,9 @@ "error": { "type": "string" }, + "expected_outcome": { + "type": "string" + }, "logical_ms": { "type": "number" }, @@ -419,6 +452,2300 @@ ], "type": "object" }, + "edit_probe": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "accounting_required": { + "type": "boolean" + }, + "edits": { + "items": { + "additionalProperties": false, + "properties": { + "accounting": { + "additionalProperties": false, + "properties": { + "acknowledgement": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "applied_generation": { + "type": "integer" + }, + "bound_rpc_id": { + "type": "string" + }, + "physical": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "read_assisted_publisher": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "reasons": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "requested_background": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "requested_generation": { + "type": "integer" + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "bound_rpc_id", + "requested_generation", + "applied_generation", + "acknowledgement", + "physical", + "requested_background", + "reasons" + ], + "type": "object" + }, + "activated_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "batch": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "acknowledgement_receipt": { + "type": "string" + }, + "acknowledgement_state": { + "type": "string" + }, + "after_sha256": { + "type": "string" + }, + "file_observation_receipt": { + "type": "string" + }, + "path": { + "type": "string" + }, + "reindex_generation": { + "type": "integer" + }, + "reindex_receipt": { + "type": "string" + }, + "relative_path": { + "type": "string" + }, + "result_reindex_generation": { + "type": "integer" + }, + "result_reindex_receipt": { + "type": "string" + }, + "transaction_error": { + "type": "string" + }, + "transaction_id": { + "type": "string" + } + }, + "required": [ + "transaction_id", + "file_observation_receipt", + "acknowledgement_receipt", + "path", + "relative_path", + "after_sha256", + "reindex_receipt", + "reindex_generation", + "result_reindex_receipt", + "result_reindex_generation", + "acknowledgement_state" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "editor": { + "type": "integer" + }, + "errors": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "evidence_bytes": { + "type": "integer" + }, + "evidence_encode_ms": { + "type": "number" + }, + "evidence_limit_bytes": { + "type": "integer" + }, + "evidence_write_ms": { + "type": "number" + }, + "expected_refusals": { + "type": "integer" + }, + "first_correct_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "graph_terminal": { + "type": "boolean" + }, + "graph_terminal_basis": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "move": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "files": { + "items": { + "additionalProperties": false, + "properties": { + "path": { + "type": "string" + }, + "receipt": { + "type": "string" + }, + "role": { + "type": "string" + } + }, + "required": [ + "path", + "role", + "receipt" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "operation_id": { + "type": "string" + }, + "reference_error": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "target_id": { + "type": "string" + } + }, + "required": [ + "operation_id", + "source_id", + "target_id", + "files" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "move_accounting": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "ack_rpc_id": { + "type": "string" + }, + "acknowledgement": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "files": { + "items": { + "additionalProperties": false, + "properties": { + "accepted_blob_sha": { + "type": "string" + }, + "direct_reconcile": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "owner_generation": { + "type": "integer" + }, + "path": { + "type": "string" + }, + "publication_route": { + "type": "string" + }, + "read_assisted_reconcile": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "receipt": { + "type": "string" + }, + "role": { + "type": "string" + }, + "write_returned_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_started_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_window_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "path", + "role", + "receipt", + "accepted_blob_sha", + "owner_generation", + "write_started_ns", + "write_returned_ns", + "write_window_ns" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "reasons": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "ack_rpc_id", + "files" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "mutation_id": { + "type": "string" + }, + "mutation_receipt": { + "type": "string" + }, + "native_write": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "after": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "inode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "mtime_ns": { + "type": "integer" + }, + "size": { + "type": "integer" + } + }, + "required": [ + "size", + "mtime_ns", + "inode" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "after_ordinal": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "before": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "inode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "mtime_ns": { + "type": "integer" + }, + "size": { + "type": "integer" + } + }, + "required": [ + "size", + "mtime_ns", + "inode" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "dependency_ready_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "replaced": { + "type": "boolean" + }, + "sender": { + "type": "string" + }, + "write_end_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_start_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_window_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "sender", + "write_start_ns", + "write_end_ns", + "write_window_ms", + "before", + "after", + "replaced" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "ordinal": { + "type": "integer" + }, + "outcome": { + "type": "string" + }, + "phases_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "planned_due_ns": { + "type": "integer" + }, + "precommit_fresh_mismatches_unassessed": { + "type": "integer" + }, + "probe_errors": { + "type": "integer" + }, + "raw_evidence": { + "type": "string" + }, + "readbacks": { + "items": { + "additionalProperties": false, + "properties": { + "correct": { + "type": "boolean" + }, + "error": { + "type": "string" + }, + "expected_outcome": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "latency_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "observation_only": { + "type": "boolean" + }, + "refusal": { + "type": "string" + } + }, + "required": [ + "kind", + "latency_ms", + "correct", + "error" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "reads_while_receipt_observed_pending": { + "type": "integer" + }, + "restart": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "disk_has_marker": { + "type": "boolean" + }, + "downtime_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "first_correct_before_kill_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "first_correct_from_ready_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "kill_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "qualification": { + "type": "string" + }, + "ready_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "receipt_pending_before_kill": { + "type": "boolean" + }, + "status_after_restart": {}, + "status_after_restart_error": { + "type": "string" + }, + "status_before_kill": {} + }, + "required": [ + "receipt_pending_before_kill", + "status_before_kill", + "kill_ns", + "ready_ns", + "downtime_ms", + "status_after_restart", + "first_correct_before_kill_ms", + "first_correct_from_ready_ms", + "disk_has_marker" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "stale_claimed_fresh": { + "type": "boolean" + }, + "start_lateness_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "started_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "t_ack_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_derived_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_file_graph_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_readback_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_relations_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_search_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "ordinal", + "marker", + "mutation_id", + "planned_due_ns", + "activated_ns", + "started_ns", + "start_lateness_ns", + "outcome", + "mutation_receipt", + "t_ack_ms", + "readbacks", + "first_correct_ms", + "accounting", + "reads_while_receipt_observed_pending", + "probe_errors", + "stale_claimed_fresh", + "precommit_fresh_mismatches_unassessed", + "graph_terminal", + "raw_evidence", + "evidence_write_ms", + "evidence_encode_ms", + "evidence_bytes", + "evidence_limit_bytes", + "errors" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "limitations": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "output_modes": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "claims_fresh": { + "type": "integer" + }, + "claims_fresh_observable": { + "type": "boolean" + }, + "decode_failures": { + "type": "integer" + }, + "precommit_fresh_mismatches_unassessed": { + "type": "integer" + }, + "probe_answers": { + "type": "integer" + }, + "reason": { + "type": "string" + }, + "stale_claimed_fresh": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "probe_answers", + "decode_failures", + "claims_fresh_observable", + "claims_fresh", + "stale_claimed_fresh", + "precommit_fresh_mismatches_unassessed" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "plan": { + "additionalProperties": false, + "properties": { + "arrival": { + "type": "string" + }, + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "editors": { + "type": "integer" + }, + "file": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mean_interarrival_ns": { + "type": "integer" + }, + "mode": { + "type": "string" + }, + "no_edit_window_ns": { + "type": "integer" + }, + "not_selected_not_tested": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "probe_period_ns": { + "type": "integer" + }, + "readers": { + "type": "integer" + }, + "repo": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "sender": { + "type": "string" + }, + "session_model": { + "type": "string" + }, + "steps": { + "items": { + "additionalProperties": false, + "properties": { + "activate_ns": { + "type": "integer" + }, + "after": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "after_event": { + "type": "string" + }, + "before_symbol": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "destination_file": { + "type": "string" + }, + "due_ns": { + "type": "integer" + }, + "editor": { + "type": "integer" + }, + "expected_line_gap": { + "type": "integer" + }, + "expected_refusal": { + "type": "boolean" + }, + "file": { + "type": "string" + }, + "following_symbol": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "mutation_id": { + "type": "string" + }, + "ordinal": { + "type": "integer" + }, + "relocated": { + "type": "boolean" + }, + "request": { + "additionalProperties": false, + "properties": { + "arguments": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, + "expected_absent_symbol": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "operation", + "arguments" + ], + "type": "object" + }, + "request_sha256": { + "type": "string" + }, + "sender": { + "type": "string" + }, + "target_symbol": { + "type": "string" + } + }, + "required": [ + "ordinal", + "activate_ns", + "due_ns", + "marker", + "mutation_id", + "request", + "request_sha256" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "tool": { + "type": "string" + }, + "version": { + "type": "integer" + } + }, + "required": [ + "version", + "arrival", + "mean_interarrival_ns", + "probe_period_ns", + "count", + "seed", + "kind", + "tool", + "language", + "mode", + "repo", + "commit", + "file", + "callee", + "session_model", + "not_selected_not_tested", + "steps" + ], + "type": "object" + }, + "plan_sha256": { + "type": "string" + }, + "reads": { + "additionalProperties": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "busy_refusals": { + "type": "integer" + }, + "calls": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "errors": { + "type": "integer" + }, + "latency_ms": { + "additionalProperties": false, + "properties": { + "bootstrap_resamples": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "max": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "not_applicable": { + "type": "integer" + }, + "p50": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p50_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p95": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p95_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p99": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p99_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + } + }, + "required": [ + "count", + "censored", + "not_applicable", + "p50", + "p95", + "p99", + "max", + "p50_ci95", + "p95_ci95", + "p99_ci95", + "bootstrap_resamples" + ], + "type": "object" + } + }, + "required": [ + "calls", + "busy_refusals", + "errors", + "censored", + "latency_ms" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "type": [ + "object", + "null" + ] + }, + "restore_batch": {}, + "restore_batch_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "role": { + "type": "string" + }, + "stats_after": {}, + "stats_before": {}, + "summary": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "measures": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "bootstrap_resamples": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "max": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "not_applicable": { + "type": "integer" + }, + "p50": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p50_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p95": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p95_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p99": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p99_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + } + }, + "required": [ + "count", + "censored", + "not_applicable", + "p50", + "p95", + "p99", + "max", + "p50_ci95", + "p95_ci95", + "p99_ci95", + "bootstrap_resamples" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "offered": { + "type": "integer" + }, + "outcomes": { + "additionalProperties": { + "type": "integer" + }, + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "offered", + "outcomes", + "measures" + ], + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "plan", + "plan_sha256", + "accounting_required", + "edits", + "restore_batch_ms", + "restore_batch", + "stats_before", + "stats_after", + "limitations" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, "edits": { "items": { "additionalProperties": false, @@ -557,6 +2884,9 @@ "null" ] }, + "expected_absent_symbol": { + "type": "string" + }, "operation": { "type": "string" }, @@ -592,6 +2922,12 @@ "null" ] }, + "expected_refusals": { + "type": "integer" + }, + "harness_incomplete": { + "type": "integer" + }, "latency_ms": { "additionalProperties": false, "properties": { @@ -722,6 +3058,9 @@ "product_expired": { "type": "integer" }, + "retryable_refusals": { + "type": "integer" + }, "successes": { "type": "integer" }, @@ -792,6 +3131,12 @@ "null" ] }, + "expected_refusals": { + "type": "integer" + }, + "harness_incomplete": { + "type": "integer" + }, "latency_ms": { "additionalProperties": false, "properties": { @@ -922,6 +3267,9 @@ "product_expired": { "type": "integer" }, + "retryable_refusals": { + "type": "integer" + }, "successes": { "type": "integer" }, @@ -1188,6 +3536,216 @@ "edit_agents": { "type": "integer" }, + "edit_probe": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "arrival": { + "type": "string" + }, + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "editors": { + "type": "integer" + }, + "file": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mean_interarrival_ns": { + "type": "integer" + }, + "mode": { + "type": "string" + }, + "no_edit_window_ns": { + "type": "integer" + }, + "not_selected_not_tested": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "probe_period_ns": { + "type": "integer" + }, + "readers": { + "type": "integer" + }, + "repo": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "sender": { + "type": "string" + }, + "session_model": { + "type": "string" + }, + "steps": { + "items": { + "additionalProperties": false, + "properties": { + "activate_ns": { + "type": "integer" + }, + "after": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "after_event": { + "type": "string" + }, + "before_symbol": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "destination_file": { + "type": "string" + }, + "due_ns": { + "type": "integer" + }, + "editor": { + "type": "integer" + }, + "expected_line_gap": { + "type": "integer" + }, + "expected_refusal": { + "type": "boolean" + }, + "file": { + "type": "string" + }, + "following_symbol": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "mutation_id": { + "type": "string" + }, + "ordinal": { + "type": "integer" + }, + "relocated": { + "type": "boolean" + }, + "request": { + "additionalProperties": false, + "properties": { + "arguments": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, + "expected_absent_symbol": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "operation", + "arguments" + ], + "type": "object" + }, + "request_sha256": { + "type": "string" + }, + "sender": { + "type": "string" + }, + "target_symbol": { + "type": "string" + } + }, + "required": [ + "ordinal", + "activate_ns", + "due_ns", + "marker", + "mutation_id", + "request", + "request_sha256" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "tool": { + "type": "string" + }, + "version": { + "type": "integer" + } + }, + "required": [ + "version", + "arrival", + "mean_interarrival_ns", + "probe_period_ns", + "count", + "seed", + "kind", + "tool", + "language", + "mode", + "repo", + "commit", + "file", + "callee", + "session_model", + "not_selected_not_tested", + "steps" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, "keep_state": { "type": "boolean" }, @@ -1205,6 +3763,9 @@ "null" ] }, + "expected_absent_symbol": { + "type": "string" + }, "operation": { "type": "string" }, @@ -1497,6 +4058,9 @@ "error": { "type": "string" }, + "expected_outcome": { + "type": "string" + }, "logical_latency_ms": { "additionalProperties": false, "properties": { @@ -1568,6 +4132,9 @@ "commit": { "type": "string" }, + "expected_outcome": { + "type": "string" + }, "repo": { "type": "string" }, diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index 676338e8b..ac950dfa7 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -28,9 +28,39 @@ "edit_anchor": { "type": "string" }, + "edit_before_template": { + "type": "string" + }, + "edit_callee": { + "type": "string" + }, + "edit_destination": { + "type": "string" + }, + "edit_destination_content": { + "type": "string" + }, + "edit_expect": { + "type": "string" + }, + "edit_initial_content": { + "type": "string" + }, + "edit_kind": { + "type": "string" + }, + "edit_output": { + "type": "string" + }, + "edit_sender": { + "type": "string" + }, "edit_template": { "type": "string" }, + "edit_tool": { + "type": "string" + }, "file": { "type": "string" }, @@ -221,6 +251,9 @@ "error": { "type": "string" }, + "expected_outcome": { + "type": "string" + }, "logical_ms": { "type": "number" }, @@ -406,6 +439,2300 @@ ], "type": "object" }, + "edit_probe": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "accounting_required": { + "type": "boolean" + }, + "edits": { + "items": { + "additionalProperties": false, + "properties": { + "accounting": { + "additionalProperties": false, + "properties": { + "acknowledgement": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "applied_generation": { + "type": "integer" + }, + "bound_rpc_id": { + "type": "string" + }, + "physical": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "read_assisted_publisher": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "reasons": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "requested_background": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "requested_generation": { + "type": "integer" + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "bound_rpc_id", + "requested_generation", + "applied_generation", + "acknowledgement", + "physical", + "requested_background", + "reasons" + ], + "type": "object" + }, + "activated_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "batch": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "acknowledgement_receipt": { + "type": "string" + }, + "acknowledgement_state": { + "type": "string" + }, + "after_sha256": { + "type": "string" + }, + "file_observation_receipt": { + "type": "string" + }, + "path": { + "type": "string" + }, + "reindex_generation": { + "type": "integer" + }, + "reindex_receipt": { + "type": "string" + }, + "relative_path": { + "type": "string" + }, + "result_reindex_generation": { + "type": "integer" + }, + "result_reindex_receipt": { + "type": "string" + }, + "transaction_error": { + "type": "string" + }, + "transaction_id": { + "type": "string" + } + }, + "required": [ + "transaction_id", + "file_observation_receipt", + "acknowledgement_receipt", + "path", + "relative_path", + "after_sha256", + "reindex_receipt", + "reindex_generation", + "result_reindex_receipt", + "result_reindex_generation", + "acknowledgement_state" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "editor": { + "type": "integer" + }, + "errors": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "evidence_bytes": { + "type": "integer" + }, + "evidence_encode_ms": { + "type": "number" + }, + "evidence_limit_bytes": { + "type": "integer" + }, + "evidence_write_ms": { + "type": "number" + }, + "expected_refusals": { + "type": "integer" + }, + "first_correct_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "graph_terminal": { + "type": "boolean" + }, + "graph_terminal_basis": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "move": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "files": { + "items": { + "additionalProperties": false, + "properties": { + "path": { + "type": "string" + }, + "receipt": { + "type": "string" + }, + "role": { + "type": "string" + } + }, + "required": [ + "path", + "role", + "receipt" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "operation_id": { + "type": "string" + }, + "reference_error": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "target_id": { + "type": "string" + } + }, + "required": [ + "operation_id", + "source_id", + "target_id", + "files" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "move_accounting": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "ack_rpc_id": { + "type": "string" + }, + "acknowledgement": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "files": { + "items": { + "additionalProperties": false, + "properties": { + "accepted_blob_sha": { + "type": "string" + }, + "direct_reconcile": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "owner_generation": { + "type": "integer" + }, + "path": { + "type": "string" + }, + "publication_route": { + "type": "string" + }, + "read_assisted_reconcile": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "residual_ns": { + "type": "integer" + }, + "sum_ns": { + "type": "integer" + }, + "wall_ns": { + "type": "integer" + }, + "within_five_percent": { + "type": "boolean" + } + }, + "required": [ + "wall_ns", + "sum_ns", + "residual_ns", + "within_five_percent" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "receipt": { + "type": "string" + }, + "role": { + "type": "string" + }, + "write_returned_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_started_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_window_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "path", + "role", + "receipt", + "accepted_blob_sha", + "owner_generation", + "write_started_ns", + "write_returned_ns", + "write_window_ns" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "reasons": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "ack_rpc_id", + "files" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "mutation_id": { + "type": "string" + }, + "mutation_receipt": { + "type": "string" + }, + "native_write": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "after": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "inode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "mtime_ns": { + "type": "integer" + }, + "size": { + "type": "integer" + } + }, + "required": [ + "size", + "mtime_ns", + "inode" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "after_ordinal": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "before": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "inode": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "mtime_ns": { + "type": "integer" + }, + "size": { + "type": "integer" + } + }, + "required": [ + "size", + "mtime_ns", + "inode" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "dependency_ready_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "replaced": { + "type": "boolean" + }, + "sender": { + "type": "string" + }, + "write_end_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_start_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "write_window_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "sender", + "write_start_ns", + "write_end_ns", + "write_window_ms", + "before", + "after", + "replaced" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "ordinal": { + "type": "integer" + }, + "outcome": { + "type": "string" + }, + "phases_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "planned_due_ns": { + "type": "integer" + }, + "precommit_fresh_mismatches_unassessed": { + "type": "integer" + }, + "probe_errors": { + "type": "integer" + }, + "raw_evidence": { + "type": "string" + }, + "readbacks": { + "items": { + "additionalProperties": false, + "properties": { + "correct": { + "type": "boolean" + }, + "error": { + "type": "string" + }, + "expected_outcome": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "latency_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "observation_only": { + "type": "boolean" + }, + "refusal": { + "type": "string" + } + }, + "required": [ + "kind", + "latency_ms", + "correct", + "error" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "reads_while_receipt_observed_pending": { + "type": "integer" + }, + "restart": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "disk_has_marker": { + "type": "boolean" + }, + "downtime_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "first_correct_before_kill_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "first_correct_from_ready_ms": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "kill_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "qualification": { + "type": "string" + }, + "ready_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "receipt_pending_before_kill": { + "type": "boolean" + }, + "status_after_restart": {}, + "status_after_restart_error": { + "type": "string" + }, + "status_before_kill": {} + }, + "required": [ + "receipt_pending_before_kill", + "status_before_kill", + "kill_ns", + "ready_ns", + "downtime_ms", + "status_after_restart", + "first_correct_before_kill_ms", + "first_correct_from_ready_ms", + "disk_has_marker" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, + "stale_claimed_fresh": { + "type": "boolean" + }, + "start_lateness_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "started_ns": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "t_ack_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_derived_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_file_graph_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_readback_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_relations_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "t_search_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "ordinal", + "marker", + "mutation_id", + "planned_due_ns", + "activated_ns", + "started_ns", + "start_lateness_ns", + "outcome", + "mutation_receipt", + "t_ack_ms", + "readbacks", + "first_correct_ms", + "accounting", + "reads_while_receipt_observed_pending", + "probe_errors", + "stale_claimed_fresh", + "precommit_fresh_mismatches_unassessed", + "graph_terminal", + "raw_evidence", + "evidence_write_ms", + "evidence_encode_ms", + "evidence_bytes", + "evidence_limit_bytes", + "errors" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "limitations": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "output_modes": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "claims_fresh": { + "type": "integer" + }, + "claims_fresh_observable": { + "type": "boolean" + }, + "decode_failures": { + "type": "integer" + }, + "precommit_fresh_mismatches_unassessed": { + "type": "integer" + }, + "probe_answers": { + "type": "integer" + }, + "reason": { + "type": "string" + }, + "stale_claimed_fresh": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "state": { + "type": "string" + } + }, + "required": [ + "state", + "probe_answers", + "decode_failures", + "claims_fresh_observable", + "claims_fresh", + "stale_claimed_fresh", + "precommit_fresh_mismatches_unassessed" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "plan": { + "additionalProperties": false, + "properties": { + "arrival": { + "type": "string" + }, + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "editors": { + "type": "integer" + }, + "file": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mean_interarrival_ns": { + "type": "integer" + }, + "mode": { + "type": "string" + }, + "no_edit_window_ns": { + "type": "integer" + }, + "not_selected_not_tested": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "probe_period_ns": { + "type": "integer" + }, + "readers": { + "type": "integer" + }, + "repo": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "sender": { + "type": "string" + }, + "session_model": { + "type": "string" + }, + "steps": { + "items": { + "additionalProperties": false, + "properties": { + "activate_ns": { + "type": "integer" + }, + "after": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "after_event": { + "type": "string" + }, + "before_symbol": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "destination_file": { + "type": "string" + }, + "due_ns": { + "type": "integer" + }, + "editor": { + "type": "integer" + }, + "expected_line_gap": { + "type": "integer" + }, + "expected_refusal": { + "type": "boolean" + }, + "file": { + "type": "string" + }, + "following_symbol": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "mutation_id": { + "type": "string" + }, + "ordinal": { + "type": "integer" + }, + "relocated": { + "type": "boolean" + }, + "request": { + "additionalProperties": false, + "properties": { + "arguments": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, + "expected_absent_symbol": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "operation", + "arguments" + ], + "type": "object" + }, + "request_sha256": { + "type": "string" + }, + "sender": { + "type": "string" + }, + "target_symbol": { + "type": "string" + } + }, + "required": [ + "ordinal", + "activate_ns", + "due_ns", + "marker", + "mutation_id", + "request", + "request_sha256" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "tool": { + "type": "string" + }, + "version": { + "type": "integer" + } + }, + "required": [ + "version", + "arrival", + "mean_interarrival_ns", + "probe_period_ns", + "count", + "seed", + "kind", + "tool", + "language", + "mode", + "repo", + "commit", + "file", + "callee", + "session_model", + "not_selected_not_tested", + "steps" + ], + "type": "object" + }, + "plan_sha256": { + "type": "string" + }, + "reads": { + "additionalProperties": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "busy_refusals": { + "type": "integer" + }, + "calls": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "errors": { + "type": "integer" + }, + "latency_ms": { + "additionalProperties": false, + "properties": { + "bootstrap_resamples": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "max": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "not_applicable": { + "type": "integer" + }, + "p50": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p50_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p95": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p95_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p99": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p99_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + } + }, + "required": [ + "count", + "censored", + "not_applicable", + "p50", + "p95", + "p99", + "max", + "p50_ci95", + "p95_ci95", + "p99_ci95", + "bootstrap_resamples" + ], + "type": "object" + } + }, + "required": [ + "calls", + "busy_refusals", + "errors", + "censored", + "latency_ms" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "type": [ + "object", + "null" + ] + }, + "restore_batch": {}, + "restore_batch_ms": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "role": { + "type": "string" + }, + "stats_after": {}, + "stats_before": {}, + "summary": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "measures": { + "additionalProperties": { + "additionalProperties": false, + "properties": { + "bootstrap_resamples": { + "type": "integer" + }, + "censored": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "max": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "not_applicable": { + "type": "integer" + }, + "p50": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p50_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p95": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p95_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + }, + "p99": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "p99_ci95": { + "additionalProperties": false, + "properties": { + "high": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + }, + "low": { + "additionalProperties": false, + "properties": { + "reason": { + "type": "string" + }, + "unit": { + "type": "string" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "value", + "unit", + "reason" + ], + "type": "object" + } + }, + "required": [ + "low", + "high" + ], + "type": "object" + } + }, + "required": [ + "count", + "censored", + "not_applicable", + "p50", + "p95", + "p99", + "max", + "p50_ci95", + "p95_ci95", + "p99_ci95", + "bootstrap_resamples" + ], + "type": "object" + }, + "type": [ + "object", + "null" + ] + }, + "offered": { + "type": "integer" + }, + "outcomes": { + "additionalProperties": { + "type": "integer" + }, + "type": [ + "object", + "null" + ] + } + }, + "required": [ + "offered", + "outcomes", + "measures" + ], + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "plan", + "plan_sha256", + "accounting_required", + "edits", + "restore_batch_ms", + "restore_batch", + "stats_before", + "stats_after", + "limitations" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, "edits": { "items": { "additionalProperties": false, @@ -544,6 +2871,9 @@ "null" ] }, + "expected_absent_symbol": { + "type": "string" + }, "operation": { "type": "string" }, @@ -579,6 +2909,12 @@ "null" ] }, + "expected_refusals": { + "type": "integer" + }, + "harness_incomplete": { + "type": "integer" + }, "latency_ms": { "additionalProperties": false, "properties": { @@ -709,6 +3045,9 @@ "product_expired": { "type": "integer" }, + "retryable_refusals": { + "type": "integer" + }, "successes": { "type": "integer" }, @@ -779,6 +3118,12 @@ "null" ] }, + "expected_refusals": { + "type": "integer" + }, + "harness_incomplete": { + "type": "integer" + }, "latency_ms": { "additionalProperties": false, "properties": { @@ -909,6 +3254,9 @@ "product_expired": { "type": "integer" }, + "retryable_refusals": { + "type": "integer" + }, "successes": { "type": "integer" }, @@ -1175,6 +3523,216 @@ "edit_agents": { "type": "integer" }, + "edit_probe": { + "anyOf": [ + { + "additionalProperties": false, + "properties": { + "arrival": { + "type": "string" + }, + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "editors": { + "type": "integer" + }, + "file": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mean_interarrival_ns": { + "type": "integer" + }, + "mode": { + "type": "string" + }, + "no_edit_window_ns": { + "type": "integer" + }, + "not_selected_not_tested": { + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "probe_period_ns": { + "type": "integer" + }, + "readers": { + "type": "integer" + }, + "repo": { + "type": "string" + }, + "seed": { + "type": "integer" + }, + "sender": { + "type": "string" + }, + "session_model": { + "type": "string" + }, + "steps": { + "items": { + "additionalProperties": false, + "properties": { + "activate_ns": { + "type": "integer" + }, + "after": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + }, + "after_event": { + "type": "string" + }, + "before_symbol": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "destination_file": { + "type": "string" + }, + "due_ns": { + "type": "integer" + }, + "editor": { + "type": "integer" + }, + "expected_line_gap": { + "type": "integer" + }, + "expected_refusal": { + "type": "boolean" + }, + "file": { + "type": "string" + }, + "following_symbol": { + "type": "string" + }, + "marker": { + "type": "string" + }, + "mutation_id": { + "type": "string" + }, + "ordinal": { + "type": "integer" + }, + "relocated": { + "type": "boolean" + }, + "request": { + "additionalProperties": false, + "properties": { + "arguments": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, + "expected_absent_symbol": { + "type": "string" + }, + "operation": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "tool", + "operation", + "arguments" + ], + "type": "object" + }, + "request_sha256": { + "type": "string" + }, + "sender": { + "type": "string" + }, + "target_symbol": { + "type": "string" + } + }, + "required": [ + "ordinal", + "activate_ns", + "due_ns", + "marker", + "mutation_id", + "request", + "request_sha256" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, + "tool": { + "type": "string" + }, + "version": { + "type": "integer" + } + }, + "required": [ + "version", + "arrival", + "mean_interarrival_ns", + "probe_period_ns", + "count", + "seed", + "kind", + "tool", + "language", + "mode", + "repo", + "commit", + "file", + "callee", + "session_model", + "not_selected_not_tested", + "steps" + ], + "type": "object" + }, + { + "type": "null" + } + ] + }, "keep_state": { "type": "boolean" }, @@ -1192,6 +3750,9 @@ "null" ] }, + "expected_absent_symbol": { + "type": "string" + }, "operation": { "type": "string" }, diff --git a/bench/load/validate.go b/bench/load/validate.go index c828041b8..85fd3863a 100644 --- a/bench/load/validate.go +++ b/bench/load/validate.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "reflect" + "strings" ) // A bounded fixed-format check supplements decoding: absent fields are not @@ -17,7 +18,12 @@ func requireFields(data []byte, prototype any) error { } t := reflect.TypeOf(prototype) for i := range t.NumField() { - name := t.Field(i).Tag.Get("json") + field := t.Field(i) + tag := field.Tag.Get("json") + if field.PkgPath != "" || tag == "-" || strings.Contains(tag, ",omitempty") { + continue + } + name := strings.Split(tag, ",")[0] if _, ok := fields[name]; !ok { return fmt.Errorf("missing required %s field", name) } @@ -124,7 +130,7 @@ func validateReport(data []byte, r Report) error { if len(phase.Operations) == 0 || len(phase.Gate) == 0 || len(phase.Status) == 0 || len(phase.Stats) == 0 { return errors.New("complete phase is missing plan/gate/status/stats observations") } - if phase.Name != "cold" && phase.Name != "restart" && len(phase.Tools) == 0 { + if phase.Name != "cold" && phase.Name != "restart" && phase.Name != startupScenario && len(phase.Tools) == 0 { return errors.New("complete workload phase has no tool observations") } } @@ -133,7 +139,7 @@ func validateReport(data []byte, r Report) error { } for _, tool := range []map[string]ToolMetrics{phase.Tools, phase.Probes} { for name, row := range tool { - if name == "" || row.Successes < 0 || row.BusyRefusals < 0 || row.ProductExpired < 0 || row.PhaseCensored < 0 { + if name == "" || row.Successes < 0 || row.BusyRefusals < 0 || row.RetryableRefusals < 0 || row.HarnessIncomplete < 0 || row.ProductExpired < 0 || row.PhaseCensored < 0 { return errors.New("invalid tool identity/count") } if err := validMetric(row.SuccessPerSec); err != nil { diff --git a/cmd/gortex/cli_daemon.go b/cmd/gortex/cli_daemon.go index 018d601b7..9e6ee8047 100644 --- a/cmd/gortex/cli_daemon.go +++ b/cmd/gortex/cli_daemon.go @@ -140,7 +140,11 @@ func extractToolResult(resp []byte) (json.RawMessage, error) { if repo == "" { repo = data.RepoPath } - return nil, &repoLoadingError{repo: repo, phase: data.Phase, percent: data.Percent} + admissionReason := "" + if data.Reason == "identity_pending" || data.Reason == "scope_binding_pending" { + admissionReason = data.Reason + } + return nil, &repoLoadingError{repo: repo, phase: data.Phase, percent: data.Percent, admissionReason: admissionReason} case string(gortexmcp.ErrCodeRepoUnavailable): return nil, fmt.Errorf("the gortex daemon could not load %s after its restart (%s). %s", data.RepoPath, data.Reason, data.Suggestion) diff --git a/cmd/gortex/cli_loading_socket_test.go b/cmd/gortex/cli_loading_socket_test.go new file mode 100644 index 000000000..50d01764f --- /dev/null +++ b/cmd/gortex/cli_loading_socket_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/testenv" +) + +// Exercise the real CLI routing/probe/MCP/error path with piped stdout. Process +// startup is deliberately outside this bound; the runtime probes also measure +// the complete installed CLI invocation on the reference host. +func TestNonTTYCLI_LoadingSocketReturnsTempFailWithin100ms(t *testing.T) { + dispatcher, mi, _, loading := loadingDispatcherSetup(t) + controller := probeController(t, fmt.Sprintf("repos:\n - path: %q\n", loading)) + controller.multiIndexer = mi + controller.graph = mi.Graph() + dir := testenv.ShortTempDir(t) + socket := filepath.Join(dir, "s") + t.Setenv("GORTEX_DAEMON_SOCKET", socket) + t.Setenv("GORTEX_DAEMON_PIDFILE", filepath.Join(dir, "p")) + t.Setenv("GORTEX_DAEMON_STATEFILE", filepath.Join(dir, "state.json")) + t.Setenv("GORTEX_AUTOSTART", "0") + srv := daemon.New(socket, "test", zap.NewNop()) + srv.Controller = controller + srv.MCPDispatcher = dispatcher + require.NoError(t, srv.Listen()) + done := make(chan error, 1) + go func() { done <- srv.Serve() }() + t.Cleanup(func() { + require.NoError(t, srv.Shutdown()) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("owned CLI loading server did not stop") + } + }) + + reader, writer, err := os.Pipe() + require.NoError(t, err) + stdout := os.Stdout + os.Stdout = writer + flag := rootCmd.PersistentFlags().Lookup("loading-wait") + changed := flag.Changed + flag.Changed = false + t.Cleanup(func() { + os.Stdout = stdout + flag.Changed = changed + _ = writer.Close() + _ = reader.Close() + }) + require.Zero(t, effectiveLoadingWait(), "piped CLI stdout must not use the interactive wait") + + started := time.Now() + _, err = requireDaemonTool(loading, "search_symbols", map[string]any{"query": "LoadProbe"}) + elapsed := time.Since(started) + var coded interface{ ExitCode() int } + require.ErrorAs(t, err, &coded) + assert.Equal(t, exitTempFail, coded.ExitCode(), "execute must propagate the loading exit contract") + assert.LessOrEqual(t, elapsed, 100*time.Millisecond) + assert.Contains(t, err.Error(), "still loading") + assert.NotContains(t, err.Error(), "gortex track") + assert.NotContains(t, err.Error(), "repos reconcile") +} diff --git a/cmd/gortex/cli_repo_loading.go b/cmd/gortex/cli_repo_loading.go index d2b24a958..50a82b0fa 100644 --- a/cmd/gortex/cli_repo_loading.go +++ b/cmd/gortex/cli_repo_loading.go @@ -38,9 +38,10 @@ func init() { // It carries exit code 75, so a script or hook that runs the CLI can retry // later without parsing the message. type repoLoadingError struct { - repo string - phase string - percent int + admissionReason string + repo string + phase string + percent int // waited is how long the CLI already waited before giving up for now. waited time.Duration } @@ -50,6 +51,13 @@ func (e *repoLoadingError) Error() string { if e.waited > 0 { msg += fmt.Sprintf("; still loading after waiting %s", e.waited.Round(time.Second)) } + if e.admissionReason != "" { + msg = fmt.Sprintf("the gortex daemon is still resolving graph admission (repo_loading, %s)", e.admissionReason) + if e.waited > 0 { + msg += fmt.Sprintf("; still loading after waiting %s", e.waited.Round(time.Second)) + } + return msg + " — graph admission for this path is not yet established; retry shortly (or pass --loading-wait to wait for it)" + } return msg + " — the repository is already tracked; retry shortly (or pass --loading-wait to wait for it)" } diff --git a/cmd/gortex/daemon.go b/cmd/gortex/daemon.go index bcaffec35..ebeaf9cb9 100644 --- a/cmd/gortex/daemon.go +++ b/cmd/gortex/daemon.go @@ -352,6 +352,8 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { } warmupTask := newDaemonWarmup(cmd.Context()) defer warmupTask.cancel() + compactionTask := newDaemonWarmup(cmd.Context()) + defer compactionTask.cancel() // Teardown is wired into every exit path, not just the control-socket // one. A SIGINT/SIGTERM is handled inside the daemon server: it calls // Server.Shutdown directly, Serve returns, and the controller's hook is @@ -393,6 +395,7 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { // Revoke every independent producer before the first ownership join. startupReporter.Cancel() warmupTask.cancel() + compactionTask.cancel() contractAnalysis.Cancel() state.shared.BeginSemanticClose() state.shared.BeginBackgroundClose() @@ -409,6 +412,7 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { // the watcher handoff barrier: startup cannot attach/publish another // watcher after StopWatcher has finished. stopDaemonWarmup(warmupTask, controller, logger) + joinDaemonCompaction(compactionTask, logger) if stopJanitor != nil { started := time.Now() logger.Info("daemon: shutdown phase start", zap.String("phase", "reconcile_janitor")) @@ -652,7 +656,7 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { state.lifecycle.EnableDeferredSeedRetirements() } - if err := srv.Listen(); err != nil { + if err := listenConfiguredDaemon(srv, state); err != nil { startupReporter.Fail(err) return err } @@ -834,12 +838,14 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { state.mcpServer.MigrateSymbolAnchors(logger) migrationStep() } + scheduleDaemonCompaction(compactionTask, warmupTask, state, logger) }) // Serve returns directly on the signal path. Register this last so startup // is canceled before deferred health/janitor joins, not only when the later // once-guarded backend teardown is reached. defer warmupTask.cancel() + defer compactionTask.cancel() serveErr := srv.Serve() // Record a real error before deferred cleanup can complete both receipts. // The watchdog must preserve normal error exit codes if an outer finalizer @@ -1073,11 +1079,15 @@ func spawnDetachedDaemon(childArgs []string) error { } child := exec.Command(exe, append([]string{"daemon", "start"}, childArgs...)...) // The child re-parses the forwarded flags itself, so nothing has to - // travel by env any more except the marker that stops it detaching - // again (and stops it clearing a stop-intent written meanwhile). + // travel by env except the child marker and SQLite's process-start temp + // directory. Resolve that with the stack's own store-path rules. // Inherited GORTEX_* overrides keep the precedence they have on a // foreground start rather than being re-injected from a flag here. - child.Env = append(os.Environ(), "GORTEX_DAEMON_CHILD=1") + child.Env, err = detachedDaemonEnv(os.Environ(), daemonBackendPath) + if err != nil { + _ = logFile.Close() + return err + } child.Stdout = logFile child.Stderr = logFile child.Stdin = nil diff --git a/cmd/gortex/daemon_compact.go b/cmd/gortex/daemon_compact.go index 368170e96..64b0c2bbc 100644 --- a/cmd/gortex/daemon_compact.go +++ b/cmd/gortex/daemon_compact.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "os" "path/filepath" @@ -31,7 +32,7 @@ import ( // graph, test fixtures — simply do not implement it. type storeCompactor interface { CompactStats() (freeBytes, totalBytes int64) - Compact() error + CompactContext(context.Context) error Path() string } @@ -66,15 +67,23 @@ func shouldCompactStore(freeBytes, totalBytes int64, diskAvailBytes uint64) bool // maybeCompactStore probes the store for the compaction capability and runs // a one-time VACUUM when shouldCompactStore says the file is mostly dead -// space. Called from warmupDaemonState right after the orphan-prefix purge -// (so the purge's freshly-freed pages are measured and reclaimed in the same -// pass) and before the warmup re-index loop (whose writes would start -// reusing freelist pages and whose readers would contend with VACUUM's -// exclusive lock). Every failure path degrades to "skip": freelist pages -// stay reusable, so a missed compaction costs disk, never correctness. +// space. Production calls the context-aware form only after the entire warmup +// producer has joined and enrichment is published. Pages reused by indexing +// therefore reduce the measured freelist before this decision. func maybeCompactStore(g graph.Store, logger *zap.Logger) { + maybeCompactStoreContext(context.Background(), g, false, logger) +} + +func maybeCompactStoreContext(ctx context.Context, g graph.Store, migrated bool, logger *zap.Logger) { + if ctx.Err() != nil { + return + } + if migrated { + logger.Info("daemon: store compaction deferred to next boot", zap.String("reason", "after_migration")) + return + } if os.Getenv("GORTEX_SKIP_STORE_COMPACT") == "1" { - logger.Debug("daemon: store compaction disabled (GORTEX_SKIP_STORE_COMPACT=1)") + logger.Debug("daemon: store compaction disabled (GORTEX_SKIP_STORE_COMPACT=1)", zap.String("reason", "disabled")) return } c, ok := g.(storeCompactor) @@ -91,11 +100,12 @@ func maybeCompactStore(g graph.Store, logger *zap.Logger) { // Unknown headroom means the ×1.5 guard cannot hold; VACUUM without it // risks filling the volume mid-rewrite, so skip. logger.Debug("daemon: store compaction skipped — disk headroom unknown", - zap.String("path", path), zap.Error(err)) + zap.String("reason", "disk_headroom_unknown"), zap.String("path", path), zap.Error(err)) return } if !shouldCompactStore(free, total, avail) { logger.Debug("daemon: store compaction not warranted", + zap.String("reason", "below_threshold_after_warmup"), zap.Int64("reclaimable_bytes", free), zap.Int64("store_bytes", total), zap.Uint64("disk_avail_bytes", avail)) @@ -106,7 +116,7 @@ func maybeCompactStore(g graph.Store, logger *zap.Logger) { zap.Int64("store_bytes", total), zap.Uint64("disk_avail_bytes", avail)) start := time.Now() - if err := c.Compact(); err != nil { + if err := c.CompactContext(ctx); err != nil { // A deferral is not a failure and must not be logged as one: the store // was mid-publish or mid-build, the maintenance lane declined to // rewrite the file underneath it, and the freelist is still there to @@ -115,7 +125,16 @@ func maybeCompactStore(g graph.Store, logger *zap.Logger) { // that VACUUM actually tried and lost. if errors.Is(err, store_sqlite.ErrMaintenanceBusy) { logger.Info("daemon: store compaction deferred — the store was busy", - zap.Duration("elapsed", time.Since(start)), zap.Error(err)) + zap.String("reason", "busy"), zap.Duration("elapsed", time.Since(start)), zap.Error(err)) + return + } + if errors.Is(err, store_sqlite.ErrVacuumTempDirectory) { + logger.Info("daemon: store compaction skipped — temporary volume unconfirmed", + zap.String("reason", "temp_directory_unconfirmed"), zap.Error(err)) + return + } + if ctx.Err() != nil { + logger.Info("daemon: store compaction canceled", zap.Error(err)) return } // Non-fatal by design: a failed VACUUM leaves the store exactly as it @@ -130,3 +149,21 @@ func maybeCompactStore(g graph.Store, logger *zap.Logger) { zap.Int64("store_bytes", after), zap.Int64("reclaimed_bytes", total-after)) } + +// scheduleDaemonCompaction admits at most one owned task. Even the warmup's +// deferred memory-release tail finishes before it measures or rewrites the +// store. Cancellation and joining use the existing startup owner, so teardown +// cannot close the backend beneath the dedicated connection. +func scheduleDaemonCompaction(task, warmup *daemonWarmup, state *daemonState, logger *zap.Logger) bool { + return task.start(func(ctx context.Context) { + warmup.wait() + maybeCompactStoreContext(ctx, state.graph, state.migratedStore, logger) + }) +} + +func joinDaemonCompaction(task *daemonWarmup, logger *zap.Logger) { + started := time.Now() + logger.Info("daemon: shutdown phase start", zap.String("phase", "post_warmup_compaction")) + task.wait() + logger.Info("daemon: shutdown phase complete", zap.String("phase", "post_warmup_compaction"), zap.Duration("elapsed", time.Since(started))) +} diff --git a/cmd/gortex/daemon_compact_lifetime_test.go b/cmd/gortex/daemon_compact_lifetime_test.go new file mode 100644 index 000000000..b4f2c8eea --- /dev/null +++ b/cmd/gortex/daemon_compact_lifetime_test.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +type ownedCompactStub struct { + graph.Store + path string + stats atomic.Int32 + entered chan struct{} + canceled chan struct{} + release chan struct{} + joined atomic.Bool +} + +func (s *ownedCompactStub) Path() string { return s.path } +func (s *ownedCompactStub) CompactStats() (int64, int64) { + s.stats.Add(1) + return 1<<30 + 1, 2 << 30 +} +func (s *ownedCompactStub) CompactContext(ctx context.Context) error { + close(s.entered) + <-ctx.Done() + close(s.canceled) + <-s.release // Actual cleanup, not the cancellation request, owns completion. + s.joined.Store(true) + return ctx.Err() +} + +func TestDaemonCompactionWaitsForWarmupCompletionAndJoinsBeforeBackend(t *testing.T) { + t.Setenv("GORTEX_SKIP_STORE_COMPACT", "0") + stub := &ownedCompactStub{ + path: filepath.Join(t.TempDir(), "store.sqlite"), + entered: make(chan struct{}), canceled: make(chan struct{}), release: make(chan struct{}), + } + warmup, task := newDaemonWarmup(t.Context()), newDaemonWarmup(t.Context()) + finishWarmup, scheduled := make(chan struct{}), make(chan struct{}) + var finishOnce, releaseOnce sync.Once + finish := func() { finishOnce.Do(func() { close(finishWarmup) }) } + release := func() { releaseOnce.Do(func() { close(stub.release) }) } + t.Cleanup(func() { + task.cancel() + warmup.cancel() + finish() + release() + warmup.wait() + task.wait() + }) + core, logs := observer.New(zap.DebugLevel) + logger := zap.New(core) + state := &daemonState{graph: stub} + require.True(t, warmup.start(func(context.Context) { + logger.Info("daemon: enrichment complete") + scheduleDaemonCompaction(task, warmup, state, logger) + close(scheduled) + <-finishWarmup + })) + select { + case <-scheduled: + case <-time.After(5 * time.Second): + t.Fatal("warmup did not schedule its owned tail") + } + require.Zero(t, stub.stats.Load(), "even statistics must wait for the producer completion barrier") + require.False(t, scheduleDaemonCompaction(task, warmup, state, logger), "one attempt per boot") + finish() + select { + case <-stub.entered: + case <-time.After(5 * time.Second): + t.Fatalf("post-warmup compaction did not start: %v", logs.All()) + } + entries := logs.All() + require.Equal(t, "daemon: enrichment complete", entries[0].Message) + require.Equal(t, "daemon: one-time store compaction starting — VACUUM may take minutes on a multi-GB store", entries[1].Message) + closed := make(chan struct{}) + teardown := newDaemonTeardown(func() { task.cancel(); joinDaemonCompaction(task, logger) }, func() error { + if !stub.joined.Load() { + t.Error("backend close preceded admitted VACUUM completion") + } + close(closed) + return nil + }) + done := make(chan error, 1) + go func() { done <- teardown() }() + select { + case <-stub.canceled: + case <-time.After(5 * time.Second): + t.Fatal("shutdown failed to cancel compaction") + } + select { + case <-closed: + t.Fatal("canceled does not mean physically joined") + default: + } + release() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("completed compaction did not release backend teardown") + } +} + +func TestDaemonCanceledCompactionDoesNotMeasureOrLaunch(t *testing.T) { + warmup, task := newDaemonWarmup(t.Context()), newDaemonWarmup(t.Context()) + t.Cleanup(func() { task.cancel(); warmup.cancel(); warmup.wait(); task.wait() }) + task.cancel() + require.False(t, scheduleDaemonCompaction(task, warmup, &daemonState{}, zap.NewNop())) +} + +func TestMaybeCompactStoreBelowThresholdAfterWarmup(t *testing.T) { + t.Setenv("GORTEX_SKIP_STORE_COMPACT", "0") + core, logs := observer.New(zap.DebugLevel) + stub := &compactStub{path: filepath.Join(t.TempDir(), "store.sqlite"), free: 1, total: 100} + maybeCompactStoreContext(t.Context(), stub, false, zap.New(core)) + require.Zero(t, stub.compacts) + entry := logs.FilterMessage("daemon: store compaction not warranted").All() + require.Len(t, entry, 1) + require.Equal(t, "below_threshold_after_warmup", entry[0].ContextMap()["reason"]) +} + +// Inject only the threshold statistics; actual compaction still runs on the +// real boot's SQLite store. A unit test need not allocate a >1GiB freelist to +// prove that a real migration is remembered for one boot only. +type bootCompactStore struct { + graph.Store + file *store_sqlite.Store + calls int +} + +func (s *bootCompactStore) Path() string { return s.file.Path() } +func (s *bootCompactStore) CompactStats() (int64, int64) { return 1<<30 + 1, 2 << 30 } +func (s *bootCompactStore) CompactContext(ctx context.Context) error { + s.calls++ + return s.file.CompactContext(ctx) +} + +func compactionBootEnv(t *testing.T) { + t.Helper() + t.Setenv("GORTEX_SKIP_STORE_COMPACT", "0") + root := t.TempDir() + for key, path := range map[string]string{ + "HOME": "home", "XDG_DATA_HOME": "data", "XDG_CONFIG_HOME": "config", "XDG_CACHE_HOME": "cache", "XDG_STATE_HOME": "state", + } { + t.Setenv(key, filepath.Join(root, path)) + } + configPath := filepath.Join(root, "daemon.yaml") + require.NoError(t, os.WriteFile(configPath, []byte("semantic:\n enabled: false\n"), 0o600)) + oldConfig, oldPath := cfgFile, daemonBackendPath + t.Cleanup(func() { cfgFile, daemonBackendPath = oldConfig, oldPath }) + cfgFile, daemonBackendPath = configPath, filepath.Join(root, "store.sqlite") +} + +func TestDaemonMigrationDefersCompactionForOneBoot(t *testing.T) { + for _, test := range []struct { + name string + version int + }{ + {name: "in-place migration", version: 39}, + {name: "supported destructive rebuild", version: 2}, + } { + t.Run(test.name, func(t *testing.T) { + compactionBootEnv(t) + initial, err := store_sqlite.Open(daemonBackendPath) + require.NoError(t, err) + require.NoError(t, initial.Close()) + db, err := sql.Open("sqlite", daemonBackendPath) + require.NoError(t, err) + // v39 exercises a real in-place step; v2 requires the supported + // destructive topology rebuild, which has no MigrationStarted event. + _, err = db.Exec(fmt.Sprintf(`PRAGMA user_version=%d`, test.version)) + require.NoError(t, err) + require.NoError(t, db.Close()) + core, logs := observer.New(zap.DebugLevel) + logger := zap.New(core) + observed := 0 + first, err := buildDaemonState(logger, func(p store_sqlite.MigrationProgress) { + if p.Phase == store_sqlite.MigrationStarted { + observed++ + } + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.shared.Close()) }) + if test.version == 39 { + require.Positive(t, observed, "the existing observer must still receive in-place migration boundaries") + } else { + require.Zero(t, observed, "rebuild is proven by the existing store ownership seam, not an invented observer event") + require.True(t, storeNeedsRebuild(first.graph)) + } + require.True(t, first.migratedStore) + file, ok := first.graph.(*store_sqlite.Store) + require.True(t, ok) + stub := &bootCompactStore{Store: first.graph, file: file} + maybeCompactStoreContext(t.Context(), stub, first.migratedStore, logger) + require.Zero(t, stub.calls) + entry := logs.FilterMessage("daemon: store compaction deferred to next boot").All() + require.Len(t, entry, 1) + require.Equal(t, "after_migration", entry[0].ContextMap()["reason"]) + require.NoError(t, first.shared.Close()) + second, err := buildDaemonState(logger) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, second.shared.Close()) }) + require.False(t, second.migratedStore, "no durable skip flag may survive to the next boot") + require.False(t, storeNeedsRebuild(second.graph)) + file, ok = second.graph.(*store_sqlite.Store) + require.True(t, ok) + stub = &bootCompactStore{Store: second.graph, file: file} + maybeCompactStoreContext(t.Context(), stub, second.migratedStore, logger) + require.Equal(t, 1, stub.calls) + require.Len(t, logs.FilterMessage("daemon: store compaction finished").All(), 1) + }) + } +} + +func TestWarmupDaemonStateDoesNotRunVacuum(t *testing.T) { + compactionBootEnv(t) + state, err := buildDaemonState(zap.NewNop()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, state.shared.Close()) }) + stub := &compactStub{Store: state.graph, path: state.backendPath, free: 1<<30 + 1, total: 2 << 30} + state.graph = stub + ready := false + watcher, _ := warmupDaemonStateContext(t.Context(), state, zap.NewNop(), func() { ready = true }, nil) + if watcher != nil { + t.Cleanup(func() { require.NoError(t, watcher.Stop()) }) + } + require.True(t, ready, "real warmup must reach queryability") + require.Zero(t, stub.compacts, "even eligible compaction must stay outside warmup") +} diff --git a/cmd/gortex/daemon_compact_test.go b/cmd/gortex/daemon_compact_test.go index 0dda95c68..ceaa7b152 100644 --- a/cmd/gortex/daemon_compact_test.go +++ b/cmd/gortex/daemon_compact_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "path/filepath" @@ -68,8 +69,8 @@ func TestShouldCompactStore(t *testing.T) { } // The store the daemon actually runs against must be the thing the boot -// compaction probes for. This is the wiring link between warmupDaemonState's -// maybeCompactStore(state.graph, …) call and the maintenance lane: a *Store +// compaction probes for. This is the wiring link between the post-warmup +// task and the maintenance lane: a *Store // that stopped satisfying storeCompactor would silently make boot compaction a // no-op instead of failing to build. var _ storeCompactor = (*store_sqlite.Store)(nil) @@ -86,9 +87,9 @@ type compactStub struct { compacts int } -func (c *compactStub) Path() string { return c.path } -func (c *compactStub) CompactStats() (free, total int64) { return c.free, c.total } -func (c *compactStub) Compact() error { c.compacts++; return c.compactErr } +func (c *compactStub) Path() string { return c.path } +func (c *compactStub) CompactStats() (free, total int64) { return c.free, c.total } +func (c *compactStub) CompactContext(context.Context) error { c.compacts++; return c.compactErr } // A deferral is not a failure. The maintenance lane refuses to rewrite the file // underneath a publish or a build, and boot must report that as the routine diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index 2b030b5d4..44fbe2671 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -1805,6 +1805,11 @@ func checkRowCounters(ctx context.Context, g graph.Store) *daemon.RowCounterStat func walReclaimStatus(st store_sqlite.WALReclaimStats) *daemon.WALReclaimStatus { out := &daemon.WALReclaimStatus{ + CancelledWriterParked: st.CancelledWriterParked, + CancelledWriteIntent: st.CancelledWriteIntent, + CancelledOther: st.CancelledOther, + WALPeakObserved: st.WALPeakObserved, + ThresholdBytes: st.ThresholdBytes, Attempts: st.Attempts, Resets: st.Resets, @@ -1841,6 +1846,10 @@ func walReclaimStatus(st store_sqlite.WALReclaimStats) *daemon.WALReclaimStatus ShrinkBytes: st.ShrinkBytes, ShrinkSliceHoldMaxMS: durationMS(st.ShrinkSliceHoldMax), } + if st.WALPeakObserved { + peak := st.ObservedPeakWALBytes + out.ObservedPeakWALBytes = &peak + } if st.PauseCount > 0 { out.PauseAvgMS = durationMS(st.PauseTotal / time.Duration(st.PauseCount)) } diff --git a/cmd/gortex/daemon_controller_view.go b/cmd/gortex/daemon_controller_view.go index 5d38ba073..165835c75 100644 --- a/cmd/gortex/daemon_controller_view.go +++ b/cmd/gortex/daemon_controller_view.go @@ -310,6 +310,10 @@ func fallbackProbeView(kind, checkoutID, repoPrefix, reason string) *daemon.Prob // and one whose view is not built yet is answered as uncovered so the hook // allows the native tool through. func (c *realController) FileCoverage(ctx context.Context, p daemon.FileCoverageParams) (daemon.FileCoverageResult, error) { + return c.fileCoverageWithProbe(ctx, p, c.resolveProbeView) +} + +func (c *realController) fileCoverageWithProbe(ctx context.Context, p daemon.FileCoverageParams, resolve func(context.Context, string) probeView) (daemon.FileCoverageResult, error) { if c == nil || p.Path == "" { return daemon.FileCoverageResult{}, nil } @@ -319,7 +323,7 @@ func (c *realController) FileCoverage(ctx context.Context, p daemon.FileCoverage } mutationProbe := c.lifecycle.CaptureCheckoutMutationProbe() - view := c.resolveProbeView(ctx, abs) + view := resolve(ctx, abs) defer view.release() out := daemon.FileCoverageResult{View: view.answer, Tracked: c.pathTracked(abs, view)} @@ -329,8 +333,13 @@ func (c *realController) FileCoverage(ctx context.Context, p daemon.FileCoverage // rows and reports the file covered while every graph call refuses with // repo_loading. The state rides on the reply so the hook can let the // native tool through instead of denying the only read path left. - if repo, loading := c.loadingRepoForPath(abs); loading { - out.RepoState, out.Repo = daemon.RepoStateLoading, repo + if rec, classified := c.pendingRepoLoadForPath(abs); classified { + if rec.State.Loading() && rec.State != indexer.RepoLoadIdentityPending { + out.RepoState, out.Repo = daemon.RepoStateLoading, rec.Label() + } else if !out.Tracked && (rec.State == indexer.RepoLoadIdentityPending || rec.State.Ready()) { + out.RepoState = daemon.RepoStateIdentityPending + return out, nil // unknown or newly ready needs a fresh authoritative probe + } } if !view.servable { return out, nil @@ -414,6 +423,10 @@ const dirCoverageWalkBudget = 500 * time.Millisecond // there. It is the scope verdict a PreToolUse hook turns into a Grep/Glob // deny; both halves are exact rather than sampled. func (c *realController) DirCoverage(ctx context.Context, p daemon.DirCoverageParams) (daemon.DirCoverageResult, error) { + return c.dirCoverageWithProbe(ctx, p, c.resolveProbeView) +} + +func (c *realController) dirCoverageWithProbe(ctx context.Context, p daemon.DirCoverageParams, resolve func(context.Context, string) probeView) (daemon.DirCoverageResult, error) { if c == nil || p.Path == "" { return daemon.DirCoverageResult{}, nil } @@ -422,13 +435,18 @@ func (c *realController) DirCoverage(ctx context.Context, p daemon.DirCoveragePa abs = resolved } - view := c.resolveProbeView(ctx, abs) + view := resolve(ctx, abs) defer view.release() out := daemon.DirCoverageResult{View: view.answer, Tracked: c.pathTracked(abs, view)} // See FileCoverage: a loading repository's scope must not deny Grep. - if repo, loading := c.loadingRepoForPath(abs); loading { - out.RepoState, out.Repo = daemon.RepoStateLoading, repo + if rec, classified := c.pendingRepoLoadForPath(abs); classified { + if rec.State.Loading() && rec.State != indexer.RepoLoadIdentityPending { + out.RepoState, out.Repo = daemon.RepoStateLoading, rec.Label() + } else if !out.Tracked && (rec.State == indexer.RepoLoadIdentityPending || rec.State.Ready()) { + out.RepoState = daemon.RepoStateIdentityPending + return out, nil // unknown or newly ready needs a fresh authoritative probe + } } if !view.servable { return out, nil diff --git a/cmd/gortex/daemon_controller_worktree_test.go b/cmd/gortex/daemon_controller_worktree_test.go index dfeaeb105..a39318f61 100644 --- a/cmd/gortex/daemon_controller_worktree_test.go +++ b/cmd/gortex/daemon_controller_worktree_test.go @@ -75,8 +75,12 @@ func buildWorktreeController(t *testing.T) (*realController, *indexer.MultiIndex reg := parser.NewRegistry() languages.RegisterAll(reg) mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // Exercise the same registration path as a cold production startup. This + // fixture asserts worktree identity/reload effects, not batch resolution. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } // An in-memory graph has no checkout catalog, so the lifecycle runs in // its degraded shape here: no identities, and the same index / watcher / diff --git a/cmd/gortex/daemon_detach_env.go b/cmd/gortex/daemon_detach_env.go new file mode 100644 index 000000000..1c773ddea --- /dev/null +++ b/cmd/gortex/daemon_detach_env.go @@ -0,0 +1,31 @@ +package main + +import ( + "fmt" + "path/filepath" + "runtime" + "strings" + + "github.com/zzet/gortex/internal/serverstack" +) + +func detachedDaemonEnv(parent []string, backendPath string) ([]string, error) { + path, err := serverstack.ResolveStorePath(backendPath) + if err != nil { + return nil, fmt.Errorf("resolve detached SQLite temp directory: %w", err) + } + child := make([]string, 0, len(parent)+2) + for _, entry := range parent { + key, _, _ := strings.Cut(entry, "=") + if runtime.GOOS == "windows" { + key = strings.ToUpper(key) + } + if key != "SQLITE_TMPDIR" && key != "GORTEX_DAEMON_CHILD" { + child = append(child, entry) + } + } + // SQLite/libc may cache the environment at first use. Set this before + // exec, replacing inherited duplicates. General TMPDIR is deliberately + // preserved: only SQLite's scratch placement changes. + return append(child, "GORTEX_DAEMON_CHILD=1", "SQLITE_TMPDIR="+filepath.Dir(path)), nil +} diff --git a/cmd/gortex/daemon_detach_env_test.go b/cmd/gortex/daemon_detach_env_test.go new file mode 100644 index 000000000..199b6d4d5 --- /dev/null +++ b/cmd/gortex/daemon_detach_env_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/platform" +) + +func TestDetachedDaemonEnvCarriesSQLiteTmpdir(t *testing.T) { + dir := t.TempDir() + parent := []string{"SQLITE_TMPDIR=/wrong", "KEEP=unchanged", "TMPDIR=/general-temp", "GORTEX_DAEMON_CHILD=0", "SQLITE_TMPDIR=/also-wrong"} + before := append([]string(nil), parent...) + for _, test := range []struct { + name, path, want string + }{ + {name: "explicit", path: filepath.Join(dir, "store volume", "custom.sqlite"), want: filepath.Join(dir, "store volume")}, + {name: "default", path: "", want: platform.StoreDir()}, + } { + t.Run(test.name, func(t *testing.T) { + env, err := detachedDaemonEnv(parent, test.path) + require.NoError(t, err) + var sqlite []string + for _, entry := range env { + if strings.HasPrefix(entry, "SQLITE_TMPDIR=") { + sqlite = append(sqlite, entry) + } + } + require.Equal(t, []string{"SQLITE_TMPDIR=" + test.want}, sqlite) + require.Contains(t, env, "GORTEX_DAEMON_CHILD=1") + require.NotContains(t, env, "GORTEX_DAEMON_CHILD=0") + require.Contains(t, env, "TMPDIR=/general-temp") + require.Contains(t, env, "KEEP=unchanged") + require.Equal(t, before, parent) + }) + } + // The environment must fail setup before exec if the intended parent is + // unusable. It must not silently select the system temp directory. + file := filepath.Join(dir, "not-a-directory") + require.NoError(t, os.WriteFile(file, []byte("owned fixture"), 0o600)) + _, err := detachedDaemonEnv(parent, filepath.Join(file, "store.sqlite")) + require.ErrorContains(t, err, "resolve detached SQLite temp directory") +} diff --git a/cmd/gortex/daemon_dispatch_lanes_test.go b/cmd/gortex/daemon_dispatch_lanes_test.go new file mode 100644 index 000000000..d794b3009 --- /dev/null +++ b/cmd/gortex/daemon_dispatch_lanes_test.go @@ -0,0 +1,507 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/graph" + gortexmcp "github.com/zzet/gortex/internal/mcp" + "github.com/zzet/gortex/internal/mcp/streamable" + "github.com/zzet/gortex/internal/query" + "github.com/zzet/gortex/internal/testenv" +) + +// Only the controlled holders are synthetic; all other calls use the real +// dispatcher, facade classifier, HTTP mount, Unix server and remote router. +// No source file is modified by the simulated mutation holders. +type transportLaneDispatcher struct { + *mcpDispatcher + started chan struct{} + release <-chan struct{} +} + +func (d *transportLaneDispatcher) Dispatch(ctx context.Context, sess *daemon.Session, frame []byte) ([]byte, error) { + var envelope struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments struct { + Hold bool `json:"lane_test_hold"` + } `json:"arguments"` + } `json:"params"` + } + if err := json.Unmarshal(frame, &envelope); err != nil { + return nil, err + } + if envelope.Method == "tools/call" && envelope.Params.Name == "edit_file" && envelope.Params.Arguments.Hold { + d.started <- struct{}{} + select { + case <-d.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + return []byte(`{"jsonrpc":"2.0","id":` + string(envelope.ID) + `,"result":{"held":true}}`), nil + } + return d.mcpDispatcher.Dispatch(ctx, sess, frame) +} + +func TestDaemonHTTPAndUnixShareLanesBeforeRemoteRouting(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(home, "data")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, "cache")) + t.Setenv("GORTEX_QUERY_LOG_DISABLE", "1") + t.Setenv("GORTEX_TOOLS", "full") + t.Setenv("GORTEX_TOOLS_MODE", "hide") + dir := testenv.ShortTempDir(t) + socket := filepath.Join(dir, "s") + t.Setenv("GORTEX_DAEMON_SOCKET", socket) + t.Setenv("GORTEX_DAEMON_PIDFILE", filepath.Join(dir, "p")) + t.Setenv("GORTEX_DAEMON_STATEFILE", filepath.Join(dir, "state.json")) + + var remoteCalls atomic.Int32 + remoteIdentity := make(chan string, 2) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteCalls.Add(1) + select { + case remoteIdentity <- r.Header.Get(streamable.HeaderSessionID): + default: + } + w.Header().Set("Content-Type", "application/json") + if r.Method != http.MethodPost || r.URL.Path != "/v1/tools/explore" { + http.Error(w, "unexpected remote endpoint", http.StatusBadRequest) + return + } + _, _ = io.WriteString(w, `{"remote":true}`) + })) + t.Cleanup(upstream.Close) + router := daemon.NewRouter(daemon.RouterConfig{Servers: &daemon.ServersConfig{Server: []daemon.ServerEntry{{Slug: "remote", URL: upstream.URL, Workspaces: []string{"lane-remote"}}}}, LocalSlug: daemon.LocalServerSentinel, Logger: zap.NewNop()}) + g := graph.New() + policy := gortexmcp.ToolPolicyConfig{Preset: "full"} + srv := gortexmcp.NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{ToolPolicy: &policy}) + inner := newMCPDispatcher(srv, nil, zap.NewNop()) + inner.SetRouter(router) + limit := daemon.MCPDispatchSnapshot().Limit + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + dispatcher := &transportLaneDispatcher{mcpDispatcher: inner, started: make(chan struct{}, limit), release: release} + unix, unixDone := startProxyRestartDaemon(t, socket, dispatcher) + t.Cleanup(func() { unblock(); stopProxyRestartDaemon(t, unix, unixDone) }) + mount := buildDaemonStreamableHandler(dispatcher, daemon.NewSessionRegistry(), router, zap.NewNop(), nil) + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // A server-side request deadline (not the HTTP client's timeout) exercises + // the required deadline-minus100ms queue budget through the real mount. + ctx, cancel := context.WithTimeout(r.Context(), 500*time.Millisecond) + defer cancel() + mount.ServeHTTP(w, r.WithContext(ctx)) + })) + t.Cleanup(httpServer.Close) + httpClient := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{Proxy: nil}} + t.Cleanup(httpClient.CloseIdleConnections) + post := func(session string, frame string) ([]byte, string) { + t.Helper() + req, err := http.NewRequest(http.MethodPost, httpServer.URL+"/mcp", bytes.NewBufferString(frame)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + if session != "" { + req.Header.Set(streamable.HeaderSessionID, session) + } + response, err := httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + payload, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("HTTP status%d: %s", response.StatusCode, payload) + } + return payload, response.Header.Get(streamable.HeaderSessionID) + } + _, sid := post("", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"lane-test","version":"1"}}}`) + if sid == "" { + t.Fatal("HTTP initialization minted no session") + } + client, err := daemon.DialTo(socket, daemon.Handshake{Mode: daemon.ModeMCP, PID: os.Getpid(), Tools: "full", ToolsMode: "hide"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = client.Close() }) + if err := client.Conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + if err := client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"lane-test","version":"1"}}}`)); err != nil { + t.Fatal(err) + } + if reply, err := client.ReadMCPFrame(); err != nil || !bytes.Contains(reply, []byte(`"result"`)) { + t.Fatalf("Unix initialize reply=%s err=%v", reply, err) + } + if err := client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","method":"notifications/initialized"}`)); err != nil { + t.Fatal(err) + } + for i := range limit { + frame := []byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":"edit_file","arguments":{"lane_test_hold":true}}}`, i+10)) + if err := client.WriteMCPFrame(frame); err != nil { + t.Fatal(err) + } + select { + case <-dispatcher.started: + case <-time.After(time.Second): + t.Fatal("Unix mutation did not occupy shared dispatch capacity") + } + } + before := daemon.MCPDispatchSnapshot() + if before.Lanes["mutating"].InFlight != limit { + t.Fatalf("Unix did not occupy process cap: %+v", before) + } + frame := `{"jsonrpc":"2.0","id":88,"method":"tools/call","params":{"name":"explore","arguments":{"operation":"outline","workspace":"lane-remote"}}}` + busy, _ := post(sid, frame) + var refusal struct { + Error struct { + Code int `json:"code"` + Data map[string]any `json:"data"` + } `json:"error"` + } + if err := json.Unmarshal(busy, &refusal); err != nil { + t.Fatal(err) + } + retry, _ := refusal.Error.Data["retry_after_ms"].(float64) + if refusal.Error.Code != -32002 || refusal.Error.Data["delivery_unknown"] != false || refusal.Error.Data["retryable"] != true || retry < 100 || retry > 5000 { + t.Fatalf("HTTP lane refusal=%s", busy) + } + if remoteCalls.Load() != 0 { + t.Fatal("HTTP remote call bypassed occupied Unix admission cap") + } + after := daemon.MCPDispatchSnapshot() + if after.Lanes["slow"].Refusals != before.Lanes["slow"].Refusals+1 { + t.Fatalf("effective explore.outline did not use slow admission: before=%+v after=%+v", before, after) + } + unblock() + for received := 0; received < limit; { + reply, err := client.ReadMCPFrame() + if err != nil { + t.Fatal(err) + } + var message struct { + ID json.RawMessage `json:"id"` + Error json.RawMessage `json:"error"` + } + if err := json.Unmarshal(reply, &message); err != nil { + t.Fatal(err) + } + if len(message.ID) == 0 { + continue + } + if len(message.Error) > 0 { + t.Fatalf("holder failed: %s", reply) + } + received++ + } + success, _ := post(sid, frame) // Busy was not a committed/replayed tool result. + if remoteCalls.Load() != 1 || !bytes.Contains(success, []byte(`"remote":true`)) { + t.Fatalf("admitted remote retry count=%d reply=%s", remoteCalls.Load(), success) + } + select { + case identity := <-remoteIdentity: + if identity != sid { + t.Fatalf("remote identity=%q want=%q", identity, sid) + } + default: + t.Fatal("remote received no session identity") + } +} + +// heldToolDispatcher runs frames through a plain mcp-go server, so an HTTP +// notifications/cancelled reaches the same in-flight cancellation a daemon +// session relies on. Its tools only wait; the classifier lets them fill the +// shared and the reserved mutating slots alike. +type heldToolDispatcher struct { + server *mcpserver.MCPServer +} + +func (d heldToolDispatcher) Dispatch(ctx context.Context, _ *daemon.Session, frame []byte) ([]byte, error) { + reply := d.server.HandleMessage(ctx, json.RawMessage(frame)) + if reply == nil { + return nil, nil + } + return json.Marshal(reply) +} + +func (heldToolDispatcher) DispatchClass(name string, _ map[string]any) daemon.MCPDispatchClass { + if name == "hold_edit" { + return daemon.MCPDispatchMutating + } + return daemon.MCPDispatchFast +} + +func TestDaemonHTTPCancellationReachesAFullDispatchPool(t *testing.T) { + status := daemon.MCPDispatchSnapshot() + limit, reserve := status.Limit, status.MutationReserve + started := make(chan struct{}, limit) + release := make(chan struct{}) + hold := func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + started <- struct{}{} + select { + case <-release: + return mcp.NewToolResultText("released"), nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + tools := mcpserver.NewMCPServer("cancel-test", "1") + tools.AddTool(mcp.NewTool("hold_read"), hold) + tools.AddTool(mcp.NewTool("hold_edit"), hold) + mount := buildDaemonStreamableHandler(heldToolDispatcher{server: tools}, daemon.NewSessionRegistry(), nil, zap.NewNop(), nil) + httpServer := httptest.NewServer(mount) + unblock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(func() { unblock(); httpServer.Close() }) + transport := &http.Transport{Proxy: nil} + t.Cleanup(transport.CloseIdleConnections) + holderClient := &http.Client{Timeout: 30 * time.Second, Transport: transport} + // The bound only has to tell "delivered now" from "delivered once a holder + // finishes": these holders never finish on their own. + cancelClient := &http.Client{Timeout: 2 * time.Second, Transport: transport} + type posted struct { + id int + status int + session string + body []byte + err error + } + post := func(client *http.Client, id int, session, frame string) posted { + out := posted{id: id} + req, err := http.NewRequest(http.MethodPost, httpServer.URL+"/mcp", bytes.NewBufferString(frame)) + if err != nil { + out.err = err + return out + } + req.Header.Set("Content-Type", "application/json") + if session != "" { + req.Header.Set(streamable.HeaderSessionID, session) + } + response, err := client.Do(req) + if err != nil { + out.err = err + return out + } + defer response.Body.Close() + out.status, out.session = response.StatusCode, response.Header.Get(streamable.HeaderSessionID) + out.body, out.err = io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return out + } + initialized := post(holderClient, 1, "", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"cancel-test","version":"1"}}}`) + if initialized.err != nil || initialized.session == "" { + t.Fatalf("HTTP initialization: session=%q err=%v body=%s", initialized.session, initialized.err, initialized.body) + } + sid := initialized.session + + // Every slot is held: the shared ones by reads, the reserved ones by edits. + results := make(chan posted, limit) + for i := range limit { + name := "hold_read" + if i >= limit-reserve { + name = "hold_edit" + } + id := 100 + i + go func() { + results <- post(holderClient, id, sid, fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"tools/call","params":{"name":%q,"arguments":{}}}`, id, name)) + }() + select { + case <-started: + case early := <-results: + t.Fatalf("holder %d returned before holding a slot: status=%d err=%v body=%s", early.id, early.status, early.err, early.body) + case <-time.After(5 * time.Second): + t.Fatalf("holder %d did not take a dispatch slot", id) + } + } + full := daemon.MCPDispatchSnapshot() + if full.Lanes["fast"].InFlight != limit-reserve || full.Lanes["mutating"].InFlight != reserve { + t.Fatalf("holders do not fill the pool: %+v", full.Lanes) + } + + cancelled := post(cancelClient, 0, sid, `{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":100}}`) + if cancelled.err != nil || cancelled.status != http.StatusAccepted { + t.Fatalf("cancellation waited for the capacity it frees: status=%d err=%v body=%s", cancelled.status, cancelled.err, cancelled.body) + } + select { + case got := <-results: + if got.id != 100 || got.err != nil || bytes.Contains(got.body, []byte("released")) { + t.Fatalf("cancelled holder: id=%d status=%d err=%v body=%s", got.id, got.status, got.err, got.body) + } + case <-time.After(time.Second): + t.Fatal("the cancelled call kept its slot") + } + // The reply follows the slot's release, so the lane already shows it. + if after := daemon.MCPDispatchSnapshot(); after.Lanes["fast"].InFlight != limit-reserve-1 || after.Lanes["mutating"].InFlight != reserve { + t.Fatalf("cancelled call did not return its slot: %+v", after.Lanes) + } + unblock() + for range limit - 1 { + select { + case got := <-results: + if got.err != nil || !bytes.Contains(got.body, []byte("released")) { + t.Fatalf("holder %d: status=%d err=%v body=%s", got.id, got.status, got.err, got.body) + } + case <-time.After(5 * time.Second): + t.Fatal("released holders did not finish") + } + } +} + +// TestDaemonHTTPRoutesWithTheTransportsInputsInsideAdmission keeps the /mcp +// mount routing on everything the transport knows about the caller: a session +// pinned to a workspace at initialize, and a call naming its own cwd. Moving +// the hop inside shared admission must not narrow those inputs to the +// dispatcher's per-call workspace and session cwd, nor change how an upstream +// failure is framed. +func TestDaemonHTTPRoutesWithTheTransportsInputsInsideAdmission(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(home, "data")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, "cache")) + t.Setenv("GORTEX_QUERY_LOG_DISABLE", "1") + t.Setenv("GORTEX_TOOLS", "full") + t.Setenv("GORTEX_TOOLS_MODE", "hide") + checkout := t.TempDir() + if err := os.WriteFile(filepath.Join(checkout, ".gortex.yaml"), []byte("workspace: cwd-remote\n"), 0o600); err != nil { + t.Fatal(err) + } + + type hop struct { + server, path, cwd string + fastInFlight int + } + hops := make(chan hop, 4) + upstream := func(name string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hops <- hop{server: name, path: r.URL.Path, cwd: r.Header.Get("X-Gortex-Cwd"), fastInFlight: daemon.MCPDispatchSnapshot().Lanes["fast"].InFlight} + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/v1/tools/find_usages" { + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = io.WriteString(w, `{"error":"unknown symbol"}`) + return + } + _, _ = io.WriteString(w, `{"content":[{"type":"text","text":"answer from `+name+`"}]}`) + })) + t.Cleanup(server.Close) + return server + } + // The first roster entry is the default for a call nothing else routes, + // so only the caller's own inputs can pick the second one. + elsewhere, remote := upstream("elsewhere"), upstream("remote") + router := daemon.NewRouter(daemon.RouterConfig{Servers: &daemon.ServersConfig{Server: []daemon.ServerEntry{ + {Slug: "elsewhere", URL: elsewhere.URL}, + {Slug: "remote", URL: remote.URL, Workspaces: []string{"pinned-remote", "cwd-remote"}}, + }}, LocalSlug: daemon.LocalServerSentinel, Logger: zap.NewNop()}) + g := graph.New() + policy := gortexmcp.ToolPolicyConfig{Preset: "full"} + srv := gortexmcp.NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{ToolPolicy: &policy}) + dispatcher := newMCPDispatcher(srv, nil, zap.NewNop()) + dispatcher.SetRouter(router) + httpServer := httptest.NewServer(buildDaemonStreamableHandler(dispatcher, daemon.NewSessionRegistry(), router, zap.NewNop(), nil)) + t.Cleanup(httpServer.Close) + httpClient := &http.Client{Timeout: 10 * time.Second, Transport: &http.Transport{Proxy: nil}} + t.Cleanup(httpClient.CloseIdleConnections) + post := func(session, frame string) ([]byte, string) { + t.Helper() + req, err := http.NewRequest(http.MethodPost, httpServer.URL+"/mcp", bytes.NewBufferString(frame)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + if session != "" { + req.Header.Set(streamable.HeaderSessionID, session) + } + response, err := httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + payload, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatalf("HTTP status %d: %s", response.StatusCode, payload) + } + return payload, response.Header.Get(streamable.HeaderSessionID) + } + initialize := func(meta string) string { + t.Helper() + _, sid := post("", `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"route-test","version":"1"}`+meta+`}}`) + if sid == "" { + t.Fatal("HTTP initialization minted no session") + } + return sid + } + routed := func(name string, reply []byte) hop { + t.Helper() + select { + case got := <-hops: + if got.server != "remote" || got.path != "/v1/tools/"+name { + t.Fatalf("hop to %s%s, want remote/v1/tools/%s; reply=%s", got.server, got.path, name, reply) + } + if got.fastInFlight < 1 { + t.Fatalf("remote hop for %s ran outside dispatch admission", name) + } + return got + default: + t.Fatalf("%s was not routed: %s", name, reply) + return hop{} + } + } + + pinned := initialize(`,"_meta":{"workspace":"pinned-remote"}`) + reply, _ := post(pinned, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Needle"}}}`) + routed("search_symbols", reply) + if !bytes.Contains(reply, []byte("answer from remote")) { + t.Fatalf("pinned session reply=%s", reply) + } + + unpinned := initialize("") + reply, _ = post(unpinned, fmt.Sprintf(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Needle","cwd":%q}}}`, checkout)) + if got := routed("search_symbols", reply); got.cwd != checkout { + t.Fatalf("call routed by cwd %q told the remote cwd %q", checkout, got.cwd) + } + if !bytes.Contains(reply, []byte("answer from remote")) { + t.Fatalf("cwd-routed reply=%s", reply) + } + + reply, _ = post(pinned, `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"find_usages","arguments":{"id":"missing"}}}`) + routed("find_usages", reply) + var failed struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(reply, &failed); err != nil { + t.Fatal(err) + } + if failed.Error.Code != -32603 || failed.Error.Message != "remote tool call failed (HTTP 422)" { + t.Fatalf("upstream 4xx reply=%s", reply) + } +} diff --git a/cmd/gortex/daemon_dispatch_search_cancel_test.go b/cmd/gortex/daemon_dispatch_search_cancel_test.go new file mode 100644 index 000000000..96c602561 --- /dev/null +++ b/cmd/gortex/daemon_dispatch_search_cancel_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + gortexmcp "github.com/zzet/gortex/internal/mcp" + "github.com/zzet/gortex/internal/query" + "github.com/zzet/gortex/internal/search" + "github.com/zzet/gortex/internal/testenv" +) + +// The real SQLite store and native FTS adapter remain in use. Only entry into +// the FTS query is delayed, deliberately ignoring cancellation until released, +// so the test proves admission recovery independently of SQLite interruption. +// This is an owned six-prefix fixture, not reference-corpus load evidence. +type heldDispatchFTSStore struct { + *store_sqlite.Store + once sync.Once + started chan context.Context + release <-chan struct{} + finished chan struct{} +} + +func (s *heldDispatchFTSStore) SearchSymbolBundlesContext(ctx context.Context, text string, limit int) ([]graph.SymbolBundle, error) { + first := false + s.once.Do(func() { first = true; s.started <- ctx }) + if first { + defer close(s.finished) + } + <-s.release + return s.Store.SearchSymbolBundlesContext(ctx, text, limit) +} + +func TestDaemonCancelledSearchOnSixRepoFTSReturnsCapacityWithin100ms(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(home, "data")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, "cache")) + t.Setenv("GORTEX_QUERY_LOG_DISABLE", "1") + t.Setenv("GORTEX_TOOLS", "full") + t.Setenv("GORTEX_TOOLS_MODE", "hide") + store, err := store_sqlite.Open(filepath.Join(home, "six.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + var nodes []*graph.Node + var items []graph.SymbolFTSItem + for i := range 6 { + repo := fmt.Sprintf("fixture-%d", i) + root := filepath.Join(home, repo) + require.NoError(t, os.MkdirAll(root, 0700)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source.go"), []byte("package fixture\nfunc NeedleSentinel(){}\n"), 0600)) + id := repo + "/source.go::NeedleSentinel" + nodes = append(nodes, &graph.Node{ID: id, Name: "NeedleSentinel", Kind: graph.KindFunction, Language: "go", RepoPrefix: repo, FilePath: "source.go", StartLine: 2, EndLine: 2}) + items = append(items, graph.SymbolFTSItem{NodeID: id, Tokens: strings.Join(search.Tokenize("NeedleSentinel"), " ")}) + } + store.AddBatch(nodes, nil) + require.NoError(t, store.BulkUpsertSymbolFTS("", items)) + count, err := store.SymbolFTSCount() + require.NoError(t, err) + require.Equal(t, 6, count) + prefixes := map[string]bool{} + for _, node := range store.AllNodesLight() { + prefixes[node.RepoPrefix] = true + } + require.Len(t, prefixes, 6) + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + held := &heldDispatchFTSStore{Store: store, started: make(chan context.Context, 1), release: release, finished: make(chan struct{})} + t.Cleanup(unblock) + engine := query.NewEngine(held) + engine.SetSearch(search.NewSymbolSearcherBackend(held)) + policy := gortexmcp.ToolPolicyConfig{Preset: "full"} + srv := gortexmcp.NewServer(engine, held, nil, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{ToolPolicy: &policy}) + dispatcher := newMCPDispatcher(srv, nil, zap.NewNop()) + dir := testenv.ShortTempDir(t) + socket := filepath.Join(dir, "s") + t.Setenv("GORTEX_DAEMON_SOCKET", socket) + t.Setenv("GORTEX_DAEMON_PIDFILE", filepath.Join(dir, "p")) + t.Setenv("GORTEX_DAEMON_STATEFILE", filepath.Join(dir, "state.json")) + server, serverDone := startProxyRestartDaemon(t, socket, dispatcher) + t.Cleanup(func() { unblock(); stopProxyRestartDaemon(t, server, serverDone) }) + client, err := daemon.DialTo(socket, daemon.Handshake{Mode: daemon.ModeMCP, Tools: "full", ToolsMode: "hide"}) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.Conn.SetDeadline(time.Now().Add(10*time.Second))) + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","capabilities":{},"clientInfo":{"name":"cancel-test","version":"1"}}}`))) + initialized, err := client.ReadMCPFrame() + require.NoError(t, err) + require.Contains(t, string(initialized), `"result"`) + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","method":"notifications/initialized"}`))) + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"needle sentinel","assist":false}}}`))) + type frameResult struct { + frame []byte + err error + } + responseCh := make(chan frameResult, 1) + go func() { + frame, readErr := client.ReadMCPFrame() + responseCh <- frameResult{frame: frame, err: readErr} + }() + var queryContext context.Context + select { + case queryContext = <-held.started: + case early := <-responseCh: + t.Fatalf("search_symbols returned before entering injected FTS: frame=%s error=%v", early.frame, early.err) + case <-time.After(5 * time.Second): + t.Fatal("search_symbols did not enter the injected native FTS query") + } + // Setup is not part of the cancellation bound. Full-suite coverage/race + // scheduling may delay query entry; time only after that entry is witnessed. + require.NoError(t, client.Conn.SetReadDeadline(time.Now().Add(100*time.Millisecond))) + canceledAt := time.Now() + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":2}}`))) + result := <-responseCh + require.NoError(t, result.err) + elapsed := time.Since(canceledAt) + require.Less(t, elapsed, 100*time.Millisecond) + require.Contains(t, string(result.frame), `"code":-32800`) + require.Contains(t, string(result.frame), `"delivery_unknown":true`) + require.Contains(t, string(result.frame), `"retryable":false`) + require.ErrorIs(t, queryContext.Err(), context.Canceled, "request context must reach the native FTS interface") + select { + case <-held.finished: + t.Fatal("injected FTS query completed before release") + default: + } + status := daemon.MCPDispatchSnapshot() + require.Zero(t, status.Lanes["fast"].InFlight) + require.Equal(t, 1, status.Lanes["fast"].Draining) + // Another real fast request can run while the canceled SQL call still + // physically drains. It does not reuse or release that orphan's lease. + require.NoError(t, client.Conn.SetReadDeadline(time.Now().Add(time.Second))) + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":3,"method":"ping"}`))) + ping, err := client.ReadMCPFrame() + require.NoError(t, err) + require.True(t, bytes.Contains(ping, []byte(`"id":3`)) && bytes.Contains(ping, []byte(`"result"`)), "ping=%s", ping) + unblock() + select { + case <-held.finished: + case <-time.After(time.Second): + t.Fatal("released FTS query did not finish") + } + require.Eventually(t, func() bool { return daemon.MCPDispatchSnapshot().Lanes["fast"].Draining == 0 }, time.Second, time.Millisecond) +} diff --git a/cmd/gortex/daemon_janitor_ownership_test.go b/cmd/gortex/daemon_janitor_ownership_test.go index 271fd0ea7..033a4d1fd 100644 --- a/cmd/gortex/daemon_janitor_ownership_test.go +++ b/cmd/gortex/daemon_janitor_ownership_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "runtime" "strings" "sync" @@ -46,7 +47,7 @@ func TestJanitorCanceledCallerRetainsAdmittedWorkerUntilRealStackClose(t *testin state, err := buildDaemonState(zap.New(core)) // Actual shared cleanup registration. require.NoError(t, err) t.Cleanup(func() { _ = state.shared.Close() }) - _, err = state.multiIndexer.IndexAll() + _, err = state.multiIndexer.TrackRepoCtx(context.Background(), gc.Repos[0]) require.NoError(t, err) nodes := state.graph.GetFileNodes("tracked/main.go") require.NotEmpty(t, nodes) diff --git a/cmd/gortex/daemon_mcp.go b/cmd/gortex/daemon_mcp.go index 7e5c20e90..44912b21e 100644 --- a/cmd/gortex/daemon_mcp.go +++ b/cmd/gortex/daemon_mcp.go @@ -74,6 +74,16 @@ func (d *mcpDispatcher) SetRouter(r *daemon.Router) { d.router.Store(r) } // without rebuilding it from servers.toml a second time. func (d *mcpDispatcher) Router() *daemon.Router { return d.router.Load() } +// DispatchClass lowers facade calls through the same immutable registry used +// by execution, before either daemon transport acquires admission capacity. +func (d *mcpDispatcher) DispatchClass(name string, arguments map[string]any) daemon.MCPDispatchClass { + if d.srv == nil { + class, _ := daemon.ToolDispatchClass(name) + return class + } + return d.srv.DispatchClass(name, arguments) +} + // Dispatch implements daemon.MCPDispatcher. It hands the raw JSON-RPC // frame to MCPServer.HandleMessage and returns the response bytes. // Empty return value means the client sent a notification (no response). @@ -123,7 +133,7 @@ func (d *mcpDispatcher) Dispatch(ctx context.Context, sess *daemon.Session, fram admissionErr = nil } var terminalFamilyLoad []indexer.RepoLoad - if pending != nil && !pending.load.State.Loading() && agents.PendingAutomaticCheckout(sess.CWD, d.familyRoots()) { + if pending != nil && pending.load.State.Unavailable() && agents.PendingAutomaticCheckout(sess.CWD, d.familyRoots()) { terminalFamilyLoad = []indexer.RepoLoad{pending.load} pending = nil // still unadmitted; only the guidance/refusal classification changes } @@ -135,7 +145,7 @@ func (d *mcpDispatcher) Dispatch(ctx context.Context, sess *daemon.Session, fram return d.familyDiscoveryRefusal(sess.CWD, frame, "family_member"), nil } if pending != nil { - if refused := d.pendingRepoRefusal(sess, frame, pending.load); refused != nil { + if refused := d.pendingRepoRefusal(sess, frame, pending); refused != nil { return refused, nil } } @@ -235,6 +245,9 @@ func (d *mcpDispatcher) Dispatch(ctx context.Context, sess *daemon.Session, fram if admissionErr != nil { out = rewriteCheckoutAdmissionResponse(peekFrameMethod(frame), out, sess.CWD, admissionErr) } + if pending != nil && pending.familyInstructions != "" { + out = rewriteMCPAdmissionInstructions(peekFrameMethod(frame), out, pending.familyInstructions) + } if pending != nil { d.notePendingRepoSession(sess, peekFrameMethod(frame), pending.load) } @@ -427,6 +440,10 @@ func checkoutAdmissionError(cwd string, inbound []byte, err error) []byte { } func rewriteCheckoutAdmissionResponse(method string, out []byte, cwd string, err error) []byte { + return rewriteMCPAdmissionInstructions(method, out, gortexmcp.CheckoutAdmissionInstructions(cwd, err)) +} + +func rewriteMCPAdmissionInstructions(method string, out []byte, instructions string) []byte { if method != "initialize" && method != "server/discover" { return out } @@ -438,7 +455,7 @@ func rewriteCheckoutAdmissionResponse(method string, out []byte, cwd string, err if !ok { return out } - result["instructions"] = gortexmcp.CheckoutAdmissionInstructions(cwd, err) + result["instructions"] = instructions updated, marshalErr := json.Marshal(reply) if marshalErr != nil { return out @@ -450,7 +467,7 @@ func (d *mcpDispatcher) cwdReachableChecked(ctx context.Context, cwd string) (bo if cwd == "" { return true, nil } - if d.isCWDTracked(cwd) { + if d.registeredCWDReachable(cwd) { return true, nil } // Workspace-root fallback: a cwd that CONTAINS tracked repos — an @@ -468,12 +485,6 @@ func (d *mcpDispatcher) cwdReachableChecked(ctx context.Context, cwd string) (bo // left the session with no working call and a remedy (`gortex track // `) that would have indexed every child a second time. if d.multiIndexer != nil { - if _, _, _, ok := d.multiIndexer.ScopeForCWD(cwd); ok { - return true, nil - } - if _, _, ok := d.multiIndexer.ContainedReposScope(cwd); ok { - return true, nil - } // Configured but not registered yet: warmup has not reached the // repository, or its load ended without registering it. Decided // before the checkout and router arms, so a worktree of a loading @@ -499,18 +510,61 @@ func (d *mcpDispatcher) cwdReachableChecked(ctx context.Context, cwd string) (bo return err == nil && reachable, err } } - rtr := d.router.Load() - if rtr == nil { - return false, nil + if rtr := d.router.Load(); rtr != nil { + lookup := rtr.LookupForCwd(cwd, "") + switch lookup.Source { + case "scope-override", "config-yaml", "roster": + return true, nil + } } - lookup := rtr.LookupForCwd(cwd, "") - switch lookup.Source { - case "scope-override", "config-yaml", "roster": - return true, nil + if err := ctx.Err(); err != nil { + return false, err + } + if d.multiIndexer != nil { + if rec, classified := d.multiIndexer.RepoLoadAdmissionForPath(cwd); classified { + if rec.State.Ready() { + // Registration can publish during the probes above. The load table is + // identity evidence, not graph authority: recheck ordinary scope once. + if d.registeredCWDReachable(cwd) { + return true, nil + } + if probe != nil { + reachable, err := probe(ctx, cwd) + if err != nil || reachable { + return err == nil && reachable, err + } + } + if instructions, known := gortexmcp.PendingAutomaticCheckoutInstructions(cwd, d.familyRoots()...); known { + return false, &repoPendingAdmission{load: rec, familyInstructions: instructions} + } + rec = indexer.RepoLoad{State: indexer.RepoLoadIdentityPending, Reason: "scope_binding_pending"} + } + if err := ctx.Err(); err != nil { + return false, err + } + return false, &repoPendingAdmission{load: rec} + } } return false, nil } +// registeredCWDReachable is the existing registry/scope positive authority, +// also rechecked when registration races a negative checkout probe. +func (d *mcpDispatcher) registeredCWDReachable(cwd string) bool { + if d.isCWDTracked(cwd) { + return true + } + if d.multiIndexer != nil { + if _, _, _, ok := d.multiIndexer.ScopeForCWD(cwd); ok { + return true + } + if _, _, ok := d.multiIndexer.ContainedReposScope(cwd); ok { + return true + } + } + return false +} + // isCWDTracked reports whether the proxy's cwd lies inside any tracked // repo. Equal paths or any subdirectory of a tracked root qualify — // e.g. a proxy in ~/projects/myapp/internal counts as tracked when diff --git a/cmd/gortex/daemon_mcp_repo_loading.go b/cmd/gortex/daemon_mcp_repo_loading.go index 8bd3cc143..807a470b0 100644 --- a/cmd/gortex/daemon_mcp_repo_loading.go +++ b/cmd/gortex/daemon_mcp_repo_loading.go @@ -17,7 +17,8 @@ import ( // failure of the check: Dispatch unwraps it and answers the session with the // repository's load state instead of treating the cwd as untracked. type repoPendingAdmission struct { - load indexer.RepoLoad + load indexer.RepoLoad + familyInstructions string } func (a *repoPendingAdmission) Error() string { @@ -34,8 +35,15 @@ func (a *repoPendingAdmission) Error() string { // the session gets the untracked rules — capabilities and track still answer, // because tracking again is the repair — with repo_unavailable and the reason // instead of repo_not_tracked. -func (d *mcpDispatcher) pendingRepoRefusal(sess *daemon.Session, frame []byte, rec indexer.RepoLoad) []byte { +func (d *mcpDispatcher) pendingRepoRefusal(sess *daemon.Session, frame []byte, pending *repoPendingAdmission) []byte { + rec := pending.load method := peekFrameMethod(frame) + if pending.familyInstructions != "" { + if method == "tools/call" && !untrackedBootstrapCall(frame) { + return familyDiscoveryError(sess.CWD, frame, "family_pending") + } + return nil + } if !rec.State.Loading() { if method == "tools/call" && !untrackedBootstrapCall(frame) { return repoUnavailableError(sess, frame, rec) @@ -53,6 +61,9 @@ func (d *mcpDispatcher) pendingRepoRefusal(sess *daemon.Session, frame []byte, r case legacy == "index_health": return d.loadingIndexHealth(frame, rec) case loadingAdmittedTool(legacy): + if rec.State == indexer.RepoLoadIdentityPending && (legacy == "track_repository" || legacy == "index_repository") { + return d.repoLoadingError(sess, frame, rec) + } return nil default: return d.repoLoadingError(sess, frame, rec) @@ -104,6 +115,17 @@ func peekFrameToolCall(frame []byte) (string, map[string]any) { // repoLoadingError is the refusal for a graph-backed frame from a loading // session. Retryable, with progress, and it never suggests tracking. func (d *mcpDispatcher) repoLoadingError(sess *daemon.Session, inbound []byte, rec indexer.RepoLoad) []byte { + if rec.State == indexer.RepoLoadIdentityPending { + reason, phase, suggestion := "identity_pending", "identity_resolution", gortexmcp.ServerInstructionsIdentityPending() + if rec.Reason == "scope_binding_pending" { + reason, phase, suggestion = rec.Reason, "scope_binding", gortexmcp.ServerInstructionsScopeBindingPending() + } + return jsonRPCErrorFrame(inbound, "graph admission is pending", map[string]any{ + "error_code": string(gortexmcp.ErrCodeRepoLoading), "retryable": true, "path": sess.CWD, + "state": reason, "reason": reason, "phase": phase, "percent": 0, + "suggestion": suggestion, + }) + } phase, percent := "starting", 0 if d.srv != nil { phase, percent = d.srv.WarmupProgress() @@ -146,17 +168,20 @@ func (d *mcpDispatcher) loadingIndexHealth(inbound []byte, rec indexer.RepoLoad) if d.srv != nil { phase, percent = d.srv.WarmupProgress() } + repos := []map[string]any{} + message := gortexmcp.ServerInstructionsIdentityPending() + reason := "" + if rec.State == indexer.RepoLoadIdentityPending { + phase, percent, reason = "identity_resolution", 0, "identity_pending" + if rec.Reason == "scope_binding_pending" { + phase, reason, message = "scope_binding", rec.Reason, gortexmcp.ServerInstructionsScopeBindingPending() + } + } else { + repos = append(repos, map[string]any{"repo": rec.Prefix, "path": rec.Root, "state": string(rec.State), "since": rec.Since}) + message = gortexmcp.RepoLoadingSuggestion(rec.Label()) + } payload, _ := json.Marshal(map[string]any{ - "status": "loading", - "phase": phase, - "percent": percent, - "repos": []map[string]any{{ - "repo": rec.Prefix, - "path": rec.Root, - "state": string(rec.State), - "since": rec.Since, - }}, - "message": gortexmcp.RepoLoadingSuggestion(rec.Label()), + "status": "loading", "phase": phase, "percent": percent, "repos": repos, "message": message, "reason": reason, }) var req struct { ID json.RawMessage `json:"id"` diff --git a/cmd/gortex/daemon_mcp_repo_loading_test.go b/cmd/gortex/daemon_mcp_repo_loading_test.go index 18934ba35..5ed6c78ad 100644 --- a/cmd/gortex/daemon_mcp_repo_loading_test.go +++ b/cmd/gortex/daemon_mcp_repo_loading_test.go @@ -273,3 +273,23 @@ func TestDispatcher_LoadingRepo_ToolsWorkInPlaceAfterRegistration(t *testing.T) assert.Empty(t, errorCode(parsed), "the same session must be served after registration: %v", parsed) assert.Contains(t, toolResultText(t, parsed), "LoadProbe", "the session's scope must bind to the repository it is in") } + +func TestDispatcher_ColdTrackFailureBecomesUnavailablePromptly(t *testing.T) { + d, mi, registered, loading := loadingDispatcherSetup(t) + mi.StartRepoLoad(loading) + // Use the real cold-track path and a genuine ownership failure, not a + // synthetic FinishRepoLoad error. The existing owner must stay registered. + _, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: loading, Name: filepath.Base(registered)}) + var collision *indexer.PrefixCollisionError + require.ErrorAs(t, err, &collision) + failed := time.Now() + mi.FinishRepoLoad(loading, err) + sess := &daemon.Session{ID: "cold_failure", CWD: loading} + data := errorData(t, dispatchJSON(t, d, sess, + `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"LoadProbe"}}}`)) + assert.Equal(t, "repo_unavailable", data["error_code"]) + assert.Equal(t, false, data["retryable"]) + assert.Contains(t, data["reason"], "already used") + assert.Less(t, time.Since(failed), time.Second, "a completed cold-track failure must not remain repo_loading") + assert.NotNil(t, mi.GetMetadata(filepath.Base(registered)), "failure must not displace the prefix owner") +} diff --git a/cmd/gortex/daemon_repo_identity_pending_test.go b/cmd/gortex/daemon_repo_identity_pending_test.go new file mode 100644 index 000000000..ad6913037 --- /dev/null +++ b/cmd/gortex/daemon_repo_identity_pending_test.go @@ -0,0 +1,176 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/daemon" + "go.uber.org/zap" +) + +func TestDispatcher_ColdIdentityPendingHasNoInventedScope(t *testing.T) { + d, mi, registered, _ := loadingDispatcherSetup(t) + physical := filepath.Join(t.TempDir(), "physical-root") + require.NoError(t, os.MkdirAll(physical, 0755)) + alias := filepath.Join(t.TempDir(), "different-configured-name") + require.NoError(t, os.Symlink(physical, alias)) + entries := []config.RepoEntry{{Path: registered}, {Path: alias, Name: "private-config-name"}} + mi.DeclareRepos(entries) + for _, cwd := range []string{physical, t.TempDir()} { + sess := &daemon.Session{ID: "identity-" + filepath.Base(cwd), CWD: cwd} + d.SessionStarted(sess, func([]byte) error { return nil }) + t.Cleanup(func() { d.SessionEnded(sess) }) + init := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"identity-test","version":"1"}}}`) + body, err := json.Marshal(init) + require.NoError(t, err) + require.Contains(t, string(body), "identity_pending") + require.NotContains(t, string(body), "INACTIVE") + require.NotContains(t, string(body), "already configured") + require.NotContains(t, string(body), "private-config-name") + require.NotContains(t, string(body), "gortex track") + discover := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":8,"method":"server/discover","params":{"_meta":{"io.modelcontextprotocol/protocolVersion":"2026-07-28","io.modelcontextprotocol/clientInfo":{"name":"claude-code","version":"2.1.312"},"io.modelcontextprotocol/clientCapabilities":{}}}}`) + discovery, marshalErr := json.Marshal(discover) + require.NoError(t, marshalErr) + require.Contains(t, string(discovery), "identity_pending") + require.NotContains(t, string(discovery), "INACTIVE") + require.NotContains(t, string(discovery), "private-config-name") + start := time.Now() + reply := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search","arguments":{"query":"RegisteredOnly"}}}`) + require.Less(t, time.Since(start), 100*time.Millisecond, "negative identity admission never waits for resolution") + data := errorData(t, reply) + require.Equal(t, "repo_loading", data["error_code"]) + require.Equal(t, true, data["retryable"]) + require.Equal(t, "identity_resolution", data["phase"]) + require.EqualValues(t, 0, data["percent"]) + require.Equal(t, "identity_pending", data["reason"]) + require.NotContains(t, data, "repo") + require.NotContains(t, data, "repo_path") + require.NotContains(t, reply, "result") + for _, tool := range []string{ + `"name":"index_health","arguments":{}`, + `"name":"workspace","arguments":{"operation":"index"}`, + } { + health := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{`+tool+`}}`) + text := toolResultText(t, health) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(text), &payload)) + require.Equal(t, "identity_pending", payload["reason"]) + require.Empty(t, payload["repos"], "an unbound diagnostic must not disclose named repositories") + } + track := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"track_repository","arguments":{"path":"."}}}`) + require.Equal(t, "identity_pending", errorData(t, track)["reason"], "unknown ownership must not duplicate a configured alias through bootstrap tracking") + } + mi.ExpectRepos(entries) + physicalSess := &daemon.Session{ID: "resolved-alias", CWD: physical} + data := errorData(t, dispatchJSON(t, d, physicalSess, `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"search","arguments":{"query":"RegisteredOnly"}}}`)) + require.Equal(t, "repo_loading", data["error_code"]) + require.Equal(t, alias, data["repo_path"], "completed owner resolution can now prove configured ownership") + unknown, err := d.cwdReachableChecked(t.Context(), t.TempDir()) + require.NoError(t, err) + require.False(t, unknown, "a completed identity set permits a definitive negative") +} + +func TestDispatcher_IdentityPendingPreservesAuthoritativePositives(t *testing.T) { + d, mi, registered, _ := loadingDispatcherSetup(t) + mi.DeclareRepos([]config.RepoEntry{{Path: registered}, {Path: filepath.Join(t.TempDir(), "unresolved")}}) + require.True(t, mi.RepoIdentityPending()) + ok, err := d.cwdReachableChecked(t.Context(), registered) + require.NoError(t, err) + require.True(t, ok) + healthySession := &daemon.Session{ID: "registered-while-identity-pending", CWD: registered} + d.SessionStarted(healthySession, func([]byte) error { return nil }) + t.Cleanup(func() { d.SessionEnded(healthySession) }) + dispatchJSON(t, d, healthySession, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"positive-control","version":"1"}}}`) + healthyReply := dispatchJSON(t, d, healthySession, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search","arguments":{"query":"RegisteredOnly"}}}`) + require.Empty(t, errorCode(healthyReply)) + require.Contains(t, toolResultText(t, healthyReply), "RegisteredOnly") + checkout := t.TempDir() + d.checkoutCWDProbe = func(_ context.Context, cwd string) (bool, error) { return cwd == checkout, nil } + ok, err = d.cwdReachableChecked(t.Context(), checkout) + require.NoError(t, err) + require.True(t, ok, "an authoritative existing checkout still serves while another root is unresolved") + remote := t.TempDir() + require.NoError(t, writeFile(filepath.Join(remote, ".gortex.yaml"), "workspace: remote-ws\n")) + cfg := &daemon.ServersConfig{Server: []daemon.ServerEntry{ + {Slug: "self", URL: "unix:///tmp/never-dialed.sock", Default: true}, + {Slug: "remote", URL: "https://remote.example.com", Workspaces: []string{"remote-ws"}}, + }} + require.NoError(t, cfg.Validate()) + d.SetRouter(daemon.NewRouter(daemon.RouterConfig{Servers: cfg, Rosters: daemon.NewWorkspaceRosterCache(0), LocalSlug: "self", Logger: zap.NewNop()})) + ok, err = d.cwdReachableChecked(t.Context(), remote) + require.NoError(t, err) + require.True(t, ok, "router authority takes precedence over an unresolved negative") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = d.cwdReachableChecked(ctx, t.TempDir()) + require.ErrorIs(t, err, context.Canceled) +} + +func TestCoverage_IdentityPendingIsNeitherTrackedNorGraphAnswer(t *testing.T) { + _, mi, registered, _ := loadingDispatcherSetup(t) + mi.DeclareRepos([]config.RepoEntry{{Path: registered}, {Path: filepath.Join(t.TempDir(), "unresolved")}}) + c := &realController{multiIndexer: mi} + path := filepath.Join(t.TempDir(), "file.go") + file, err := c.FileCoverage(t.Context(), daemon.FileCoverageParams{Path: path}) + require.NoError(t, err) + require.Equal(t, daemon.RepoStateIdentityPending, file.RepoState) + require.Empty(t, file.Repo) + require.False(t, file.Tracked) + require.False(t, file.Answered) + require.False(t, file.Covered) + dir, err := c.DirCoverage(t.Context(), daemon.DirCoverageParams{Path: filepath.Dir(path)}) + require.NoError(t, err) + require.Equal(t, daemon.RepoStateIdentityPending, dir.RepoState) + require.Empty(t, dir.Repo) + require.False(t, dir.Tracked) + require.False(t, dir.Answered) + require.False(t, dir.HasSource) + healthy, err := c.FileCoverage(t.Context(), daemon.FileCoverageParams{Path: filepath.Join(registered, "a.go")}) + require.NoError(t, err) + require.Empty(t, healthy.RepoState) +} + +func TestExtractToolResult_IdentityPendingDoesNotClaimTracking(t *testing.T) { + _, err := extractToolResult([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"identity pending","data":{"error_code":"repo_loading","retryable":true,"reason":"identity_pending","phase":"identity_resolution","percent":0}}}`)) + var loading *repoLoadingError + require.ErrorAs(t, err, &loading) + require.Equal(t, exitTempFail, loading.ExitCode()) + require.Contains(t, err.Error(), "identity_pending") + require.NotContains(t, err.Error(), "already tracked") + require.NotContains(t, err.Error(), "gortex track") +} + +func TestDispatcher_ColdDarwinTmpAliasUsesIdentityPending(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("requires Darwin's /tmp -> /private/tmp alias") + } + canonical, err := filepath.EvalSymlinks("/tmp") + require.NoError(t, err) + if canonical != "/private/tmp" { + t.Skip("host does not expose the /tmp -> /private/tmp alias") + } + root, err := os.MkdirTemp("/tmp", "g1023-wire-alias-") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(root)) }) + d, mi := trackedPathMCPSetup(t, t.TempDir()) + mi.DeclareRepos([]config.RepoEntry{{Path: root}}) + sess := &daemon.Session{ID: "actual-tmp-alias", CWD: filepath.Join(canonical, filepath.Base(root))} + d.SessionStarted(sess, func([]byte) error { return nil }) + t.Cleanup(func() { d.SessionEnded(sess) }) + initialized := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"alias-test","version":"1"}}}`) + raw, err := json.Marshal(initialized) + require.NoError(t, err) + require.Contains(t, string(raw), "identity_pending") + require.NotContains(t, string(raw), "INACTIVE") + data := errorData(t, dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Anything"}}}`)) + require.Equal(t, "repo_loading", data["error_code"]) + require.Equal(t, "identity_pending", data["reason"]) + require.NotContains(t, data, "repo_path") +} diff --git a/cmd/gortex/daemon_repo_loading.go b/cmd/gortex/daemon_repo_loading.go index 0159b3d41..4e3074229 100644 --- a/cmd/gortex/daemon_repo_loading.go +++ b/cmd/gortex/daemon_repo_loading.go @@ -5,20 +5,27 @@ import ( "github.com/zzet/gortex/internal/indexer" ) -// announceConfiguredRepos makes every configured repository read as loading. -// -// The warmup goroutine calls it first, ahead of checkout seeding rather than -// when warmup reaches its job loop: seeding takes seconds on a large worktree -// farm, and for that whole window the admission gate otherwise told sessions -// their configured repository was untracked. It is not called before Listen: -// it stats every configured root, and a hung mount must not keep the socket -// from coming up. +// listenConfiguredDaemon publishes lexical configured roots before opening +// either listener. It performs no filesystem validation of those roots; warmup +// does that later so a hung mount cannot delay serving other configured repos. +func listenConfiguredDaemon(srv *daemon.Server, state *daemonState) error { + if state != nil && state.multiIndexer != nil && state.configManager != nil { + if gc := state.configManager.Global(); gc != nil { + state.multiIndexer.DeclareRepos(gc.ReposSnapshot()) + } + } + return srv.Listen() +} + +// announceConfiguredRepos fills the declarations' filesystem identity before +// checkout seeding. Serving has already started, so filesystem work stays here +// rather than in the lexical declaration that precedes Listen. func announceConfiguredRepos(state *daemonState) { if state == nil || state.multiIndexer == nil || state.configManager == nil { return } if gc := state.configManager.Global(); gc != nil { - state.multiIndexer.ExpectRepos(gc.Repos) + state.multiIndexer.ExpectRepos(gc.ReposSnapshot()) } } @@ -60,16 +67,12 @@ func publicRepoState(state indexer.RepoLoadState) string { return "" } -// loadingRepoForPath names the configured repository abs belongs to when that -// repository is still loading, for the coverage replies the hooks turn into a -// deny. Lock-free; reads no store. -func (c *realController) loadingRepoForPath(abs string) (repo string, loading bool) { +// pendingRepoLoadForPath reads one snapshot: ownership and unresolved identity +// must never be separated by a concurrent warmup publication. Ready records +// still require ordinary coverage authority, not permission from this table. +func (c *realController) pendingRepoLoadForPath(abs string) (indexer.RepoLoad, bool) { if c == nil || c.multiIndexer == nil { - return "", false - } - rec, pending := c.multiIndexer.PendingRepoForPath(abs) - if !pending || !rec.State.Loading() { - return "", false + return indexer.RepoLoad{}, false } - return rec.Label(), true + return c.multiIndexer.RepoLoadAdmissionForPath(abs) } diff --git a/cmd/gortex/daemon_repo_publication_test.go b/cmd/gortex/daemon_repo_publication_test.go new file mode 100644 index 000000000..e34f14ff7 --- /dev/null +++ b/cmd/gortex/daemon_repo_publication_test.go @@ -0,0 +1,146 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/indexer" +) + +func TestDispatcher_IdentityPublicationDuringCheckoutProbe(t *testing.T) { + for _, register := range []bool{false, true} { + name := "identity" + if register { + name = "registration" + } + t.Run(name, func(t *testing.T) { + d, mi, registered, _ := loadingDispatcherSetup(t) + physical := t.TempDir() + require.NoError(t, writeFile(filepath.Join(physical, "source.go"), "package source\nfunc Published() {}\n")) + alias := filepath.Join(t.TempDir(), "configured-alias") + require.NoError(t, os.Symlink(physical, alias)) + entries := []config.RepoEntry{{Path: registered}, {Path: alias, Name: "published"}} + mi.DeclareRepos(entries) + calls := 0 + d.checkoutCWDProbe = func(ctx context.Context, path string) (bool, error) { + calls++ + require.Equal(t, physical, path) + mi.ExpectRepos(entries) + if register { + _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: physical, Name: "published"}) + require.NoError(t, err) + } + return false, nil // The earlier negative probe is now stale. + } + reachable, err := d.cwdReachableChecked(t.Context(), physical) + require.Equal(t, 1, calls) + if register { + require.NoError(t, err) + require.True(t, reachable, "normal registered scope must be rechecked after publication") + } else { + var pending *repoPendingAdmission + require.ErrorAs(t, err, &pending) + require.False(t, reachable) + require.Equal(t, alias, pending.load.Root) + require.Equal(t, indexer.RepoLoadQueued, pending.load.State) + } + }) + } +} + +func TestCoverage_IdentityPublicationDuringPublicViewProbe(t *testing.T) { + for _, door := range []string{"file", "directory"} { + t.Run(door, func(t *testing.T) { + _, mi, registered, _ := loadingDispatcherSetup(t) + physical := t.TempDir() + alias := filepath.Join(t.TempDir(), "configured-alias") + require.NoError(t, os.Symlink(physical, alias)) + entries := []config.RepoEntry{{Path: registered}, {Path: alias, Name: "published"}} + mi.DeclareRepos(entries) + c := &realController{multiIndexer: mi} + probe := func(ctx context.Context, path string) probeView { + view := c.resolveProbeView(ctx, path) + mi.ExpectRepos(entries) + return view + } + if door == "file" { + result, err := c.fileCoverageWithProbe(t.Context(), daemon.FileCoverageParams{Path: filepath.Join(physical, "source.go")}, probe) + require.NoError(t, err) + require.Equal(t, daemon.RepoStateLoading, result.RepoState) + require.Equal(t, "published", result.Repo) + require.False(t, result.Covered) + } else { + result, err := c.dirCoverageWithProbe(t.Context(), daemon.DirCoverageParams{Path: physical}, probe) + require.NoError(t, err) + require.Equal(t, daemon.RepoStateLoading, result.RepoState) + require.Equal(t, "published", result.Repo) + require.False(t, result.HasSource) + } + }) + } +} + +func TestDispatcher_ReadyFamilyIdentityDoesNotGrantUnprovenCheckoutScope(t *testing.T) { + d, mi, registered, primary := loadingDispatcherSetup(t) + linked := t.TempDir() + admin := filepath.Join(primary, ".git", "worktrees", "linked") + require.NoError(t, os.MkdirAll(admin, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(admin, "commondir"), []byte("../.."), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(linked, ".git"), []byte("gitdir: "+admin), 0644)) + entries := []config.RepoEntry{{Path: registered}, {Path: primary, Name: "loading"}} + mi.DeclareRepos(entries) + calls := 0 + d.checkoutCWDProbe = func(ctx context.Context, path string) (bool, error) { + calls++ + require.Equal(t, linked, path) + if calls == 1 { + mi.ExpectRepos(entries) + _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: primary, Name: "loading"}) + require.NoError(t, err) + } + return false, nil // No authoritative checkout binding is available yet. + } + sess := &daemon.Session{ID: "registered-family-not-route", CWD: linked} + reply := dispatchJSON(t, d, sess, `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"LoadProbe"}}}`) + require.Equal(t, 2, calls, "a matched-ready identity permits only one ordinary authority recheck") + data := errorData(t, reply) + require.Equal(t, "view_building", data["error_code"]) + require.Equal(t, "family_pending", data["reason"]) + require.Equal(t, "discovery_pending", data["state"]) + require.NotContains(t, data, "repo_path") + require.NotContains(t, reply, "result", "family identity alone grants no graph scope") + // Retain the actual typed result before a later filesystem change. Metadata + // rendering and graph refusal must not re-probe and become untracked. + d.checkoutCWDProbe = nil + _, err := d.cwdReachableChecked(t.Context(), linked) + var classified *repoPendingAdmission + require.ErrorAs(t, err, &classified) + require.NotEmpty(t, classified.familyInstructions) + require.NoError(t, os.Remove(filepath.Join(linked, ".git"))) + frame := []byte(`{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"read","arguments":{}}}`) + require.Contains(t, string(d.pendingRepoRefusal(sess, frame, classified)), `"error_code":"view_building"`) + initialized := rewriteMCPAdmissionInstructions("initialize", []byte(`{"jsonrpc":"2.0","id":8,"result":{"instructions":"INACTIVE"}}`), classified.familyInstructions) + require.Contains(t, string(initialized), "view_building") + require.NotContains(t, string(initialized), "INACTIVE") + unknown := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(unknown, ".git"), 0755)) + require.False(t, mi.RepoIdentityPending()) + outside := &daemon.Session{ID: "foreign-family", CWD: unknown} + negative := dispatchJSON(t, d, outside, `{"jsonrpc":"2.0","id":9,"method":"tools/call","params":{"name":"read","arguments":{}}}`) + require.Equal(t, "repo_not_tracked", errorCode(negative), "a genuine foreign family stays definitively untracked after identity completion") +} + +func TestExtractToolResult_ScopeBindingPendingDoesNotClaimTracking(t *testing.T) { + _, err := extractToolResult([]byte(`{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"scope binding pending","data":{"error_code":"repo_loading","retryable":true,"reason":"scope_binding_pending","phase":"scope_binding","percent":0}}}`)) + var loading *repoLoadingError + require.ErrorAs(t, err, &loading) + require.Equal(t, exitTempFail, loading.ExitCode()) + require.Contains(t, err.Error(), "scope_binding_pending") + require.NotContains(t, err.Error(), "already tracked") + require.NotContains(t, err.Error(), "gortex track") +} diff --git a/cmd/gortex/daemon_startup_loading_test.go b/cmd/gortex/daemon_startup_loading_test.go new file mode 100644 index 000000000..64c12ee46 --- /dev/null +++ b/cmd/gortex/daemon_startup_loading_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/daemon" + gortexmcp "github.com/zzet/gortex/internal/mcp" +) + +// Exercise the real pre-listen helper and public MCP socket before ANY warmup +// announcement, seeding or repository indexing. Starting registration by hand +// here would conceal the startup window this test is intended to cover. +func TestConfiguredLoadingPublishedBeforeServing(t *testing.T) { + registered := t.TempDir() + d, mi := trackedPathMCPSetup(t, registered) + root := filepath.Join(t.TempDir(), "configured") + admin := filepath.Join(root, ".git", "worktrees", "linked") + linked := filepath.Join(t.TempDir(), "linked") + require.NoError(t, os.MkdirAll(admin, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(root, "pkg"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(linked, "pkg"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(admin, "commondir"), []byte("../..\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(linked, ".git"), []byte("gitdir: "+admin+"\n"), 0o600)) + c := probeController(t, fmt.Sprintf("repos:\n - path: %q\n name: startup\n", root)) + state := &daemonState{multiIndexer: mi, configManager: c.configManager} + + file, err := os.CreateTemp(os.TempDir(), "gx-startup-*.sock") + require.NoError(t, err) + socket := file.Name() + require.NoError(t, file.Close()) + require.NoError(t, os.Remove(socket)) + t.Setenv("GORTEX_DAEMON_PIDFILE", filepath.Join(t.TempDir(), "daemon.pid")) + t.Setenv("GORTEX_DAEMON_STATEFILE", filepath.Join(t.TempDir(), "daemon.state.json")) + srv := daemon.New(socket, "test", zap.NewNop()) + srv.MCPDispatcher = d + require.NoError(t, listenConfiguredDaemon(srv, state)) + served := make(chan error, 1) + go func() { served <- srv.Serve() }() + t.Cleanup(func() { + require.NoError(t, srv.Shutdown()) + select { + case err := <-served: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Error("owned startup test server did not stop") + } + }) + + for name, cwd := range map[string]string{ + "primary": root, "subdirectory": filepath.Join(root, "pkg"), + "linked": linked, "linked subdirectory": filepath.Join(linked, "pkg"), + } { + t.Run(name, func(t *testing.T) { + client, err := daemon.DialTo(socket, daemon.Handshake{Mode: daemon.ModeMCP, CWD: cwd}) + require.NoError(t, err) + defer client.Close() + require.NoError(t, client.Conn.SetDeadline(time.Now().Add(5*time.Second))) + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"startup-probe","version":"1"}}}`))) + initialized, err := client.ReadMCPFrame() + require.NoError(t, err) + var initReply struct { + Result struct { + Instructions string `json:"instructions"` + } `json:"result"` + } + require.NoError(t, json.Unmarshal(initialized, &initReply)) + expectedInstructions := gortexmcp.ServerInstructionsLoading("startup", "starting", 0) + if strings.HasPrefix(name, "linked") { + expectedInstructions = gortexmcp.ServerInstructionsIdentityPending() + } + assert.Contains(t, initReply.Result.Instructions, expectedInstructions, + "the first real initialize must carry an honest loading variant before identity warmup") + require.NoError(t, client.WriteMCPFrame([]byte(`{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Startup"}}}`))) + frame, err := client.ReadMCPFrame() + require.NoError(t, err) + var reply map[string]any + require.NoError(t, json.Unmarshal(frame, &reply)) + data := errorData(t, reply) + assert.Equal(t, "repo_loading", data["error_code"]) + assert.Equal(t, true, data["retryable"]) + if strings.HasPrefix(name, "linked") { + assert.Equal(t, "identity_pending", data["reason"]) + assert.NotContains(t, data, "repo_path", "an unresolved lexical declaration cannot name a linked request's family") + } else { + assert.Equal(t, root, data["repo_path"]) + } + assert.Contains(t, data, "phase") + assert.Contains(t, data, "percent") + assert.NotContains(t, string(frame), "gortex track") + assert.NotContains(t, string(frame), "repos reconcile") + }) + } + assert.Empty(t, mi.GetMetadata("startup"), "pending declarations must never register or admit graph access") + untracked := &daemon.Session{ID: "unconfigured", CWD: t.TempDir()} + before := errorData(t, dispatchJSON(t, d, untracked, + `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Startup"}}}`)) + assert.Equal(t, "repo_loading", before["error_code"]) + assert.Equal(t, "identity_pending", before["reason"]) + assert.NotContains(t, before, "repo_path") + // The real existing warmup owner now resolves identity. A linked path can + // then name its configured family and an unrelated path is truly negative. + announceConfiguredRepos(state) + linkedSession := &daemon.Session{ID: "resolved-linked", CWD: linked} + after := errorData(t, dispatchJSON(t, d, linkedSession, + `{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Startup"}}}`)) + assert.Equal(t, "repo_loading", after["error_code"]) + assert.Equal(t, root, after["repo_path"]) + assert.Equal(t, "repo_not_tracked", errorCode(dispatchJSON(t, d, untracked, + `{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"Startup"}}}`))) +} diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 8f097b733..9c252ddec 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -84,6 +84,9 @@ type daemonState struct { // record so out-of-band readers (`gortex repos`) can find the same store // instead of assuming the platform default. backendPath string + // migratedStore is scoped to this open/boot, not persisted. A migration + // gets one complete re-index/enrichment boot before VACUUM is reconsidered. + migratedStore bool } // buildDaemonState builds the daemon's stack through the shared @@ -104,6 +107,15 @@ func buildDaemonState(logger *zap.Logger, observers ...store_sqlite.MigrationObs if len(observers) > 0 { migrationObserver = observers[0] } + migrated := false + observeMigration := func(progress store_sqlite.MigrationProgress) { + if progress.Phase == store_sqlite.MigrationStarted { + migrated = true + } + if migrationObserver != nil { + migrationObserver(progress) + } + } ss, err := serverstack.NewSharedServer(serverstack.SharedServerConfig{ Lifecycle: serverstack.LifecycleDaemon, Backend: daemonBackend, @@ -112,7 +124,7 @@ func buildDaemonState(logger *zap.Logger, observers ...store_sqlite.MigrationObs Global: gc, Logger: logger, Version: version, - MigrationObserver: migrationObserver, + MigrationObserver: observeMigration, Embedder: serverstack.EmbedderRequest{ FlagChanged: daemonEmbeddingsChanged, FlagEnabled: daemonEmbeddings, @@ -143,6 +155,7 @@ func buildDaemonState(logger *zap.Logger, observers ...store_sqlite.MigrationObs overlays: ss.Overlays, shared: ss, backendPath: ss.StorePath, + migratedStore: migrated || storeNeedsRebuild(ss.Graph), resolverLSPRegistry: ss.ResolverLSPRegistry, lspRouter: ss.LSPRouter, } @@ -607,24 +620,9 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z return nil, timings } - // One-time store compaction, guarded (see daemon_compact.go). Placed HERE - // on purpose: after the orphan purge, so the pages it just freed are - // measured and reclaimed in the same pass; before the warmup re-index loop - // and the resolver/enrichment passes, whose writes would reuse freelist - // pages mid-measurement and whose readers would contend with VACUUM's - // exclusive lock. The socket is already listening (it opens before this - // goroutine by design), but at this point in boot the daemon's own - // workload is quiet and a rare early client query either finishes first or - // makes the VACUUM fail cleanly at busy_timeout — a skip, not a fault. - // Running before the socket opened instead would hold the daemon - // unreachable for the whole VACUUM, turning a compacting boot into an - // apparent hang. - finishCompact := startWarmupStep(logger, "store_compaction_check") - maybeCompactStore(state.graph, logger) - finishCompact() - if ctx.Err() != nil { - return nil, timings - } + // Re-index and resolver writes can reuse pages freed by the purge. Boot + // compaction measures again only after enrichment and the warmup producer + // have finished; a VACUUM must never delay this startup path. // Register a per-repo resolver-time LSP helper for every // tracked repo BEFORE the parallel warmup loop fires. The diff --git a/cmd/gortex/daemon_status_repo_counts_test.go b/cmd/gortex/daemon_status_repo_counts_test.go index 9cb431a04..e558d44c8 100644 --- a/cmd/gortex/daemon_status_repo_counts_test.go +++ b/cmd/gortex/daemon_status_repo_counts_test.go @@ -54,7 +54,7 @@ func TestStatus_TrackedRepoRowFollowsTheLiveGraph(t *testing.T) { reg := parser.NewRegistry() languages.RegisterAll(reg) mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() + _, err = mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dir}) require.NoError(t, err) metas := mi.AllMetadata() @@ -130,7 +130,7 @@ func TestStatus_TwoReposEachReportTheirOwnBucket(t *testing.T) { // A is indexed while it is the only configured repo. g := graph.New() mi1 := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi1.IndexAll() + _, err = mi1.TrackRepoCtx(context.Background(), config.RepoEntry{Path: dirA}) require.NoError(t, err) var priorA map[string]int64 for _, m := range mi1.AllMetadata() { diff --git a/cmd/gortex/daemon_status_storage_test.go b/cmd/gortex/daemon_status_storage_test.go index 24971a06d..4c592295d 100644 --- a/cmd/gortex/daemon_status_storage_test.go +++ b/cmd/gortex/daemon_status_storage_test.go @@ -193,6 +193,33 @@ func TestStatusCarriesStorageAndBuildLaneWhileBusy(t *testing.T) { assert.Nil(t, st.Views, "the holder's identity is its own block, never the views census") } +func TestStatusWALObservationCausesAndNullablePeak(t *testing.T) { + for _, observed := range []bool{false, true} { + snapshot := store_sqlite.WALReclaimStats{ + CancelledWriterParked: 3, CancelledWriteIntent: 2, CancelledOther: 1, + WALPeakObserved: observed, + } + got := walReclaimStatus(snapshot) + require.EqualValues(t, 3, got.CancelledWriterParked) + require.EqualValues(t, 2, got.CancelledWriteIntent) + require.EqualValues(t, 1, got.CancelledOther) + require.Equal(t, observed, got.WALPeakObserved) + encoded, err := json.Marshal(got) + require.NoError(t, err) + var wire map[string]any + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.Contains(t, wire, "observed_peak_wal_bytes") + if observed { + require.NotNil(t, got.ObservedPeakWALBytes) + require.Zero(t, *got.ObservedPeakWALBytes) + require.Equal(t, float64(0), wire["observed_peak_wal_bytes"]) + } else { + require.Nil(t, got.ObservedPeakWALBytes) + require.Nil(t, wire["observed_peak_wal_bytes"]) + } + } +} + func TestStatusStorageReportsTheDaemonsClosePolicy(t *testing.T) { for _, mode := range []string{"skip", "full"} { t.Run(mode, func(t *testing.T) { diff --git a/cmd/gortex/daemon_streamable.go b/cmd/gortex/daemon_streamable.go index e90558b11..34e76398f 100644 --- a/cmd/gortex/daemon_streamable.go +++ b/cmd/gortex/daemon_streamable.go @@ -111,7 +111,43 @@ func (d *daemonStreamableDispatcher) Dispatch(ctx context.Context, frame []byte) defer d.unbridge(sess.ID) defer d.registry.RemoveByID(sess.ID) } - return d.inner.Dispatch(ctx, sess, frame) + inner := d.inner + if route := streamable.ToolRouteFromContext(ctx); route != nil { + inner = transportRoutedDispatcher{MCPDispatcher: d.inner, route: route} + } + reply, err := daemon.DispatchMCPFrame(ctx, sess, frame, inner) + if busy := daemon.MCPDispatchBusyResponse(frame, err); len(busy) > 0 { + return busy, nil + } + return reply, err +} + +// transportRoutedDispatcher runs the streamable transport's own tools/call +// routing attempt once shared admission has granted the frame a slot, then +// falls through to the wrapped dispatcher for a call that stays local. The +// order is the one the transport used when it routed before dispatch, so a +// remote hop is admitted like a local call without changing what it routes on +// (session workspace pin, the call's cwd) or how its failure is framed. +type transportRoutedDispatcher struct { + daemon.MCPDispatcher + route streamable.ToolRoute +} + +func (d transportRoutedDispatcher) Dispatch(ctx context.Context, sess *daemon.Session, frame []byte) ([]byte, error) { + if out, ok := d.route(ctx); ok { + return out, nil + } + return d.MCPDispatcher.Dispatch(ctx, sess, frame) +} + +// DispatchClass keeps the wrapped dispatcher's admission class: a routed call +// is classified by its local name, exactly like the unwrapped dispatch. +func (d transportRoutedDispatcher) DispatchClass(name string, arguments map[string]any) daemon.MCPDispatchClass { + if classifier, ok := d.MCPDispatcher.(daemon.MCPDispatchClassifier); ok { + return classifier.DispatchClass(name, arguments) + } + class, _ := daemon.ToolDispatchClass(name) + return class } // acquire returns the synthetic daemon.Session for a streamable @@ -218,11 +254,12 @@ func buildDaemonStreamableHandler(disp daemon.MCPDispatcher, reg *daemon.Session store := streamable.NewMemoryStore(sessionTTL) wrapped := wrapStreamableStoreWithCleanup(store, bridge.onSessionEnded) transport := streamable.New(streamable.Config{ - Dispatcher: bridge, - Store: wrapped, - Logger: logger, - Router: router, - SessionTTL: sessionTTL, + Dispatcher: bridge, + Store: wrapped, + Logger: logger, + Router: router, + DispatcherRoutesTools: true, + SessionTTL: sessionTTL, InitializeHook: func(_ context.Context, state *streamable.SessionState) { // Bridge the new streamable session into the daemon's // session registry up-front so the daemon-status block diff --git a/cmd/gortex/ensure_daemon.go b/cmd/gortex/ensure_daemon.go index eb5af8f47..a62dc011e 100644 --- a/cmd/gortex/ensure_daemon.go +++ b/cmd/gortex/ensure_daemon.go @@ -24,16 +24,19 @@ const ( daemonUnavailable // autostart off, or spawn failed/timed out ) +// spawnLockInactivityTimeout is deliberately an inactivity budget, not a +// total startup budget. The lock winner holds the spawn lock until the +// detached child's socket is live; a schema migration can legitimately keep +// that socket closed for longer than a minute. Fresh PID-bound startup state +// extends this deadline in the same way spawnDetachedDaemon extends its own +// wait, while a missing/stale heartbeat still releases callers promptly. +// Tests shorten it only in an isolated helper process; there is no user +// configuration or environment override for this production budget. +var spawnLockInactivityTimeout = 60 * time.Second + const ( - // spawnLockInactivityTimeout is deliberately an inactivity budget, not a - // total startup budget. The lock winner holds the spawn lock until the - // detached child's socket is live; a schema migration can legitimately keep - // that socket closed for longer than a minute. Fresh PID-bound startup state - // extends this deadline in the same way spawnDetachedDaemon extends its own - // wait, while a missing/stale heartbeat still releases callers promptly. - spawnLockInactivityTimeout = 60 * time.Second - spawnLockPollInterval = 100 * time.Millisecond - spawnProgressFreshness = 10 * time.Second + spawnLockPollInterval = 100 * time.Millisecond + spawnProgressFreshness = 10 * time.Second // spawnFailCooldown bounds serial retries of a broken spawn: within // the window at most one spawn attempt runs across contending callers. spawnFailCooldown = 5 * time.Second diff --git a/cmd/gortex/migration_window_proxy_test.go b/cmd/gortex/migration_window_proxy_test.go new file mode 100644 index 000000000..3277f50a1 --- /dev/null +++ b/cmd/gortex/migration_window_proxy_test.go @@ -0,0 +1,348 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/gofrs/flock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/graph/store_sqlite" + gortexmcp "github.com/zzet/gortex/internal/mcp" + "github.com/zzet/gortex/internal/testenv" +) + +const migrationProxyTestInactivity = 50 * time.Millisecond + +type migrationProxyWaitWitness struct { + At time.Time `json:"at"` + Checks int `json:"checks"` + LockContended bool `json:"lock_contended"` + State daemon.RuntimeState `json:"state"` +} + +type migrationProxyHelperInput struct { + Root string `json:"root"` + ParentPID int `json:"parent_pid"` + SpawnLock string `json:"spawn_lock"` + GlobalPath string `json:"global_config"` +} + +// The helper's TestMain installs a new sandbox before this test runs. Restore +// only these paths derived from its parent's fresh short fixture, not an +// inherited environment dump or any of the user's per-user state. +func migrationProxyOwnedEnv(root string) map[string]string { + home := filepath.Join(root, "home") + return map[string]string{ + "HOME": home, "USERPROFILE": home, + "AppData": filepath.Join(home, "AppData", "Roaming"), + "LocalAppData": filepath.Join(home, "AppData", "Local"), + "XDG_CONFIG_HOME": filepath.Join(root, "config"), + "XDG_DATA_HOME": filepath.Join(root, "data"), + "XDG_CACHE_HOME": filepath.Join(root, "cache"), + "XDG_STATE_HOME": filepath.Join(root, "state"), + "XDG_RUNTIME_DIR": filepath.Join(root, "runtime"), + "GORTEX_DAEMON_SOCKET": filepath.Join(root, "s"), + "GORTEX_DAEMON_PIDFILE": filepath.Join(root, "p"), + "GORTEX_DAEMON_STATEFILE": filepath.Join(root, "state.json"), + "GORTEX_DAEMON_LOGFILE": filepath.Join(root, "daemon.log"), + "GORTEX_DAEMON_SNAPSHOT": filepath.Join(root, "daemon.gob.gz"), + } +} + +// This helper runs the real MCP selection and proxy in a fresh process. Only +// that process shortens the private inactivity budget. Its liveness seam still +// returns the actual socket result; the extra lock/heartbeat reads only record +// that it reached the contended wait before the parent begins its hold. +func TestMigrationWindowMCPProxyHelper(t *testing.T) { + if os.Getenv("GORTEX_TEST_MIGRATION_PROXY") != "1" { + t.Skip("invoked only by TestMCPWaitsThroughMigrationWindow") + } + spawnLockInactivityTimeout = migrationProxyTestInactivity + inputPath := os.Getenv("GORTEX_TEST_MIGRATION_INPUT") + require.True(t, filepath.IsAbs(inputPath)) + body, err := os.ReadFile(inputPath) + require.NoError(t, err) + var input migrationProxyHelperInput + require.NoError(t, json.Unmarshal(body, &input)) + require.Equal(t, os.Getppid(), input.ParentPID) + require.True(t, filepath.IsAbs(input.Root)) + require.True(t, strings.HasPrefix(filepath.Base(input.Root), "gx")) + require.Equal(t, filepath.Join(input.Root, "migration-helper-input.json"), inputPath) + for name, value := range migrationProxyOwnedEnv(input.Root) { + t.Setenv(name, value) + } + require.Equal(t, input.SpawnLock, daemon.SpawnLockPath()) + require.Equal(t, input.GlobalPath, config.DefaultGlobalConfigPath()) + witnessDir := input.Root + checks := 0 + isDaemonRunning = func() bool { + running := daemon.IsRunning() + checks++ + if !running && checks >= 2 { + probe := flock.New(daemon.SpawnLockPath()) + locked, err := probe.TryLock() + require.NoError(t, err) + require.NoError(t, probe.Close()) + if locked { + return running // the parent may just have released startup + } + state, ok := daemon.ReadRuntimeState() + if !ok || state.StartupPhase != daemon.StartupMigrating { + return running // migration may have completed before Listen + } + require.True(t, state.StartupProgressFresh(time.Now(), spawnProgressFreshness)) + w := migrationProxyWaitWitness{At: time.Now(), Checks: checks, LockContended: true, State: state} + body, err := json.Marshal(w) + require.NoError(t, err) + path := filepath.Join(witnessDir, "latest.json") + if checks == 2 { + path = filepath.Join(witnessDir, "first.json") + } + require.NoError(t, os.WriteFile(path+".tmp", body, 0o600)) + require.NoError(t, os.Rename(path+".tmp", path)) + } + return running + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + mcpCmd.SetContext(ctx) + require.NoError(t, runMCP(mcpCmd, nil)) +} + +// This exercises an actual pending SQLite migration, heartbeat file, spawn +// lock, runMCP selection, stdio relay and public loading reply. The short hold +// proves renewal from a fresh PID-bound startup record, not the reporter's +// two-second periodic heartbeat. Historical schema-v21 workload acceptance is +// separate; this fixture deliberately replays the idempotent v40 step. +func TestMCPWaitsThroughMigrationWindow(t *testing.T) { + base := testenv.ShortTempDir(t) + // Keep this migration regression's config and actual child CWD under the + // same physical spelling. Alias admission is a separate unresolved case; + // /tmp versus /private/tmp must not conceal the migration wait contract. + base, err := filepath.EvalSymlinks(base) + require.NoError(t, err) + testenv.SandboxAt(t, filepath.Join(base, "home")) + for name, value := range migrationProxyOwnedEnv(base) { + t.Setenv(name, value) + } + t.Setenv("GORTEX_AUTOSTART", "1") + t.Setenv("GORTEX_TELEMETRY", "0") + t.Setenv("GORTEX_TOOLS", "") + root := filepath.Join(base, "repo") + require.NoError(t, os.MkdirAll(root, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "main.go"), []byte("package p\nfunc MigrationProbe() {}\n"), 0o600)) + globalPath := config.DefaultGlobalConfigPath() + require.NoError(t, os.MkdirAll(filepath.Dir(globalPath), 0o700)) + // Opting into the fallback makes a premature embedded selection visible; + // it must not replace the configured daemon during an active migration. + require.NoError(t, os.WriteFile(globalPath, []byte(fmt.Sprintf("repos:\n - path: %q\n name: migrating\nmcp:\n allow_embedded: true\n", root)), 0o600)) + oldConfig, oldPath, oldBackend := cfgFile, daemonBackendPath, daemonBackend + t.Cleanup(func() { cfgFile, daemonBackendPath, daemonBackend = oldConfig, oldPath, oldBackend }) + cfgFile, daemonBackendPath, daemonBackend = filepath.Join(base, "repo.yaml"), filepath.Join(base, "store.sqlite"), "sqlite" + require.NoError(t, os.WriteFile(cfgFile, []byte("semantic:\n enabled: false\n"), 0o600)) + seed, err := store_sqlite.Open(daemonBackendPath) + require.NoError(t, err) + closeSeed := sync.OnceValue(seed.Close) + t.Cleanup(func() { _ = closeSeed() }) + require.NoError(t, closeSeed()) + db, err := sql.Open("sqlite", daemonBackendPath) + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec("PRAGMA user_version=39") + require.NoError(t, err) + require.NoError(t, db.Close()) + + lock := flock.New(daemon.SpawnLockPath()) + require.NoError(t, os.MkdirAll(filepath.Dir(daemon.SpawnLockPath()), 0o700)) + require.NoError(t, lock.Lock()) + t.Cleanup(func() { require.NoError(t, lock.Close()) }) + reporter := newDaemonStartupReporter(zap.NewNop()) + t.Cleanup(reporter.Stop) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + type buildResult struct { + state *daemonState + err error + } + built := make(chan buildResult, 1) + go func() { + state, buildErr := buildDaemonState(zap.NewNop(), func(p store_sqlite.MigrationProgress) { + reporter.ObserveMigration(p) + if p.Version == 40 && p.Phase == store_sqlite.MigrationStarted { + close(entered) + <-release + } + }) + built <- buildResult{state, buildErr} + }() + var result buildResult + buildJoined := false + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + if !buildJoined { + select { + case result = <-built: + case <-time.After(10 * time.Second): + t.Error("migration build did not join") + return + } + } + if result.state != nil { + require.NoError(t, result.state.shared.Close()) + } + }) + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("the actual v40 migration did not start") + } + state, ok := daemon.ReadRuntimeState() + require.True(t, ok) + require.Equal(t, daemon.StartupMigrating, state.StartupPhase) + require.Equal(t, os.Getpid(), state.PID) + require.False(t, daemon.IsRunning(), "the real socket must be absent during migration") + + executable, err := os.Executable() + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + child := exec.CommandContext(ctx, executable, "-test.run=^TestMigrationWindowMCPProxyHelper$") + child.Dir = root + inputPath := filepath.Join(base, "migration-helper-input.json") + input, err := json.Marshal(migrationProxyHelperInput{ + Root: base, ParentPID: os.Getpid(), SpawnLock: daemon.SpawnLockPath(), GlobalPath: globalPath, + }) + require.NoError(t, err) + require.NoError(t, os.WriteFile(inputPath, input, 0o600)) + child.Env = append(os.Environ(), "GORTEX_TEST_MIGRATION_PROXY=1", "GORTEX_TEST_MIGRATION_INPUT="+inputPath) + stdin, err := child.StdinPipe() + require.NoError(t, err) + stdout, err := child.StdoutPipe() + require.NoError(t, err) + var stderr bytes.Buffer + child.Stderr = &stderr + require.NoError(t, child.Start()) + exited := make(chan error, 1) + go func() { exited <- child.Wait() }() + reaped := false + t.Cleanup(func() { + _ = stdin.Close() + cancel() + if !reaped { + select { + case <-exited: + case <-time.After(2 * time.Second): + t.Error("owned MCP helper did not reap") + } + } + }) + frames := streamProxyFrames(ctx, stdout) + _, err = fmt.Fprintln(stdin, `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-03-26","clientInfo":{"name":"migration-probe","version":"1"}}}`) + require.NoError(t, err) + var first migrationProxyWaitWitness + require.Eventually(t, func() bool { + body, readErr := os.ReadFile(filepath.Join(base, "first.json")) + return readErr == nil && json.Unmarshal(body, &first) == nil + }, 5*time.Second, 5*time.Millisecond) + require.True(t, first.LockContended) + require.Equal(t, os.Getpid(), first.State.PID) + require.Equal(t, daemon.StartupMigrating, first.State.StartupPhase) + // Time the hold only after an independent subprocess has witnessed both + // the actual failed flock and the actual active migration heartbeat. + select { + case frame := <-frames: + t.Fatalf("premature fallback/embedded output during migration: %s (%v)", frame.frame, frame.err) + case childErr := <-exited: + reaped = true + t.Fatalf("MCP exited during migration: %v", childErr) + case <-time.After(3 * migrationProxyTestInactivity): + } + var latest migrationProxyWaitWitness + body, err := os.ReadFile(filepath.Join(base, "latest.json")) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(body, &latest)) + require.Greater(t, latest.Checks, first.Checks, "the real contended wait must continue after its initial inactivity budget") + require.True(t, latest.LockContended) + require.Equal(t, first.State.PID, latest.State.PID) + select { + case frame := <-frames: + t.Fatalf("MCP produced output before migration release: %s (%v)", frame.frame, frame.err) + default: + } + releaseOnce.Do(func() { close(release) }) + select { + case result = <-built: + buildJoined = true + case <-ctx.Done(): + t.Fatal("migration did not complete after release") + } + require.NoError(t, result.err) + srv := daemon.New(os.Getenv("GORTEX_DAEMON_SOCKET"), "test", zap.NewNop()) + srv.MCPDispatcher = newMCPDispatcher(result.state.mcpServer, result.state.multiIndexer, zap.NewNop()) + require.NoError(t, listenConfiguredDaemon(srv, result.state)) + reporter.Serving(daemonBackendPath) + done := make(chan error, 1) + go func() { done <- srv.Serve() }() + t.Cleanup(func() { + require.NoError(t, srv.Shutdown()) + select { + case serveErr := <-done: + require.NoError(t, serveErr) + case <-time.After(5 * time.Second): + t.Error("owned migration-window server did not join") + } + }) + require.NoError(t, lock.Unlock()) + readReply := func() []byte { + t.Helper() + select { + case frame := <-frames: + require.NoError(t, frame.err) + return frame.frame + case <-ctx.Done(): + t.Fatal("MCP reply deadline") + return nil + } + } + var initialized struct { + Result struct{ Instructions string } `json:"result"` + } + require.NoError(t, json.Unmarshal(readReply(), &initialized)) + assert.Contains(t, initialized.Result.Instructions, gortexmcp.ServerInstructionsLoading("migrating", "starting", 0)) + _, err = fmt.Fprintln(stdin, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search_symbols","arguments":{"query":"MigrationProbe"}}}`) + require.NoError(t, err) + reply := readReply() + var parsed map[string]any + require.NoError(t, json.Unmarshal(reply, &parsed)) + data := errorData(t, parsed) + assert.Equal(t, "repo_loading", data["error_code"]) + assert.Equal(t, true, data["retryable"]) + assert.Contains(t, data, "phase") + assert.Contains(t, data, "percent") + assert.NotContains(t, string(reply), "gortex track") + assert.NotContains(t, string(reply), "repos reconcile") + require.NoError(t, stdin.Close()) + select { + case childErr := <-exited: + reaped = true + require.NoError(t, childErr, "%s", stderr.String()) + case <-ctx.Done(): + t.Fatal("MCP helper did not exit after closing stdio") + } + assert.NotContains(t, stderr.String(), "embedded") +} diff --git a/cmd/gortex/repo_inventory_missing_test.go b/cmd/gortex/repo_inventory_missing_test.go index 53ff7d2cc..3fe0053c0 100644 --- a/cmd/gortex/repo_inventory_missing_test.go +++ b/cmd/gortex/repo_inventory_missing_test.go @@ -117,7 +117,14 @@ func newInventoryController(t *testing.T, repos []config.RepoEntry) *realControl reg := parser.NewRegistry() languages.RegisterAll(reg) mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, _ = mi.IndexAll() + mi.ExpectRepos(repos) + for _, entry := range repos { + mi.StartRepoLoad(entry.Path) + _, trackErr := mi.TrackRepoCtx(context.Background(), entry) + // Missing entries deliberately remain configured but unavailable; + // inventory/status assertions below must still report those entries. + mi.FinishRepoLoad(entry.Path, trackErr) + } return &realController{graph: g, multiIndexer: mi, configManager: cm, logger: zap.NewNop()} } diff --git a/docs/edit-loop-measurements.md b/docs/edit-loop-measurements.md new file mode 100644 index 000000000..8202a4ddd --- /dev/null +++ b/docs/edit-loop-measurements.md @@ -0,0 +1,660 @@ +# Edit-loop timing observations + +`mutation_status` adds `phases_ms`, `phase_states`, and `phase_timing` beside +the existing `publication_phases` timeline. The timeline keeps its existing +schema, origin and timestamp meanings. The new projection is read from the +same immutable snapshot, with no new graph/store queries or edit-path I/O. + +This is partial instrumentation. Presence of every key does **not** mean every +lap has been measured, or that the phase-sum acceptance criterion is met. + +The optional `phase_timing.patch_accounting` object is a separate partition of +one exact accepted winning physical attempt P. Its actual setup, inspection, +executor-control and finalization intervals pause around the same actual +admission/lane/deep operations. `patch_analysis` names the observed dataflow, +affected-plan and reference-fact block; resolver meaning stays unchanged. This +uses fixed ordered scalar state, never elapsed-minus-known work or interval +history. Early deep snapshots/logs remain immutable and executor-contained while +outer finalization is open. The actual-return completion publishes a fresh copy +only after exact winner/membership and agreement checks with those early values. + +`observed_partition_complete` requires valid integer bounds, all applicable +requested deep laps and a disjoint complete P partition. Missing deep evidence +fails it even if broad control totals numerically fill the wall. `sum_state` +remains `partial` for a valid local partition pending actual acceptance tests; +`missing_laps`, `unclassified_gap`, `invalid_order` and `not_observed` explain +refusals. The residual is diagnostic. Initial quiet/retry backoff, dispatcher +queueing and discarded attempts are not part of this smaller winning-P wall. +Its `background_coverage` remains `not_established`: it does not establish full +per-edit background, ACK, actual100-edit or5%/performance acceptance. + +`execution_disposition: native_mtime_duplicate` records the actual native +watcher receipt-match return before executor entry. Only that observed branch +can mark its unexecuted deep and executor-control laps `not_applicable`, with +null durations. Contradictory execution evidence is invalid; an unknown branch +still requires the requested deep evidence. A complete partition of this small +no-op P does not measure the earlier graph publisher. Independent read-assisted +publication R remains non-additive, and its missing deep coverage stays missing. + +An accepted `read_assisted_publication` can carry its own `phase_accounting`. +Its window keeps the actual coordinator execution start and return, after +repository lane/batch admission. The same fixed scalar collector measures the +real gate, parse, apply, search, resolve, mtimes, derived and optional analysis +operations. `repair_control` records the actual authority/setup, interstitial +and return work, pausing around those named operations. No pre-R lane, debounce, +P admission or finalization is borrowed. The immutable accounting copy retains +its coordinator generation and is linked only after actual owner, applied bytes +and existing ledger acceptance checks. Missing or contradictory laps refuse +completeness. These R durations must never be added to overlapping A, B or P. +The collector keeps waiting when complete no-op P precedes R retention or B +publication. Only a validated immutable phase failure (or the existing finite +collection bound) ends that wait as incomplete; absent late evidence is not a +proof that the missing work will never be observed. + +## Duration evidence + +`phases_ms` always lists these keys for a found mutation receipt: + +- Acknowledgement: `dispatcher_queue`, `middleware`, `parse`, `disk_write`, + `publish`, and the explicit additions `reindex_wait`, `edit_control`, + and `caller_result`. +- Patch: `debounce`, `lane_wait`, `gate_wait`, `patch_parse`, `apply`, + `search_lap`, `resolve`, `mtimes`, and `derived`, plus the additive actual + `cohort_admission` and `batch_admission` envelopes. + +`edit_control` records actual leaf preparation, validation, formatting, receipt +and response-control intervals online. It opens at leaf entry, pauses around +each existing parse/write/admission/wait operation, resumes at that operation's +actual end, and closes at leaf return. Repeated chunks use fixed scalar totals +and ordering checks. A parse guard that is not applicable keeps that parse +verdict; its actual dispatch interval is control work. No remainder becomes +measured work. Pre-leaf refusal has no invented leaf interval. + +`caller_result` is the actual handler-finish through caller-facing reply-finish +interval: bound-handler delivery/finalization before transport serialization. +It starts where `reply_prepare` ends. A detached reply before physical handler +finish has unavailable/outside-window control or caller-result evidence; no +post-reply work is clipped into ACK. The original RPC identity, per-file receipt +ownership and requested phase meanings remain intact. Subtotals stay labelled +partial pending complete-window controls and actual100-edit validation. Observed +production totals and containment use integer nanoseconds before conversion to +public milliseconds; fractional rounding never authorizes tolerance or clipping. +A missing actual reading retains conservative unavailable/partial evidence. + +A duration is a monotonic millisecond number, or `null`. `phase_states` gives +the evidence for each key: + +| State | Meaning | +| --- | --- | +| `measured` | Both boundaries were observed in order; zero is valid. | +| `pending` | The start was observed, but the end has not been observed and the receipt is not terminal. | +| `not_observed` | Boundaries are absent, including on paths without a publication timeline. | +| `not_applicable` | A graph-refresh/background record has no source-mutation acknowledgement. | +| `invalid_order` | Observed offsets contradict the interval; the duration stays null. | + +The acknowledgement projection measures `disk_write` (`disk_write_started` to +`receipt_committed`) and `publish` (`receipt_committed` to `ticket_enqueued`) +for MCP-edit timelines. The latter includes post-commit capture and ticket +admission. It does not include waiting for the admitted publication to finish. +The projection uses nanosecond offsets before converting to milliseconds, +rather than subtracting rounded timeline values or parsing wall-clock strings. + +Canonical single-file commits retain their own disk-start boundary and timing +record through the existing bounded commit ledger. Admission binds the original +watcher sequence before debounce can wake; only the existing ticket-result +consumer records completion. A newer coalesced application can serve that +original ticket, but does not become its ticket identity, a checkout generation, +or invented per-receipt patch laps. Native events have no MCP acknowledgement. + +Immediate-success edits keep the existing empty public freshness-receipt field; +their commit receipt can still render its retained timing. Different files or +versions within one request own distinct disk intervals even though they share +the middleware origin. Their receipt walls overlap and must not be added as RPC +time. Timing pointers are cleared on ledger expiry/cap eviction even when source +evidence keeps a commit object alive; a late callback cannot resurrect them. +Rebinding a commit to a replacement freshness receipt does not borrow that +replacement's timing or retarget the original record. Atomic batch/journal +timing and response/transport timing remain separate pending work. + +Canonical point execution additionally collects `cycle_started`, +`patch_execution_started`, and `patch_execution_finished` on a local attempt. +Only the exact successful winner at the existing guarded completion can absorb +those stamps, outside the watcher lock and before result delivery. Older +coalesced receipts receive completion without the winner's execution spans. +Retry, superseded and stopped attempts discard their collectors; native paths +without an instrumented outstanding waiter allocate no collector. Outstanding waiter ownership is +released by the existing completion/stop paths rather than a tracing registry. + +When a native successor serves instrumented predecessors, status may additionally +expose `coalesced_publication`: a separate bounded recorder entry for that actual +physical winner, with explicit requested and applied generations. The original +receipt's accepted-P fields remain unavailable. Each actually served retained +predecessor may refer to the same winner; these are references to shared work, +not additive per-edit spans. When the actual native arm reading is available, +the independent winner's origin is that observed arm start and `debounce` uses +that winner's arm-to-callback pair. Otherwise its origin remains callback entry +and quiet wait remains unavailable. Original receipt timestamps never move. P +still measures callback entry through actual return, excluding debounce, and +`background_coverage` remains `not_established`. Eviction, absent ownership, +failure or supersession leaves this independent evidence unavailable; weak links +cannot retain evicted entries or rebind a previously served receipt. + +For the specifically observed case of one requested timer canceled before its +callback and one native successor, `requested_background` may expose an +independent B window: accepted scheduling-call entry through the actual serving +attempt's return. MultiWatcher routing, admission checks, mutex wait and setup +precede the first timer arm in the genuinely measured +`scheduling_admission_setup` lap. Request parsing and disk work before scheduler +entry are outside B; A and B can overlap and must not be added together. + +The fixed partition measures initial timer-arm work, its canceled quiet wait, +the successful timer-stop call, actual coalescing setup, successor timer-arm work, +successor quiet wait, and the already validated physical P partition. These use +actual ordered endpoints, not wall time minus known laps. The B wall is measured +independently from its own start/end. Original accepted-P fields remain null. +Earlier snapshots remain unchanged; only a fresh late observation can contain B. +The winner's whole `debounce` pair is not added again beside B's measured +successor arm/quiet split. Status renders related P and B from one fenced +immutable observation, so complete B cannot accompany an older missing P. + +Prior callback entry, Stop=false, repeated resets, retries, multiple observed +predecessors, missing/reversed boundaries, incomplete P or lost membership leave +B unavailable. A newer event before guarded acceptance rejects the attempt; +later work after acceptance cannot erase the historical serving identity during +final release. No additional active-owner registry retains completed waiters. This case-specific +partition keeps `sum_state:partial` and does not establish the100-edit or broader +background/performance acceptance gates. + +The execution pair surrounds the common point executor, including its actual +standalone or MultiWatcher parse/apply/resolve/derived implementation. It excludes +before-image inspection and post-executor event work. `cycle_started` names the +accepted runnable attempt, which may follow retries. These are outer execution +events, not a pure debounce/lane/gate measurement, a full patch wall, or a split +of nested deep phases. No deep checkout-builder phase is relabelled onto the +contextless canonical pipeline, and internal storm/atomic batches remain outside +this observation. These executor events alone do not establish a full patch wall; +no phase-sum or CPU attribution acceptance follows from these events. + +For an accepted initial canonical point attempt, `debounce` measures the actual +initial timer arm through callback entry. It excludes the callback's claim lock, +cohort admission and repository lane. Only the exact claimed generation's +successful winner publishes the immutable pair; older coalesced receipts do not +borrow it. A retry's timer is backoff, so a winning retry leaves `debounce` +null/`not_observed` rather than borrowing its failed initial attempt's interval. +The existing physical-attempt log carries the same optional quiet-wait evidence +under its attempt ticket, before final receipt validation. + +For an accepted canonical point attempt, `lane_wait` uses the actual repository +lane channel wait and grant, after batch admission and its hook. The pair is +separate from executor-contained laps and is published only for that exact +winner, with enqueue/cycle/executor containment validated. Canceled waits have +no grant; rejected or coalesced attempts do not borrow another attempt's pair. +This does not measure the cohort semaphore, batch lock, or full patch lifetime. + +Two additive admission envelopes use actual calls: `cohort_admission` surrounds +the watcher's bounded cohort-slot admission, and `batch_admission` surrounds the +repository batch RLock/admission hook when either exists. They include their own +call/lock/hook work and are not pure repository lane waits. Complete pairs require +exact winner ownership, ordered enqueue/cycle/executor/terminal boundaries, and +cohort-before-batch-before-lane ordering. An absent batch lock/hook remains +unobserved, rather than a fabricated measured zero. These pairs also extend the +same immutable attempt log; setup, inspection and finalization gaps remain +unclassified, so accounting is still partial. Initial debounce and retry backoff +precede P and must never be summed against P's smaller wall. + +The same optional attempt-local collector now follows the actual standalone or +MultiWatcher point executor to its precise derived-tail call. The raw pair +`patch_derived_started` to `patch_derived_finished` becomes `phases_ms.derived` +only with observed admission and enclosing executor boundaries in monotonic +order. Resolver, reference-fact, semantic and metadata work before that call +is excluded. A branch that bypasses the derived call has no pair and remains +null/`not_observed`; an actually reached zero-duration call is measured zero. +The existing exact-winner absorption remains the only publication point, so +failed/retried/stopped/coalesced sibling attempts cannot publish these spans. +There is no ambient Indexer tracing state or additional history. + +## Canonical cumulative patch laps + +A bound canonical attempt also collects a fixed set of scalar totals and +observation counts, without retaining per-file spans or a tracing registry. +Repeated chunks and same/cross-repository resolver calls contribute their actual +elapsed intervals. First-to-last timestamp envelopes are not used as a sum. + +| Lap | Actual observed boundary | +| --- | --- | +| `gate_wait` | The call to `reach.BeginTopologyMutation`, until acquisition returns. | +| `patch_parse` | Each chunk's preparation: source read, parse admission, extraction and fingerprint preparation; excludes prior-graph loading. | +| `apply` | Deletion and staged graph application, explicitly split before/after the search refresh block. | +| `search_lap` | The explicit search/FTS refresh block; structural FTS deletion stays with graph replacement in `apply`. | +| `resolve` | Actual deferred same-repository/name and cross-repository resolver calls; intervening dataflow, reference-fact and semantic work is excluded. | +| `mtimes` | Source-version receipt validation/mtime persistence, deletion-mtime pruning and MultiIndexer metadata publication. | +| `derived` | The precise derived-tail call above. | + +The collector rejects overlap, inverted clocks, incomplete begin/end counts, +and intervals outside the original admission/executor/terminal window. Its +scalar derived interval must match the existing raw derived pair, so they cannot +be counted twice or conceal an overlapping subtotal. The legacy +`publication_phases` JSON fields stay unchanged; its first-wins timeline alone +cannot reconstruct a repeated lap. `phases_ms` is projected from the same +immutable in-process receipt snapshot with its accepted scalar evidence. + +A reached zero-duration operation is measured zero. A bypassed or unobserved +operation remains null. The legacy combined parse/apply fallback makes the full +`patch_parse`, `apply` and `search_lap` totals unavailable for that attempt, +including any earlier measured chunks; it is not assigned a fabricated split. +Other independently proven laps remain available. Uninstrumented transitions, +contract work, semantic/dataflow work and notification costs stay in the +publication residual. This does not establish complete 5% sum coverage. + +For observed canonical attempts, the existing `graph patch complete` line adds +`attempt__ms`, `attempt__state`, `attempt__count` and the original +`attempt_ticket`. Its `patch_lap_scope` is +`physical_attempt_before_receipt_validation`: the log happens before final +watcher winner validation and does not claim that a superseded attempt's timing +will be published into any receipt. The encoder formats an immutable copy and +never samples clocks or a live collector. Native/unobserved patches keep their +original log fields, with no new timing snapshot, phase clock or encoder. +No extra patch line is emitted. Total instrumentation overhead remains +unmeasured and needs the prescribed no-regression runs. + +## Separate lifetimes + +`phase_timing.clock` is `monotonic` when a publication or acknowledgement observation exists, otherwise +`unavailable`; `coverage` is currently `partial`. + +- `ack_wall_ms` is null until the outer bound tool call returns a caller-facing + result/error. Its denominator is entry into `boundToolHandler` through that + return, **before** daemon serialization/socket delivery. The separate + `acknowledgement` object names this scope and shared RPC identity. It is not + the harness's send-to-receive wall. Legacy publication origin remains MCP + middleware entry and does not move to this earlier boundary. +- `patch_wall_ms` is the accepted canonical timer callback entry through actual + mutation return, after the existing completion, Done delivery and cohort + release. It excludes prior debounce/retry backoff and timing-publication + overhead. It may finish after the receipt terminal and caller ACK; those + walls are independent and must never be summed. An early snapshot can have + a terminal receipt but a null patch wall until physical return. A later + fresh immutable observation appears only for the exact still-retained + accepted attempt; absent recorder ownership, eviction, failure or a foreign + attempt leaves it unavailable. Previously returned snapshots do not change. + The early patch log keeps its original scope; one matching completion line + records the final wall and attempt retry ordinal. This endpoint evidence + remains partial accounting, without a complete phase partition or5% claim. + One coalesced patch may serve multiple receipts. +- `publication_wall_ms`, when terminal boundaries exist, measures + `received` to `ticket_completed` or `ticket_failed`. It is a receipt interval, + not the acknowledgement time or the physical patch wall. +- Without a shared RPC acknowledgement, `publication_unattributed_ms` is that publication wall minus the measured, + disjoint acknowledgement and observed patch-lap intervals. Patch intervals + must be wholly inside the admitted executor and publication window. Invalid ordering leaves + it null. The residual contains + uninstrumented work and waits; it does not classify them. In particular it + does not measure the acknowledgement's default reindex wait. + When shared RPC laps replace per-file acknowledgement values, this residual + is unavailable: subtracting them from a different-origin per-file interval + would be invalid. + +Acknowledgement and asynchronous patch execution can overlap. Their walls +must not be added. Likewise `cycle_started` to `admitted` is not labelled pure +lane wait, and existing nested structural-apply spans are not summed as +disjoint patch laps. Complete sum accounting needs explicit boundaries at +those operations; it cannot be obtained by renaming the timestamp gaps. + +Pending/unobserved durations must remain missing in benchmark inputs. They +must not become zero-valued successful observations. This initial projection +does not claim the 5% phase-sum, 100-edit, runtime-overhead, or real read-back +acceptance gates in #1005. + +## Store-level WAL observations + +The store's `WALReclaimStats` adds cancellation/refusal counts for the first +consumed writer-parked, announced-intent or original-operation cancellation +cause of a final non-reset reclaim attempt. A positively observed parked writer +has precedence when both demand signals are positive; this is observation +precedence, not a claim that only one signal caused contention. Multiple polls, +writer-credit stages and helper cleanup do not become multiple attempts. +Successful final resets, normal helper cleanup, cycle/bulk withdrawal and a +child budget expiration are not counted as other caller/shutdown cancellation. +The existing common writer-waiting error remains available through `errors.Is`. + +`ObservedPeakWALBytes` is a sampled high-water mark since the shared store core +opened. `WALPeakObserved` distinguishes an actual size observation from missing +evidence; in-memory/unsupported stores do not fabricate a zero WAL observation. +It reuses existing successful size reads and adds no file/stat/SQL work to edits. +Generation handles share the same peak; independent stores do not. Truncate, +shrink and status reads never reset it. This is a lower bound on instantaneous +physical peak and cannot be subtracted into a warm-run peak after startup. +These store observations do not by themselves establish per-run peak or +cancellation/CPU attribution acceptance. + +`graph_stats` and `gortex://stats` share a `wal_reclaim_observation` block. Its +scope is `physical_store_since_open`, independent of the selected graph/view or +repository. The optional store capability is queried once per payload through +its existing `WALReclaimStats` snapshot: a short counter mutex and atomic loads, +with no file stat, WAL-index read, SQL query or writer wait. Unsupported/masked +stores report `snapshot_available=false`, a reason and null counters/peak; +there is no unwrap or SQL fallback. A supported zero counter is numeric zero. + +The fields `cancelled_writer_parked`, `cancelled_write_intent` and +`cancelled_other` preserve the store's final-attempt observation semantics. +`observed_peak_wal_bytes` is null while `wal_peak_observed=false`. An actually +observed empty WAL is `0` with `wal_peak_observed=true`; it is different from +missing evidence. `peak_scope=sampled_since_store_open_lower_bound` explicitly +excludes an exact instantaneous or warm-run peak. In-memory stores have no WAL +size observation even when their counter snapshot is supported. + +`observation_epoch` reuses the Server's immutable observation binding and +`epoch_scope=server_binding`. It is **not** a durable store-open identity or a +per-run reset token. Production binds the Server's physical store at construction; +generation handles share the store core and its observations. A literal Server +without that binding has an empty epoch. Do not join samples across a changed +binding/reopen, or subtract two high-water marks into a run peak. Stats reads, +WAL resets and generation changes never clear these counters. + +Daemon status JSON also exposes these new causes and nullable peak under +`storage.wal_reclaim`, extending its already-collected snapshot. This adds no +status collection call or I/O; the new MCP block does not call the daemon's +file-size/WAL-index collection path. Other daemon storage/status fields keep +their existing semantics. + +## Private acknowledgement ownership (transport pending) + +Measured source-tool calls now own one private, call-scoped acknowledgement +record at the `boundToolHandler` boundary. It reuses the existing five-tool +classification. Nested calls on the same Server borrow its identity, while a +different Server owns a different record. Reads allocate no acknowledgement +record or context value and take no acknowledgement clock sample. + +The origin is `mcp_bound_call`, before the deadline firewall; it is neither +dispatcher entry nor the legacy `toolReceivedAt` middleware origin. The latter +and `publication_phases` remain unchanged. Caller response-ready freezes when +the outer bound call returns. Actual handler completion is observed separately, +including a detached physical commit that finishes later. Those walls overlap +and must never be added together or presented as client receipt/socket delivery. +Missing endpoints stay pending, missing origins stay unavailable and invalid +ordering stays invalid; none becomes measured zero. Snapshots own their scalar +values and never evaluate a live clock during formatting. + +Physical file receipts written by one RPC now share that private observation +through the existing bounded mutation ledger. Both expiry and capacity eviction +clear the reference, even when pending-source evidence keeps the physical receipt +alive. A late handler can finish its call-scoped observation but cannot reinstall +it on an evicted record. Raw/unobserved writes have no MCP acknowledgement. + +The ownership does not attach pointers to `batchTransactions`, whose existing +map has no bounded eviction. Actual parse/disk/enqueue +laps and additive-sum validation are partial work. It does not change the validated actual-completion counters or CPU/WAL +sampling. Read/native-path and mutation overhead still require measurement. + +The daemon's existing allocated slot lease now retains immutable original +admission-entry and actual channel-grant endpoints. A source-tool observer +copies them once; no context, registry, classifier or per-tool transport parse +is added. Replay/cache followers and unbounded/no-lease paths have no grant +observation. Releasing capacity or attaching another grant cannot replace the +original pair; a refused first attempt cannot borrow a later attempt's grant. +Nil-lease primitives sample no clock. Successfully admitted leased requests, +including reads, add two monotonic clock samples and existing-lease mutex work; +that overhead is unmeasured, with a primitive benchmark supplied for later +measurement rather than a performance claim. Current HTTP paths without that +admission lease remain unavailable. Private queue and grant-to-observer +preamble snapshots precede the bound-call origin: scoped wire exposure is +described below, and they must never be added to its existing subtotal +or silently widen its wall. No dispatcher policy or capacity changes here. + +The private fixed acknowledgement accumulator now observes the actual pre-write +parse-gate call in `edit_file`/`write_file`, the `AtomicWriteFile` call (ending +before committed-hash/receipt bookkeeping), and canonical/checkout refresh +enqueue calls. Repeated chunks occupy fixed slots rather than a span history. +An unsupported parse gate is not applicable; an instrumented raw commit that +did not call a gate does not acquire a fabricated parse duration. Other tools' +actual parser and durability work is not inferred from these slots. + +Each slot retains begin/completion counts, actual elapsed execution and interval +validity. Snapshot `ObservedMS` is completed execution; `AckMS` is an alternative +value present only when every applicable chunk is valid and contained in the +frozen reply window. A late interval keeps its full execution observation and +no acknowledgement duration. Overlap, missing ends and invalid containment +produce explicit states, not clipped or residual-derived timings. The +`LapsDisjoint` flag alone proves neither coverage nor the 5% sum requirement; +middleware, waits, reply preparation and unsupported paths remain unclassified. +No additional patch log line is emitted by acknowledgement projection. + +Atomic batches now reserve one scalar `batch-ack-*` reference only in the +original execution arm. The timing payload occupies a distinct batch-only kind +inside the same bounded mutation ledger and shares its existing expiry/cap; +there is no second registry or timer. Transaction state retains no timing +pointer or snapshot, and the durable manifest/version is unchanged. Duplicate, +status and recovery replies cannot replace the original RPC wall. Eviction or +restart makes the private lookup explicitly unavailable. + +The batch-only ledger kind cannot enter physical receipt/replay/freshness/path +indexes, source-evidence registration, physical status/listings, commit notes, +publication records or disk/graph outcome setters. It supplies no invented +disk or freshness verdict. The private lookup checks the scalar reference and +fixed-size transaction identity before borrowing an owned timing snapshot. +Batch text splices have no parser lap. Forward file writes/removals and the +deduplicated target-directory sync now have concrete timing boundaries, as +described below. Journal, backup, rollback, cleanup and planning work remain +unclassified; complete critical-path coverage and the measured 5% sum gate +remain pending. + +## Receipt acknowledgement wire and critical-path denominator + +`mutation_status` adds `acknowledgement`, with a bounded shared `rpc_id`, scope, +caller response-ready and actual-handler walls/states, per-lap begin/completion +counts, and owned duration values. Its nine fixed acknowledgement keys include +explicitly unavailable dispatcher queue/preamble when no bounded lease was +observed, and an unavailable transport boundary. +The existing fifteen-key `phases_ms`/`phase_states` +shape remains: its six acknowledgement keys use the shared RPC observation +when available; its nine patch keys retain their independent per-receipt +evidence. `publication_phases` stays byte-for-byte the existing snapshot. Legacy +records without an acknowledgement retain independently valid publication laps +and explicitly unavailable RPC ownership, never an origin borrowed from polling. + +`reindex_wait` now surrounds the existing passive receipt selects in canonical +and batch paths. It excludes enqueue/capture, outcome formatting and synchronous +inline reindex work. The timers, cancellation behavior and dispatch-slot release +are unchanged. Immediate completed tickets can have a real reached near-zero +wait; an async checkout admission that never waits has no invented duration. + +Batch edit/status JSON or TOON responses carry a response-only +`acknowledgement_receipt` reference; compact text carries the same reference. +Query it through `mutation_status`. This explicit `batch_acknowledgement` variant +has no disk/graph verdict, path or physical mutation listing entry. Durable +transaction status remains authoritative for those facts. Eviction, restart or +wrong checkout/incarnation returns unavailable timing rather than a different +call's clock. Neither this envelope nor a timing snapshot enters the journal. + +The current additive set is only valid, complete and disjoint measured chunks +of middleware, parse-gate, disk-call, ticket-admission, passive-wait and reply +cleanup work. Middleware measures the existing bound-call observation origin +(after source-tool classification) to the first wrapped leaf invocation. It +includes the firewall handoff and real view/repository/mutation preparation; +those nested costs are not added again. Reply preparation measures the leaf's +return (including its own defers and any in-leaf encoding) to the existing actual +handler-completion endpoint, after middleware decorators, view/repository release +and activity cleanup. It excludes final protocol serialization/socket delivery +and the later caller firewall return. A pre-leaf refusal has measured middleware +and not-applicable reply preparation; an unwrapped raw invocation has neither. +Nested facade/raw calls cannot replace the first leaf boundaries. Completed +operation laps must lie inside the leaf, disjoint from both outer intervals; +misplaced laps invalidate the subtotal instead of being double-counted. + +A phase crossing +the frozen response boundary has only its complete execution observation and +no acknowledgement duration. `observed_ack_subtotal_ms` sums only eligible +chunks. `ack_unattributed_ms` is a labelled unclassified gap to the bound-call +wall, never a phase; invalid counts, overlaps or totals leave both null instead +of clipping. `sum_state=partial` cannot satisfy the 5% gate, even if a particular +gap happens to be small. Several file receipts with one `rpc_id` must be +deduplicated; adding their RPC walls would multiply the same call. + +For the eventual 5% check, acknowledgement critical-path stages must cover the +same caller-facing wall once each; background patch stages cannot be added to +that wall. The current bound-call wall is a measured subset of the harness's +monotonic edit-send→ack-receive wall. The missing admission/serialization/client +transport boundaries and other handler work keep that full comparison open. +The physical patch subtotal has its own accepted-attempt patch wall; the receipt +publication wall is a third, overlapping lifetime. No combined acknowledgement +plus asynchronous-patch sum is asserted against any of these denominators. + +### Concrete batch/move boundaries and admission envelope + +Atomic batch forward source-file writes, removals and the existing deduplicated +target-directory sync are separate `disk_write` chunks. Journal/backup/rollback +and later cleanup durability remain unclassified, not nested into the same +subtotal. A failed forward call still has observed work; it does not imply a +successful commit. Canonical batch set admission, point fallback and checkout +batch admission now measure their actual scheduler call only. The original +Background/WithoutCancel scheduler lifetimes, synchronous fallback work and +shared passive wait policy are unchanged; fallback reindex execution is not +called queue admission. A later status/recovery RPC cannot modify the bounded +original batch ACK. + +Move source durability surrounds each existing `AtomicWriteFile`, excluding +parent-directory preparation. Explicit Go parser calls cover source, target, +package-clause inference and caller rewriting. `parse_scope` identifies these +as `explicit_go_parser_calls_formatting_unattributed`: `go/format` internally +combines parsing and formatting and remains unclassified, not falsely reported +as pure parse. File writes/edits name their actual old/new parse gate. Symbol +and atomic batch text splices explicitly report not-applicable parse with null +time; unsupported non-Go moves retain their tool refusal and no parser time. +The requested tool/language matrix therefore contains unsupported combinations; +these source controls do not claim 120 successful combinations or a benchmark. + +Queue and grant-to-observer preamble values are exposed with +`dispatcher_scope=queue_and_preamble_precede_bound_call_subtotal` when the +original lease proves them. They are **not** added to `observed_ack_subtotal_ms` +or subtracted from its bound-call wall. A separately named +`dispatcher_to_caller_response_ready_ms` measures original bounded admission +entry to outer caller return, still before transport serialization/delivery; +its state distinguishes pending, invalid and unavailable. The existing +`caller_response_ready_ms` and `phase_timing.acknowledgement_wall_ms` retain +their original after-classification bound-call origin. These are alternative +envelopes, not additive walls. No partial subtotal becomes a complete 5% proof. + +## Logical source-operation witness (transport pending) + +The MCP Server privately counts actual completion of the five measured source +operations: `write_file`, `edit_file`, `edit_symbol`, `batch_edit`, and +`move_symbol`, including their `edit`/`refactor` facade spellings and selector +inference. These counters span that Server's sessions and views. The daemon's +SharedServer serves its clients through the same Server; the counters are not +per selected repository, per client or exclusive process-CPU attribution. + +A fixed request-local witness runs inside the actual handler execution, including +when the deadline firewall has already answered the caller. A nested handler on +the same Server borrows its owner; a distinct Server has distinct counters. +Normal reads do not allocate a witness. Hooks only observe existing successful +forward source-write/removal seams and already-read before images: no additional +source reads, graph/store queries or process samples are introduced. A raw caller +without a witness also skips the added byte comparison. + +| Private counter | Actual evidence | +| --- | --- | +| `completed` | At least one proven source change, the whole source operation committed, and the actual handler returned a nonnil, non-tool-error result without a Go error or typed committed-journal warning. This is source completion, not graph freshness or acknowledgement delivery. | +| `committedWithError` | The complete source effect landed, but the actual handler result was nil/error/`IsError`, or a typed `committed`/`committed` batch receipt carried a journal warning. Committed bytes are not labelled unapplied. | +| `failedAfterSourceEffect` | Forward source effects were observed, but the operation did not reach complete commit. This includes rollback and retained/conflicted partial effects. | +| `rolledBack` | The subset above with typed batch `DiskStatus=rolled_back`. Rollback writes are not counted as another edit. | +| `unknownSourceComparison` | An otherwise clean complete write lacked readable prior bytes, so a logical content change cannot be established. It is excluded from clean completed edits instead of guessing a no-op/change. | + +The first four outcome classes above are exclusive except for the named rollback +subset. A successful filesystem call is positive effect evidence; absence of a +witness does not prove that an opaque I/O error left every byte unchanged. Existing +typed disk/transaction verdicts remain authoritative. Failed-source counts cover +observed forward effects, not a new filesystem failure-detection mechanism. + +The original batch handler performs physical writes, directory durability and +journal publication synchronously. A caller can stop waiting while that handler +continues its deliberately non-cancellable commit. Only that original new +transaction owns source effects; duplicate/status/recovery replies own no +new source operation. The typed Status/DiskStatus distinguish aborted, rolled +back or conflicted outcomes from committed source plus a terminal-journal warning; +`err == nil` and a non-tool-error response alone are insufficient. Multi-file +batch and move writes contribute one logical operation, not one per file. + +Idempotency replay, status-only calls, dry runs, unchanged rewrites and refusals +before source effects do not increment completed edits. Native editor stimuli +remain harness-owned: filesystem events/coalesced watcher patches cannot identify +a logical native edit and are never added to these MCP counters. + +Process CPU sampling is separate from these operation hooks. Protocol fixtures +are not installed-client or performance evidence. + +## Process CPU/count stats window + +`graph_stats` and `gortex://stats` both add `process_edit_window` beside their +view-derived graph fields. Its `count_scope` is `server_all_sessions_views` and +its `cpu_scope` is `whole_process_including_background`. Process CPU includes +reads, indexing, other sessions, unfinished work and any other Server in the +process. It is not exclusive CPU attributed to the selected graph or an edit. + +An immutable `epoch` identifies this Server's observation window. At the end of +construction, before handlers become callable, it captures process CPU between +two zero-count snapshots. A literal/uninitialized Server reports no baseline; +a failed baseline is retained rather than silently replaced on a later read. +No stats read resets counters or the epoch. + +`baseline` and `sample` expose raw cumulative `cpu_seconds`, monotonic +`sample_started_seconds`/`sample_finished_seconds` within that epoch, and full +`counts_before`/`counts_after` brackets. A normal stats read takes exactly one +process CPU sample. Unavailable CPU is null with `cpu_available=false` and a +reason; a known zero CPU sample is a valid zero. Unix uses current-process +user+system Getrusage; Windows uses GetProcessTimes kernel+user durations; +unsupported platforms report unavailable. + +`cpu_seconds_per_edit` divides the process CPU delta from that same baseline +by clean completed source operations from the matching zero-count baseline. +It is numeric only for available, ordered CPU/clocks, a stable complete count +bracket, at least one clean completed edit, and no committed-with-error, +partial-effect or unknown-comparison outcome in the window. Otherwise it is +null with `average_state=unavailable` and an explicit `average_reason`. +A measured value is labelled `measured_process_window`, not exclusive edit CPU. +The rollback subset is never added again as another operation. + +Raw samples remain available even when that since-baseline average is null. +A harness can compare later warm-run endpoints only with the same nonempty +epoch, valid stable brackets and an eligible outcome delta. Native logical +operations still require the harness's own completed-stimulus denominator. +Sampled since-store-open WAL peaks cannot be subtracted into a warm-window peak. +No run-reset API, per-edit/process CPU syscall or tracing registry is introduced. +Instrumentation cost, real client read-back, complete phase sums and the full +workload matrix remain unmeasured acceptance work. + +## Load-harness per-edit times + +The edit-and-probe load cell (`bench/load`) reports per-edit client times from +edit send: ACK, the last correct immediate readback, and the slowest first +correct answer in each visibility group (file graph, relations, search). Its +`t_derived_ms` instead reads this record's own terminal `publication_phases` +timeline on the daemon's monotonic clock, from `received` to +`patch_derived_finished`. That value is not additive with client times, and a +coalesced receipt (derived in the serving publication's timeline) leaves it +unavailable. Its `phases_ms` copy keeps each producer null as unavailable with +the producer's phase state, and a build without `phases_ms` (an older baseline) +reports every lap unavailable rather than zero. Quantiles use nearest rank with +seeded percentile-bootstrap intervals over the complete offered denominator: a +record that ended before its value was observed ranks above every observed +value, so a tail quantile that falls among such records is unavailable rather +than understated. + +Probes can also run as compact text or through `gortex call`. Compact rows +carry no IDs, edge rows or freshness fields, so they prove only a call +addition's presence; CLI stdout drops auxiliary notes and re-encodes numbers +through float64, so only exactly representable JSON is credited and expected +absences are refused. GCX is reported unavailable rather than decoded as TOON. +Each report accounts every output mode, and a mode that cannot carry a +freshness claim reports stale claimed-fresh answers as unavailable, not 0. + +Native and lifecycle cells change files around MCP: a shell write after a tool +edit, a same-size editor save, and Go file create/delete pairs. Their records +have no acknowledgement or receipt, so ACK, phases and derived time are +unavailable; each keeps its own write window and file identity instead. The +restart cell kills the daemon while the edit's ACK is outstanding and its +committed bytes still have a pending receipt (the ACK itself waits for that +receipt), and reports post-restart first-correct times from send and from +ready; a receipt that was already terminal before the kill qualifies the record +rather than passing. +TypeScript and Python relocation is measured as the product's exact refusal +with unchanged bytes. diff --git a/docs/mcp.md b/docs/mcp.md index a4397a928..a56f42aac 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -267,8 +267,7 @@ Tools start working **in place** when the repository registers: no reconnect is Every tool call and resource read is bounded, on every transport. A request that overruns its budget is **abandoned**: the client gets a terminal, structured -error and the transport's slot is released immediately, so the session keeps -serving later requests. The handler itself keeps running — it may be inside a +error so the session keeps serving later requests. The handler itself can keep running — it may be inside a store call that cannot be interrupted — so treat any side effect of an abandoned call as *unknown* and re-read before assuming it did or did not land. @@ -276,17 +275,49 @@ call as *unknown* and re-read before assuming it did or did not land. |---|---|---| | Tool call / resource read / prompt fetch (all transports) | 60 s | `GORTEX_MCP_TOOL_TIMEOUT` (Go duration; `off` disables) | | Daemon socket, per JSON-RPC request | 60 s, terminal `-32001` | — | -| Daemon socket, concurrent dispatches | 8 | `GORTEX_MCP_MAX_CONCURRENT_DISPATCHES` (max 64) | -| Daemon socket, dispatch queue wait | 5 s, then terminal `-32002` with `queue_wait_ms` | `GORTEX_MCP_DISPATCH_QUEUE_WAIT` (Go duration, max 30 s; `0`/`off` restores the instant refusal) | +| Daemon socket and HTTP `/mcp`, shared concurrent dispatches | 8 | `GORTEX_MCP_MAX_CONCURRENT_DISPATCHES` (max 64) | +| Daemon socket and HTTP `/mcp`, tool dispatch queue wait | Up to 5 s, clamped to the request deadline minus 100 ms; terminal `-32002` includes `queue_wait_ms` and `retry_after_ms` | `GORTEX_MCP_DISPATCH_QUEUE_WAIT` (Go duration, max 30 s; `0`/`off` restores the instant refusal) | | Control RPC (`daemon status` / `search_symbols` / `proxy`) | 30 s, terminal `timeout` | — | | Control RPC (`shutdown`) | unbounded on the daemon — the store flush precedes the ack. `gortex daemon stop` bounds its own wait at 30 s, then watches the process for up to 2 min and force-kills | — | | Control RPC (`track` / `untrack` / `reload` / `enrich_*`) | unbounded by design | — | -When every dispatch slot is busy, a request waits up to the queue budget for -one; at most 16× the dispatch cap may wait, and the wait counts against the -request timeout. A source edit gives its slot back while it waits for its graph -refresh to publish, so editing agents do not hold slots that other requests -need. +MCP tool calls to the daemon, over its socket or `/mcp`, share three admission +lanes. REST `/v1/tools/` calls run their handlers directly and are not +admitted through these lanes. Mutating calls have two reserved slots and can +also use free shared slots. Slow calls (`outline`, `dead_code` +and analysis) hold at most two slots; other calls use the fast lane. Free slots +serve queued mutating calls first, then fast, then slow. Sessions take turns +within each lane; a session holding four slots yields to another waiting +session. For small cap overrides, the reservation is `min(2, cap-1)` and the +slow limit is `min(2, cap-reservation)`, so cap 1 remains usable. + +Each lane permits at most 16× the total dispatch cap to wait. Queue time counts +against the request timeout, leaving 100 ms for a refusal to reach the client. +Busy replies are retryable and include `retry_after_ms`, estimated from the +lane's observed drain rate and clamped to 100–5,000 ms (1 s before observations). +An edit that waits for graph publication returns mutation capacity before +entering the slow lane, and stops queueing for it once publication lands: a +finished publication never waits for a slot. If slow admission fails after the +disk commit, the edit returns its pending receipt instead of a retryable busy +refusal. + +Cancellation returns logical lane capacity within 100 ms while the process +has room in its four-handler draining allowance. Abandoned handlers, including +inner tool handlers whose middleware has already returned, stay counted until +they actually return. When all four drains are occupied, further cancelled +handlers remain capacity-charged; physical dispatcher work stays bounded by +the configured cap plus four. Initialization (`initialize` or `server/discover`) +and notifications share total capacity but are exempt from tool reservations; +notifications wait without a queue budget. `notifications/cancelled` takes no +slot on either transport, so a full pool cannot hold back the cancellation that +frees it. `subscriptions/listen` stays unbounded and consumes no dispatch slot. + +Daemon status exposes `mcp_dispatch`: lane in-flight, queued, draining and +cancelled-but-held counts, refusals, reserved-slot use and queue-wait histograms +for admitted, refused and cancelled calls. Histogram buckets are disjoint +upper bounds in milliseconds and include zero-wait admissions. The metrics +surface reports the corresponding `views_mcp_dispatch_*` series without +session or request identifiers. The handler bound fires just inside any deadline the transport already imposed, so the client receives the tool-shaped diagnosis rather than an opaque transport diff --git a/internal/daemon/mcp_dispatch_bench_test.go b/internal/daemon/mcp_dispatch_bench_test.go new file mode 100644 index 000000000..a2cae4550 --- /dev/null +++ b/internal/daemon/mcp_dispatch_bench_test.go @@ -0,0 +1,35 @@ +package daemon + +import ( + "context" + "testing" +) + +// BenchmarkDispatchAdmission isolates admission/release bookkeeping, including +// the production metrics path. semaphore_only is a primitive lower bound, not +// the former full dispatcher or an end-to-end performance comparison. +func BenchmarkDispatchAdmission(b *testing.B) { + b.Run("semaphore_only", func(b *testing.B) { + slots := newMCPDispatchSlots(8) + b.ReportAllocs() + for b.Loop() { + slots <- struct{}{} + <-slots + } + }) + for _, class := range [...]MCPDispatchClass{MCPDispatchFast, MCPDispatchMutating, MCPDispatchSlow} { + b.Run(class.String(), func(b *testing.B) { + owner := &mcpDispatchCoordinator{} + pool := owner.pool(newMCPDispatchSlots(8)) + ctx := withMCPDispatchQueueWait(context.Background(), 0) + b.ReportAllocs() + for b.Loop() { + grant, err := pool.acquire(ctx, class, "one-session") + if err != nil { + b.Fatal(err) + } + grant.complete() + } + }) + } +} diff --git a/internal/daemon/mcp_dispatch_class.go b/internal/daemon/mcp_dispatch_class.go new file mode 100644 index 000000000..8d7e8d0d6 --- /dev/null +++ b/internal/daemon/mcp_dispatch_class.go @@ -0,0 +1,256 @@ +package daemon + +// MCPDispatchClass is the admission lane of an effective tool call. It does +// not replace ToolEffects or any authorization/mode policy. +type MCPDispatchClass uint8 + +const ( + MCPDispatchFast MCPDispatchClass = iota + MCPDispatchMutating + MCPDispatchSlow +) + +func (c MCPDispatchClass) String() string { + switch c { + case MCPDispatchMutating: + return "mutating" + case MCPDispatchSlow: + return "slow" + default: + return "fast" + } +} + +// toolDispatchClasses is the complete local tool catalogue. New registrations +// must make an explicit cost/admission decision here; catalogue tests cover +// live, deferred and facade implementations. Facade calls are lowered through +// their existing operation registry before this table is consulted. +// +// Incidental read caches/telemetry do not turn a read into a mutation. Legacy +// analyze is deliberately slow for every kind, including its conditional +// durable kinds; the effect-safe workspace_admin facade is mutating instead. +// Conditional legacy change_contract acknowledgements and co-change refreshes +// persist state, while their read facades fix those write switches off. +var toolDispatchClasses = map[string]MCPDispatchClass{ + + "apply_code_action": MCPDispatchMutating, + "batch_edit": MCPDispatchMutating, + "change_contract": MCPDispatchMutating, + "delete_scope": MCPDispatchMutating, + "edit": MCPDispatchMutating, + "edit_file": MCPDispatchMutating, + "edit_memory": MCPDispatchMutating, + "edit_symbol": MCPDispatchMutating, + "enrich_churn": MCPDispatchMutating, + "enrich_releases": MCPDispatchMutating, + "export_graph": MCPDispatchMutating, + "feedback": MCPDispatchMutating, + "find_co_changing_symbols": MCPDispatchMutating, + "fix_all_in_file": MCPDispatchMutating, + "forget_checkout": MCPDispatchMutating, + "generate_docs": MCPDispatchMutating, + "generate_skill": MCPDispatchMutating, + "generate_wiki": MCPDispatchMutating, + "index_repository": MCPDispatchMutating, + "inline_symbol": MCPDispatchMutating, + "move_symbol": MCPDispatchMutating, + "notebook_save": MCPDispatchMutating, + "notebook_used": MCPDispatchMutating, + "overlay_merge": MCPDispatchMutating, + "post_review": MCPDispatchMutating, + "publish_review": MCPDispatchMutating, + "reconcile_checkouts": MCPDispatchMutating, + "refactor": MCPDispatchMutating, + "reindex_repository": MCPDispatchMutating, + "remember": MCPDispatchMutating, + "rename_memory": MCPDispatchMutating, + "rename_symbol": MCPDispatchMutating, + "safe_delete_symbol": MCPDispatchMutating, + "save_note": MCPDispatchMutating, + "save_scope": MCPDispatchMutating, + "scaffold": MCPDispatchMutating, + "set_active_project": MCPDispatchMutating, + "set_primary_checkout": MCPDispatchMutating, + "store_memory": MCPDispatchMutating, + "suppress_finding": MCPDispatchMutating, + "surface_memories": MCPDispatchMutating, + "track_repository": MCPDispatchMutating, + "untrack_repository": MCPDispatchMutating, + "workspace_admin": MCPDispatchMutating, + "write_file": MCPDispatchMutating, + + "analyze": MCPDispatchSlow, + "audit_agent_config": MCPDispatchSlow, + "audit_health": MCPDispatchSlow, + "contracts": MCPDispatchSlow, + "dead_code": MCPDispatchSlow, + "find_clones": MCPDispatchSlow, + "get_architecture": MCPDispatchSlow, + "get_churn_rate": MCPDispatchSlow, + "get_communities": MCPDispatchSlow, + "get_coupling_metrics": MCPDispatchSlow, + "get_extraction_candidates": MCPDispatchSlow, + "get_knowledge_gaps": MCPDispatchSlow, + "get_processes": MCPDispatchSlow, + "get_recent_changes": MCPDispatchSlow, + "get_repo_outline": MCPDispatchSlow, + "get_surprising_connections": MCPDispatchSlow, + "get_untested_symbols": MCPDispatchSlow, + "lint_file": MCPDispatchSlow, + "list_inspections": MCPDispatchSlow, + "outline": MCPDispatchSlow, + "replay_episode": MCPDispatchSlow, + "run_inspections": MCPDispatchSlow, + "verify_citation": MCPDispatchSlow, + "why": MCPDispatchSlow, + + "agent_registry": MCPDispatchFast, + "api_impact": MCPDispatchFast, + "ask": MCPDispatchFast, + "batch_symbols": MCPDispatchFast, + "capabilities": MCPDispatchFast, + "change": MCPDispatchFast, + "check_guards": MCPDispatchFast, + "check_onboarding_performed": MCPDispatchFast, + "check_references": MCPDispatchFast, + "compare_branches": MCPDispatchFast, + "compare_with_overlay": MCPDispatchFast, + "conflicts_prs": MCPDispatchFast, + "context_closure": MCPDispatchFast, + "critique_review": MCPDispatchFast, + "ctx_grep": MCPDispatchFast, + "ctx_peek": MCPDispatchFast, + "ctx_slice": MCPDispatchFast, + "ctx_stats": MCPDispatchFast, + "detect_changes": MCPDispatchFast, + "diff_context": MCPDispatchFast, + "distill_session": MCPDispatchFast, + "explain_change_impact": MCPDispatchFast, + "explain_view": MCPDispatchFast, + "explore": MCPDispatchFast, + "export_context": MCPDispatchFast, + "find_declaration": MCPDispatchFast, + "find_files": MCPDispatchFast, + "find_implementations": MCPDispatchFast, + "find_import_path": MCPDispatchFast, + "find_overrides": MCPDispatchFast, + "find_usages": MCPDispatchFast, + "flow_between": MCPDispatchFast, + "get_active_project": MCPDispatchFast, + "get_artifact": MCPDispatchFast, + "get_call_chain": MCPDispatchFast, + "get_callers": MCPDispatchFast, + "get_cfg": MCPDispatchFast, + "get_class_hierarchy": MCPDispatchFast, + "get_cluster": MCPDispatchFast, + "get_code_actions": MCPDispatchFast, + "get_dependencies": MCPDispatchFast, + "get_dependents": MCPDispatchFast, + "get_diagnostics": MCPDispatchFast, + "get_edit_plan": MCPDispatchFast, + "get_editing_context": MCPDispatchFast, + "get_file_summary": MCPDispatchFast, + "get_pr_impact": MCPDispatchFast, + "get_symbol": MCPDispatchFast, + "get_symbol_history": MCPDispatchFast, + "get_symbol_source": MCPDispatchFast, + "get_test_targets": MCPDispatchFast, + "gortex_wakeup": MCPDispatchFast, + "graph_completion_search": MCPDispatchFast, + "graph_query": MCPDispatchFast, + "graph_stats": MCPDispatchFast, + "grep_results": MCPDispatchFast, + "head_results": MCPDispatchFast, + "index_health": MCPDispatchFast, + "list_checkouts": MCPDispatchFast, + "list_prs": MCPDispatchFast, + "list_repos": MCPDispatchFast, + "list_scopes": MCPDispatchFast, + "mutation_status": MCPDispatchFast, + "nav": MCPDispatchFast, + "notebook_find": MCPDispatchFast, + "notebook_list": MCPDispatchFast, + "notebook_show": MCPDispatchFast, + "overlay": MCPDispatchFast, + "overlay_branches": MCPDispatchFast, + "overlay_delete": MCPDispatchFast, + "overlay_drop": MCPDispatchFast, + "overlay_drop_branch": MCPDispatchFast, + "overlay_fork": MCPDispatchFast, + "overlay_keepalive": MCPDispatchFast, + "overlay_list": MCPDispatchFast, + "overlay_push": MCPDispatchFast, + "overlay_register": MCPDispatchFast, + "overlay_switch": MCPDispatchFast, + "plan_turn": MCPDispatchFast, + "pr": MCPDispatchFast, + "pr_review_context": MCPDispatchFast, + "pr_risk": MCPDispatchFast, + "prefetch_context": MCPDispatchFast, + "preview_edit": MCPDispatchFast, + "proxy_disable": MCPDispatchFast, + "proxy_enable": MCPDispatchFast, + "proxy_status": MCPDispatchFast, + "query_memories": MCPDispatchFast, + "query_notes": MCPDispatchFast, + "query_project": MCPDispatchFast, + "read": MCPDispatchFast, + "read_file": MCPDispatchFast, + "recall": MCPDispatchFast, + "relations": MCPDispatchFast, + "response": MCPDispatchFast, + "review": MCPDispatchFast, + "review_pack": MCPDispatchFast, + "search": MCPDispatchFast, + "search_artifacts": MCPDispatchFast, + "search_ast": MCPDispatchFast, + "search_symbols": MCPDispatchFast, + "search_text": MCPDispatchFast, + "session": MCPDispatchFast, + "set_planning_mode": MCPDispatchFast, + "sibling_diff_context": MCPDispatchFast, + "simulate_chain": MCPDispatchFast, + "smart_context": MCPDispatchFast, + "subscribe_daemon_health": MCPDispatchFast, + "subscribe_diagnostics": MCPDispatchFast, + "subscribe_graph_invalidated": MCPDispatchFast, + "subscribe_stale_refs": MCPDispatchFast, + "subscribe_workspace_readiness": MCPDispatchFast, + "suggest_pattern": MCPDispatchFast, + "suggest_queries": MCPDispatchFast, + "suggest_reviewers": MCPDispatchFast, + "suggested_review_questions": MCPDispatchFast, + "symbols_for_ranges": MCPDispatchFast, + "taint_paths": MCPDispatchFast, + "tool_profile": MCPDispatchFast, + "tools_search": MCPDispatchFast, + "trace": MCPDispatchFast, + "trace_path": MCPDispatchFast, + "triage_prs": MCPDispatchFast, + "unsubscribe_daemon_health": MCPDispatchFast, + "unsubscribe_diagnostics": MCPDispatchFast, + "unsubscribe_graph_invalidated": MCPDispatchFast, + "unsubscribe_stale_refs": MCPDispatchFast, + "unsubscribe_workspace_readiness": MCPDispatchFast, + "verify_change": MCPDispatchFast, + "walk_graph": MCPDispatchFast, + "winnow_symbols": MCPDispatchFast, + "workflow": MCPDispatchFast, + "workspace": MCPDispatchFast, + "workspace_info": MCPDispatchFast, +} + +// ToolDispatchClass returns the explicit lane and whether the tool is known. +// Unknown tool names stay fast so normal tool validation supplies the error; +// they are not allowed to acquire a privileged mutation reservation. +func ToolDispatchClass(name string) (MCPDispatchClass, bool) { + class, known := toolDispatchClasses[name] + return class, known +} + +// MCPDispatchClassifier lowers a tools/call to its effective admission class +// without executing the handler or changing session state. Implementations +// reuse the product's operation registry for facade calls. +type MCPDispatchClassifier interface { + DispatchClass(name string, arguments map[string]any) MCPDispatchClass +} diff --git a/internal/daemon/mcp_dispatch_class_test.go b/internal/daemon/mcp_dispatch_class_test.go new file mode 100644 index 000000000..2c73473a5 --- /dev/null +++ b/internal/daemon/mcp_dispatch_class_test.go @@ -0,0 +1,15 @@ +package daemon + +import "testing" + +func TestDispatchClassIncludesEveryDurableWriter(t *testing.T) { + for name := range ToolEffects { + if !IsMutating(name) { + continue + } + class, known := ToolDispatchClass(name) + if !known || class != MCPDispatchMutating { + t.Errorf("durable writer %q class=%s known=%v", name, class, known) + } + } +} diff --git a/internal/daemon/mcp_dispatch_completion_test.go b/internal/daemon/mcp_dispatch_completion_test.go new file mode 100644 index 000000000..3347da55c --- /dev/null +++ b/internal/daemon/mcp_dispatch_completion_test.go @@ -0,0 +1,155 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" +) + +func retainedDispatchCompletion(t *testing.T, grant *mcpScheduledGrant) func() { + t.Helper() + release := grant.retainCompletion() + if release == nil { + t.Fatal("live dispatch did not retain its handler completion") + } + t.Cleanup(release) + return release +} + +func TestDispatchRetainedHandlersKeepDrainBoundAfterDispatcherReturns(t *testing.T) { + pool := testDispatchPool(1) + var releases []func() + for i := range mcpDispatchDrainLimit + 1 { + grant := takeScheduled(t, pool, MCPDispatchFast, fmt.Sprint(i)) + releases = append(releases, retainedDispatchCompletion(t, grant)) + grant.complete() // The outer firewall returned; the leaf did not. + } + if pool.owner.draining != 4 || len(pool.slots) != 1 || len(pool.owner.blocked) != 1 { + t.Fatalf("physical work escaped drain bound: draining=%d held=%d blocked=%d", pool.owner.draining, len(pool.slots), len(pool.owner.blocked)) + } + _, err := pool.acquire(withMCPDispatchQueueWait(t.Context(), 0), MCPDispatchFast, "sixth") + if !errors.Is(err, errMCPDispatchSaturated) { + t.Fatalf("full draining allowance lost its capacity charge: %v", err) + } + releases[0]() // One leaf exits; the charged fifth is promoted to draining. + if pool.owner.draining != 4 || len(pool.slots) != 0 || len(pool.owner.blocked) != 0 { + t.Fatal("leaf completion did not promote the capacity-charged cancellation") + } + keeper := takeScheduled(t, pool, MCPDispatchFast, "unrelated") + releases[0]() // A late duplicate cannot release the keeper or another drain. + if len(pool.slots) != 1 || pool.owner.draining != 4 { + t.Fatal("duplicate completion changed another dispatch's ownership") + } + for _, release := range releases[1:] { + release() + } + if pool.owner.draining != 0 || len(pool.slots) != 1 { + t.Fatal("retained completion released unrelated live capacity") + } + keeper.complete() +} + +func TestDispatchCompletionOwnerSurvivesDetachWithoutAdoptingReattach(t *testing.T) { + pool := testDispatchPool(1) + ctx := withMCPDispatchSlotLease(t.Context()) + lease := mcpDispatchSlotLeaseForContext(ctx) + old := takeScheduled(t, pool, MCPDispatchFast, "old") + attached := lease.attach(nil) + attached.scheduled = old + owned := WithMCPDispatchCompletionOwner(ctx) + parent := RetainMCPDispatchCompletion(owned) + if parent == nil { + t.Fatal("outer firewall did not retain physical work") + } + t.Cleanup(parent) + lease.detach(attached) + old.complete() + fresh := takeScheduled(t, pool, MCPDispatchFast, "fresh") + replacement := lease.attach(nil) + replacement.scheduled = fresh + // A nested firewall can enter after its outer dispatch has detached. Its + // inherited context must still name the old operation, not the new attach. + nested := RetainMCPDispatchCompletion(WithMCPDispatchCompletionOwner(owned)) + if nested == nil { + t.Fatal("nested firewall lost its captured physical owner") + } + t.Cleanup(nested) + parent() + if pool.owner.draining != 1 || len(pool.slots) != 1 { + t.Fatal("parent return released a nested leaf or the replacement") + } + nested() + if pool.owner.draining != 0 || len(pool.slots) != 1 { + t.Fatal("nested leaf completion changed the replacement's ownership") + } + if RetainMCPDispatchCompletion(owned) != nil { + t.Fatal("stale owner revived an already completed dispatch") + } + lease.detach(replacement) + fresh.complete() +} + +func TestDispatchCompletionOwnerFollowsSlowTransferButNotPassiveRelease(t *testing.T) { + pool := testDispatchPool(3) + ctx := withMCPDispatchSlotLease(withMCPDispatchQueueWait(t.Context(), 0)) + lease := mcpDispatchSlotLeaseForContext(ctx) + mutation := takeScheduled(t, pool, MCPDispatchMutating, "edit") + attached := lease.attach(nil) + attached.scheduled = mutation + owned := WithMCPDispatchCompletionOwner(ctx) + if err := MoveMCPDispatchToSlowLane(owned, nil); err != nil { + t.Fatal(err) + } + slow := lease.scheduledGrant(attached, mutation) + t.Cleanup(slow.complete) + release := RetainMCPDispatchCompletion(owned) + if release == nil { + t.Fatal("slow wait lost its physical completion owner") + } + t.Cleanup(release) + slow.complete() + if pool.draining[MCPDispatchSlow] != 1 || pool.active[MCPDispatchMutating] != 0 { + t.Fatal("abandoned wait did not retain the transferred slow grant") + } + release() + lease.detach(attached) + passive := takeScheduled(t, pool, MCPDispatchMutating, "passive") + attached = lease.attach(nil) + attached.scheduled = passive + passiveCtx := WithMCPDispatchCompletionOwner(ctx) + if !ReleaseMCPDispatchSlot(passiveCtx) { + t.Fatal("explicit passive release did not return capacity") + } + if RetainMCPDispatchCompletion(passiveCtx) != nil || len(pool.slots) != 0 { + t.Fatal("passive wait revived its released grant") + } + if RetainMCPDispatchCompletion(context.Background()) != nil { + t.Fatal("standalone request acquired daemon completion ownership") + } +} + +func TestDispatchRetainedCompletionConcurrentWithOuterReturn(t *testing.T) { + pool := testDispatchPool(1) + grant := takeScheduled(t, pool, MCPDispatchFast, "outer") + parent := retainedDispatchCompletion(t, grant) + var group sync.WaitGroup + for range 32 { + group.Go(func() { + release := grant.retainCompletion() + if release == nil { + t.Error("live parent lost nested completion ownership") + return + } + grant.complete() + release() + release() + }) + } + group.Wait() + parent() + if pool.owner.draining != 0 || len(pool.slots) != 0 { + t.Fatal("concurrent nested/outer completion leaked capacity") + } +} diff --git a/internal/daemon/mcp_dispatch_context.go b/internal/daemon/mcp_dispatch_context.go new file mode 100644 index 000000000..6caf9f707 --- /dev/null +++ b/internal/daemon/mcp_dispatch_context.go @@ -0,0 +1,120 @@ +package daemon + +import ( + "context" + "encoding/json" + "errors" + "sync/atomic" +) + +type mcpDispatchRequestContextKey struct{} +type mcpDispatchRequestInfo struct { + class MCPDispatchClass + session string + protocol bool +} + +var mcpDispatchConnectionSequence atomic.Uint64 + +func isMCPInitializationRequest(method string) bool { + return method == "initialize" || method == "server/discover" +} + +func withMCPDispatchRequest(ctx context.Context, info mcpDispatchRequestInfo) context.Context { + return context.WithValue(ctx, mcpDispatchRequestContextKey{}, info) +} +func mcpDispatchRequestForContext(ctx context.Context) mcpDispatchRequestInfo { + info, _ := ctx.Value(mcpDispatchRequestContextKey{}).(mcpDispatchRequestInfo) + return info +} +func prepareMCPDispatchContext(ctx context.Context, session string, envelope mcpRequestEnvelope, classifier MCPDispatchClassifier) context.Context { + if _, prepared := ctx.Value(mcpDispatchRequestContextKey{}).(mcpDispatchRequestInfo); prepared { + return ctx + } + // The table proves only known canonical local names. Unknown/qualified + // remote tools remain fast; the shared total cap and session fairness still + // apply. Do not invent remote effects from a spelling or claim census parity. + class, _ := ToolDispatchClass(envelope.Params.Name) + if envelope.Method == "tools/call" && classifier != nil { + var arguments map[string]any + _ = json.Unmarshal(envelope.Params.Arguments, &arguments) + class = classifier.DispatchClass(envelope.Params.Name, arguments) + } + if envelope.Method != "tools/call" { + class = MCPDispatchFast + } + ctx = withMCPDispatchRequest(ctx, mcpDispatchRequestInfo{class: class, session: session, protocol: isMCPInitializationRequest(envelope.Method) || len(envelope.ID) == 0 && envelope.Method != ""}) + if isLongLivedMCPRequest(envelope.Method) || isMCPCancellation(envelope.Method) { + ctx = withMCPDispatchUnbounded(ctx) + } + return ctx +} + +// isMCPCancellation marks the one notification that must not wait for a slot. +// It only gives capacity back, so queueing it behind a full pool would keep +// that capacity held exactly when it is needed. The Unix reader cancels in +// place and never dispatches it; HTTP hands it to the MCP server, which owns +// the in-flight request it cancels. +func isMCPCancellation(method string) bool { + return method == "notifications/cancelled" +} + +// DispatchMCPFrame is the shared admission/replay entry for the daemon's Unix +// and HTTP transports. Their ordering, cancellation and framing stay with the +// transport; replay hits and in-flight followers never acquire a second slot. +func DispatchMCPFrame(ctx context.Context, sess *Session, frame []byte, dispatcher MCPDispatcher) ([]byte, error) { + var envelope mcpRequestEnvelope + if _, prepared := ctx.Value(mcpDispatchRequestContextKey{}).(mcpDispatchRequestInfo); !prepared { + _ = json.Unmarshal(frame, &envelope) // Public dispatch retains normal validation. + id := "" + if sess != nil { + id = sess.ID + } + classifier, _ := dispatcher.(MCPDispatchClassifier) + ctx = prepareMCPDispatchContext(ctx, id, envelope, classifier) + } + if mcpDispatchSlotLeaseForContext(ctx) == nil && len(envelope.ID) > 0 { + ctx = withMCPDispatchSlotLease(ctx) + } + reply, _, err := sess.dispatchMCPOnceContext(ctx, frame, func() ([]byte, error) { + reply, err := dispatcher.Dispatch(ctx, sess, frame) + if ctxErr := ctx.Err(); ctxErr != nil && (!isMCPDispatchUnbounded(ctx) || err != nil || len(reply) == 0) { + return nil, ctxErr + } + return reply, err + }) + return reply, err +} + +// MCPDispatchBusyResponse gives HTTP the same retryable JSON-RPC refusal as +// Unix without turning dispatch errors into generic transport errors. It is +// nil for notifications or non-admission failures; those keep transport policy. +func MCPDispatchBusyResponse(frame []byte, err error) []byte { + if !errors.Is(err, errMCPDispatchSaturated) { + return nil + } + var envelope mcpRequestEnvelope + if json.Unmarshal(frame, &envelope) != nil || len(envelope.ID) == 0 { + return nil + } + if _, valid := canonicalMCPRequestID(envelope.ID); !valid { + return nil + } + return mcpDispatchBusyResponse(envelope.ID, envelope.Method, err) +} +func mcpDispatchBusyResponse(id json.RawMessage, method string, err error) []byte { + data := map[string]any{"outcome": "server_busy", "retryable": true, "retry_after_ms": mcpDispatchBusyRetryAfter.Milliseconds()} + if method == "tools/call" { + data["delivery_unknown"] = false + } + var busy *mcpDispatchBusyError + if errors.As(err, &busy) { + data["queue_wait_ms"] = busy.waited.Milliseconds() + retry := busy.retryAfter + if retry <= 0 { + retry = mcpDispatchBusyRetryAfter + } + data["retry_after_ms"] = retry.Milliseconds() + } + return mcpErrorResponse(id, -32002, "MCP dispatcher is busy", data) +} diff --git a/internal/daemon/mcp_dispatch_context_test.go b/internal/daemon/mcp_dispatch_context_test.go new file mode 100644 index 000000000..d2e8b86de --- /dev/null +++ b/internal/daemon/mcp_dispatch_context_test.go @@ -0,0 +1,81 @@ +package daemon + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" +) + +func TestDispatchEnvelopeLeavesMalformedToolArgumentsToPublicValidation(t *testing.T) { + for _, value := range []string{`[]`, `"invalid"`, `42`, `null`} { + frame := []byte(`{"jsonrpc":"2.0","id":17,"method":"tools/call","params":{"name":"read","arguments":` + value + `}}`) + var envelope mcpRequestEnvelope + if err := json.Unmarshal(frame, &envelope); err != nil { + t.Fatalf("admission consumed handler validation for arguments=%s: %v", value, err) + } + if string(envelope.ID) != "17" || string(envelope.Params.Arguments) != value { + t.Fatalf("envelope identity/payload changed: %+v", envelope) + } + } +} + +func TestSharedDispatchEntryReplayDoesNotAcquireAnotherSlot(t *testing.T) { + slots := newMCPDispatchSlots(1) + ctx := withMCPDispatchQueueWait(withMCPDispatchSlots(t.Context(), slots), 0) + sess := &Session{ID: "same-session", LogicalSessionID: "same-logical-session"} + frame := []byte(`{"jsonrpc":"2.0","id":9,"method":"tools/call","params":{"name":"read_file"}}`) + calls := 0 + dispatcher := testMCPDispatcherFunc(func(context.Context, *Session, []byte) ([]byte, error) { + calls++ + return []byte(`{"jsonrpc":"2.0","id":9,"result":{"ok":true}}`), nil + }) + first, err := DispatchMCPFrame(ctx, sess, frame, dispatcher) + if err != nil { + t.Fatal(err) + } + holder := takeScheduled(t, mcpSchedulerFor(slots), MCPDispatchMutating, "holder") + defer holder.complete() + replay, err := DispatchMCPFrame(ctx, sess, frame, dispatcher) + if err != nil || string(replay) != string(first) || calls != 1 || len(slots) != 1 { + t.Fatalf("cached replay double-admitted: err=%v calls=%d held=%d first=%s replay=%s", err, calls, len(slots), first, replay) + } +} + +func TestDiscoveryUsesProtocolCapacityWithoutLegacyNotificationBarrier(t *testing.T) { + slots := newMCPDispatchSlots(2) + var zero time.Duration + started, release := make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(unblock) + conn, reader, _ := startMCPConnectionWithHooksTest(t, time.Second, func(ctx context.Context, frame []byte) ([]byte, error) { + var envelope mcpRequestEnvelope + if err := json.Unmarshal(frame, &envelope); err != nil { + return nil, err + } + if envelope.Method == "server/discover" { + if !mcpDispatchRequestForContext(ctx).protocol { + return nil, errors.New("discovery was not admitted as protocol negotiation") + } + close(started) + <-release + } + return []byte(`{"jsonrpc":"2.0","id":` + string(envelope.ID) + `,"result":{"ok":true}}`), nil + }, &mcpConnectionHooks{dispatchSlots: slots, dispatchQueueWait: &zero}) + writeMCPTestFrame(t, conn, `{"jsonrpc":"2.0","id":1,"method":"server/discover","params":{}}`) + awaitMCPTestSignal(t, started, "modern discovery did not reach protocol dispatch") + // Modern negotiation is stateless: no initialize/initialized barrier is + // synthesized. Its protocol slot must not consume the only shared read slot. + writeMCPTestFrame(t, conn, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"read_file"}}`) + read := readMCPTestResponse(t, conn, reader) + if string(read.ID) != "2" || read.Error != nil || read.Result["ok"] != true { + t.Fatalf("tool after modern discovery: %+v", read) + } + unblock() + discovery := readMCPTestResponse(t, conn, reader) + if string(discovery.ID) != "1" || discovery.Error != nil { + t.Fatalf("discovery response: %+v", discovery) + } +} diff --git a/internal/daemon/mcp_dispatch_lanes_test.go b/internal/daemon/mcp_dispatch_lanes_test.go new file mode 100644 index 000000000..1f807b388 --- /dev/null +++ b/internal/daemon/mcp_dispatch_lanes_test.go @@ -0,0 +1,218 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + "time" +) + +func scheduledContext(t *testing.T, slots chan struct{}, class MCPDispatchClass, session string) context.Context { + t.Helper() + return withMCPDispatchSlotLease(withMCPDispatchRequest(withMCPDispatchSlots(t.Context(), slots), mcpDispatchRequestInfo{class: class, session: session})) +} + +func TestDispatchProtocolCompletionWakesToolLane(t *testing.T) { + slots := newMCPDispatchSlots(1) + ctx := withMCPDispatchRequest(withMCPDispatchSlots(t.Context(), slots), mcpDispatchRequestInfo{protocol: true}) + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + t.Cleanup(unblock) + go func() { + defer close(done) + _, _ = dispatchMCPWithContext(ctx, func() ([]byte, error) { close(started); <-release; return nil, nil }) + }() + awaitMCPTestSignal(t, started, "protocol handler not admitted") + queued := queueScheduled(t, mcpSchedulerFor(slots), MCPDispatchFast, "reader") + unblock() + receiveScheduled(t, queued).complete() + awaitMCPTestSignal(t, done, "protocol dispatch did not finish") +} + +func TestDispatchCancellationReturnsCapacityWithin100ms(t *testing.T) { + slots := newMCPDispatchSlots(1) + ctx, cancel := context.WithCancel(scheduledContext(t, slots, MCPDispatchFast, "reader")) + defer cancel() + started, done := make(chan struct{}), make(chan error, 1) + go func() { + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { close(started); <-ctx.Done(); return nil, ctx.Err() }) + done <- err + }() + awaitMCPTestSignal(t, started, "context-aware handler not admitted") + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancel error=%v", err) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("cancellation did not return within100ms") + } + if len(slots) != 0 { + t.Fatal("canceled context-aware handler still occupies dispatch capacity") + } +} + +func TestDispatchUncooperativeCancellationCannotExceedFourDrains(t *testing.T) { + slots := newMCPDispatchSlots(1) + pool := mcpSchedulerFor(slots) + release := make(chan struct{}) + unblock := sync.OnceFunc(func() { close(release) }) + var bodies sync.WaitGroup + t.Cleanup(func() { unblock(); bodies.Wait() }) + for i := range mcpDispatchDrainLimit + 1 { + ctx, cancel := context.WithCancel(scheduledContext(t, slots, MCPDispatchMutating, fmt.Sprintf("session%d", i))) + started, done := make(chan struct{}), make(chan error, 1) + bodies.Add(1) + bodyDone := sync.OnceFunc(bodies.Done) + go func() { + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { defer bodyDone(); close(started); <-release; return nil, nil }) + if mcpDispatchNotStarted(err) { + bodyDone() + } + done <- err + }() + awaitMCPTestSignal(t, started, "uncooperative handler did not start") + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancel error=%v", err) + } + case <-time.After(100 * time.Millisecond): + t.Fatal("canceled caller still waiting") + } + } + pool.owner.mu.Lock() + if pool.owner.draining != 4 || pool.draining[MCPDispatchMutating] != 4 || pool.active[MCPDispatchMutating] != 1 || len(slots) != 1 { + t.Errorf("draining=%d pooldrain=%v active=%v held=%d", pool.owner.draining, pool.draining, pool.active, len(slots)) + } + pool.owner.mu.Unlock() + ctx := withMCPDispatchQueueWait(scheduledContext(t, slots, MCPDispatchMutating, "overflow"), 0) + ran := false + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { ran = true; return nil, nil }) + if ran || !errors.Is(err, errMCPDispatchSaturated) { + t.Fatalf("full draining allowance ran=%v error=%v", ran, err) + } + unblock() +} + +func TestDispatchEditWaitMovesToSlowAfterReturningMutationCapacity(t *testing.T) { + slots := newMCPDispatchSlots(3) + pool := mcpSchedulerFor(slots) + slow := takeScheduled(t, pool, MCPDispatchSlow, "outline") + ctx := scheduledContext(t, slots, MCPDispatchMutating, "editor") + entered, slowStarted, finish := make(chan struct{}), make(chan struct{}), make(chan struct{}) + done := make(chan error, 1) + unblock := sync.OnceFunc(func() { close(finish) }) + t.Cleanup(unblock) + go func() { + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { + close(entered) + if err := MoveMCPDispatchToSlowLane(ctx, nil); err != nil { + return nil, err + } + if err := MoveMCPDispatchToSlowLane(ctx, nil); err != nil { + return nil, err + } // idempotent + close(slowStarted) + <-finish + return []byte(`{"ok":true}`), nil + }) + done <- err + }() + awaitMCPTestSignal(t, entered, "mutation handler did not start") + awaitMCPDispatchQueueWaiters(t, slots, 1) + pool.owner.mu.Lock() + if pool.active[MCPDispatchMutating] != 0 { + t.Errorf("mutation slot retained during slow admission: %v", pool.active) + } + pool.owner.mu.Unlock() + // Both reserved mutation slots can be used while the old edit waits. + edit1 := takeScheduled(t, pool, MCPDispatchMutating, "next1") + edit2 := takeScheduled(t, pool, MCPDispatchMutating, "next2") + edit1.complete() + edit2.complete() + slow.complete() + awaitMCPTestSignal(t, slowStarted, "post-commit wait did not enter slow lane") + pool.owner.mu.Lock() + if pool.active[MCPDispatchSlow] != 1 || pool.active[MCPDispatchMutating] != 0 { + t.Errorf("edit wait lane=%v", pool.active) + } + pool.owner.mu.Unlock() + unblock() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("slow edit wait did not complete") + } + if len(slots) != 0 { + t.Fatalf("slow wait leaked %d tokens", len(slots)) + } +} + +func TestDispatchEditWaitNeedsNoSlowSlotOncePublished(t *testing.T) { + for _, when := range []string{"while_queued", "before_moving"} { + t.Run(when, func(t *testing.T) { + slots := newMCPDispatchSlots(8) + pool := mcpSchedulerFor(slots) + takeScheduled(t, pool, MCPDispatchSlow, "outline") + takeScheduled(t, pool, MCPDispatchSlow, "dead_code") + ctx := scheduledContext(t, slots, MCPDispatchMutating, "editor") + published := make(chan struct{}) + if when == "before_moving" { + close(published) + } + moved, finish, done := make(chan error, 1), make(chan struct{}), make(chan error, 1) + unblock := sync.OnceFunc(func() { close(finish) }) + t.Cleanup(unblock) + go func() { + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { + moved <- MoveMCPDispatchToSlowLane(ctx, published) + <-finish + return []byte(`{"ok":true}`), nil + }) + done <- err + }() + if when == "while_queued" { + awaitMCPDispatchQueueWaiters(t, slots, 1) + close(published) + } + // The queue budget is seconds; a finished publication must not spend it. + select { + case err := <-moved: + if err != nil { + t.Fatalf("published edit wait: %v", err) + } + case <-time.After(time.Second): + t.Fatal("published edit wait stayed queued behind the slow lane") + } + pool.owner.mu.Lock() + mutating := 0 + if when == "before_moving" { + mutating = 1 // nothing left to wait for, so nothing to move + } + if pool.active[MCPDispatchSlow] != 2 || pool.active[MCPDispatchMutating] != mutating || pool.lanes[MCPDispatchSlow].queued != 0 { + t.Errorf("published edit wait lanes: active=%v slow queued=%d", pool.active, pool.lanes[MCPDispatchSlow].queued) + } + pool.owner.mu.Unlock() + unblock() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("edit dispatch did not complete") + } + if len(slots) != 2 { + t.Fatalf("published edit wait left %d tokens, want the two slow holders", len(slots)) + } + }) + } +} diff --git a/internal/daemon/mcp_dispatch_queue_test.go b/internal/daemon/mcp_dispatch_queue_test.go index ea27e8b04..14d821739 100644 --- a/internal/daemon/mcp_dispatch_queue_test.go +++ b/internal/daemon/mcp_dispatch_queue_test.go @@ -297,6 +297,7 @@ func TestServeMCPDispatchQueueCancellationWhileQueued(t *testing.T) { func TestServeMCPDispatchQueueDeadlineWhileQueued(t *testing.T) { f := startMCPDispatchQueueFixture(t, 300*time.Millisecond, 2*time.Second) f.hold(t) + queuedAt := time.Now() writeMCPTestFrame(t, f.conn, mcpDispatchQueueToolCall("2")) responses := readMCPTestResponsesByID(t, f.conn, f.reader, 2) @@ -305,11 +306,22 @@ func TestServeMCPDispatchQueueDeadlineWhileQueued(t *testing.T) { t.Fatalf("started holder deadline response = %#v", holder) } queued := responses["2"] - if queued.Error == nil || queued.Error.Code != mcpRequestTimeoutCode { + if queued.Error == nil || queued.Error.Code != -32002 { t.Fatalf("queued deadline response = %#v", queued) } - if data := queued.Error.Data; data["outcome"] != "deadline" || data["deadline_ms"] != float64(300) || data["retryable"] != false || data["delivery_unknown"] != false { - t.Fatalf("queued deadline data = %#v", data) + // The lane budget deliberately reserves 100ms of the request deadline + // for its not-started busy reply. The exact clamp is separately pinned + // by TestDispatchSchedulerDeadlineBudgetAndSmallOverrides; allow broad + // scheduling slack here instead of a tight wall-clock tolerance. + data := assertMCPDispatchQueueBusy(t, queued, "2") + if waited, ok := data["queue_wait_ms"].(float64); !ok || waited <= 0 || waited >= 300 { + t.Fatalf("queued budget did not leave response time: %#v", data) + } + if retry, ok := data["retry_after_ms"].(float64); !ok || retry < 100 || retry > 5000 { + t.Fatalf("queued retry guidance = %#v", data) + } + if elapsed := time.Since(queuedAt); elapsed >= 2*time.Second { + t.Fatalf("deadline-clamped calls consumed configured 2s queue budget: %s", elapsed) } if got := f.callCount("2"); got != 0 { t.Fatalf("queued request ran %d times", got) diff --git a/internal/daemon/mcp_dispatch_scheduler.go b/internal/daemon/mcp_dispatch_scheduler.go new file mode 100644 index 000000000..08967accd --- /dev/null +++ b/internal/daemon/mcp_dispatch_scheduler.go @@ -0,0 +1,570 @@ +package daemon + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/zzet/gortex/internal/viewmetrics" +) + +const ( + mcpDispatchSlowLimit = 2 + mcpDispatchMutationReserve = 2 + mcpDispatchSessionLimit = 4 + mcpDispatchDrainLimit = 4 + mcpDispatchDeadlineReserve = 100 * time.Millisecond +) + +// One short mutex owns admission and lease accounting, including the process +// drain bound across pools. Production has one pool; isolated pools retain the +// existing channel injection seam. No handler or I/O runs under this mutex. +type mcpDispatchCoordinator struct { + mu sync.Mutex + pools map[chan struct{}]*mcpDispatchScheduler + draining int + blocked []*mcpScheduledGrant +} + +var mcpDispatchProcess = mcpDispatchCoordinator{pools: make(map[chan struct{}]*mcpDispatchScheduler)} + +type mcpDispatchScheduler struct { + owner *mcpDispatchCoordinator + slots chan struct{} + reserve, slowLimit int + active [3]int + draining [3]int + nonMutating int + reservedAdmissions uint64 + protocolWaiting int + held map[string]int + lanes [3]mcpDispatchLane +} + +type mcpDispatchLane struct { + sessions []*mcpDispatchSessionQueue + next int + queued int + lastDrain time.Time + drainInterval time.Duration + wait [3]mcpDispatchWaitStats + refusals uint64 +} + +type mcpDispatchSessionQueue struct { + session string + requests []*mcpDispatchWaiter +} + +type mcpDispatchWaiter struct { + class MCPDispatchClass + session string + queuedAt time.Time + expiresAt time.Time + ctx context.Context + err error + done chan struct{} + grant *mcpScheduledGrant + observed bool +} + +type mcpGrantState uint8 + +const ( + mcpGrantActive mcpGrantState = iota + mcpGrantDraining + mcpGrantReleased +) + +type mcpScheduledGrant struct { + pool *mcpDispatchScheduler + class MCPDispatchClass + session string + state mcpGrantState + protocol bool + cancelled bool + // A middleware firewall can return before the handler it abandoned. + // Keep the physical completion charge until both have returned. + retainedHandlers int + dispatchReturned bool +} + +func mcpSchedulerFor(slots chan struct{}) *mcpDispatchScheduler { + return mcpDispatchProcess.pool(slots) +} + +func (c *mcpDispatchCoordinator) pool(slots chan struct{}) *mcpDispatchScheduler { + c.mu.Lock() + defer c.mu.Unlock() + if pool := c.pools[slots]; pool != nil { + return pool + } + limit := cap(slots) + reserve := min(mcpDispatchMutationReserve, max(0, limit-1)) + pool := &mcpDispatchScheduler{owner: c, slots: slots, reserve: reserve, slowLimit: min(mcpDispatchSlowLimit, limit-reserve), held: make(map[string]int)} + if c.pools == nil { + c.pools = make(map[chan struct{}]*mcpDispatchScheduler) + } + c.pools[slots] = pool + return pool +} + +func mcpEffectiveQueueWait(ctx context.Context, configured time.Duration, now time.Time) time.Duration { + if deadline, ok := ctx.Deadline(); ok { + return max(0, min(configured, deadline.Sub(now)-mcpDispatchDeadlineReserve)) + } + return max(0, configured) +} + +func (s *mcpDispatchScheduler) otherSessionWaitingLocked(session string) bool { + for i := range s.lanes { + queues := s.lanes[i].sessions + if len(queues) > 1 || len(queues) == 1 && queues[0].session != session { + return true + } + } + return false +} + +func (s *mcpDispatchScheduler) eligibleLocked(class MCPDispatchClass, session string) bool { + if len(s.slots) >= cap(s.slots) { + return false + } + if class != MCPDispatchMutating && s.nonMutating >= cap(s.slots)-s.reserve { + return false + } + if class == MCPDispatchSlow && s.active[class] >= s.slowLimit { + return false + } + return s.held[session] < mcpDispatchSessionLimit || !s.otherSessionWaitingLocked(session) +} + +func (s *mcpDispatchScheduler) grantLocked(class MCPDispatchClass, session string) *mcpScheduledGrant { + select { + case s.slots <- struct{}{}: + default: + return nil // Protocol admission or an injected holder took capacity. + } + s.active[class]++ + viewmetrics.AddGauge(viewmetrics.MCPDispatchInFlight, 1, class.String()) + if class == MCPDispatchMutating && s.active[class] <= s.reserve { + s.reservedAdmissions++ + viewmetrics.Count(viewmetrics.MCPDispatchReservedAdmissionsTotal) + viewmetrics.AddGauge(viewmetrics.MCPDispatchReservedSlotsUsed, 1) + } + if class != MCPDispatchMutating { + s.nonMutating++ + } + s.held[session]++ + return &mcpScheduledGrant{pool: s, class: class, session: session} +} + +// scheduleLocked honors class priority and rotates whole sessions within each +// class. A temporarily ineligible session cannot block another class/session. +func (s *mcpDispatchScheduler) scheduleLocked() { + for { + admitted := false + for _, class := range [...]MCPDispatchClass{MCPDispatchMutating, MCPDispatchFast, MCPDispatchSlow} { + lane := &s.lanes[class] + for offset := 0; offset < len(lane.sessions); offset++ { + index := (lane.next + offset) % len(lane.sessions) + q := lane.sessions[index] + waiter := q.requests[0] + now := time.Now() + if waiter.ctx != nil && waiter.ctx.Err() != nil { + waiter = s.popLocked(class, index) + s.observeQueuedWaitLocked(waiter, mcpWaitCancelled, now) + waiter.err = fmt.Errorf("%w: %w", errMCPDispatchNotStarted, waiter.ctx.Err()) + close(waiter.done) + admitted = true + break + } + if !waiter.expiresAt.IsZero() && !now.Before(waiter.expiresAt) { + waiter = s.popLocked(class, index) + s.observeQueuedWaitLocked(waiter, mcpWaitRefused, now) + waiter.err = &mcpDispatchBusyError{waited: now.Sub(waiter.queuedAt), retryAfter: s.retryAfterLocked(class), lane: class} + close(waiter.done) + admitted = true + break + } + if !s.eligibleLocked(class, q.session) { + continue + } + grant := s.grantLocked(class, q.session) + if grant == nil { + return + } + waiter = s.popLocked(class, index) + s.observeQueuedWaitLocked(waiter, mcpWaitAdmitted, now) + waiter.grant = grant + close(waiter.done) + admitted = true + break + } + if admitted { + break + } + } + if !admitted { + return + } + } +} + +// popLocked advances the round-robin cursor for both grants and terminal +// queued outcomes. Empty session queues never remain in the ring. +func (s *mcpDispatchScheduler) popLocked(class MCPDispatchClass, index int) *mcpDispatchWaiter { + lane := &s.lanes[class] + q := lane.sessions[index] + waiter := q.requests[0] + q.requests[0] = nil + q.requests = q.requests[1:] + lane.queued-- + viewmetrics.AddGauge(viewmetrics.MCPDispatchQueueDepth, -1, class.String()) + if len(q.requests) == 0 { + copy(lane.sessions[index:], lane.sessions[index+1:]) + lane.sessions[len(lane.sessions)-1] = nil + lane.sessions = lane.sessions[:len(lane.sessions)-1] + lane.next = index + } else { + lane.next = index + 1 + } + if len(lane.sessions) > 0 { + lane.next %= len(lane.sessions) + } else { + lane.next = 0 + } + return waiter +} + +func (s *mcpDispatchScheduler) enqueueLocked(waiter *mcpDispatchWaiter) { + viewmetrics.AddGauge(viewmetrics.MCPDispatchQueueDepth, 1, waiter.class.String()) + lane := &s.lanes[waiter.class] + for _, q := range lane.sessions { + if q.session == waiter.session { + q.requests = append(q.requests, waiter) + lane.queued++ + return + } + } + lane.sessions = append(lane.sessions, &mcpDispatchSessionQueue{session: waiter.session, requests: []*mcpDispatchWaiter{waiter}}) + lane.queued++ +} + +func (s *mcpDispatchScheduler) removeLocked(waiter *mcpDispatchWaiter) { + lane := &s.lanes[waiter.class] + for index, q := range lane.sessions { + if q.session != waiter.session { + continue + } + for at, request := range q.requests { + if request != waiter { + continue + } + copy(q.requests[at:], q.requests[at+1:]) + q.requests[len(q.requests)-1] = nil + q.requests = q.requests[:len(q.requests)-1] + lane.queued-- + viewmetrics.AddGauge(viewmetrics.MCPDispatchQueueDepth, -1, waiter.class.String()) + if len(q.requests) == 0 { + copy(lane.sessions[index:], lane.sessions[index+1:]) + lane.sessions[len(lane.sessions)-1] = nil + lane.sessions = lane.sessions[:len(lane.sessions)-1] + if index < lane.next { + lane.next-- + } + if len(lane.sessions) > 0 { + lane.next %= len(lane.sessions) + } else { + lane.next = 0 + } + } + return + } + } +} + +func (s *mcpDispatchScheduler) retryAfterLocked(class MCPDispatchClass) time.Duration { + lane := &s.lanes[class] + estimate := mcpDispatchBusyRetryAfter + if lane.drainInterval > 0 { + // A drain releases one admission opportunity; include the waiting frontier + // without allowing multiplication to overflow the bounded estimate. + ahead := time.Duration(max(1, lane.queued+1)) + estimate = min(5*time.Second, lane.drainInterval) * ahead + } + return max(100*time.Millisecond, min(5*time.Second, estimate)) +} + +func (s *mcpDispatchScheduler) acquire(ctx context.Context, class MCPDispatchClass, session string) (*mcpScheduledGrant, error) { + if class != MCPDispatchFast && class != MCPDispatchMutating && class != MCPDispatchSlow { + class = MCPDispatchFast + } + now := time.Now() + s.owner.mu.Lock() + if err := ctx.Err(); err != nil { + s.observeWaitLocked(class, mcpWaitCancelled, 0) + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + return nil, fmt.Errorf("%w: %w", errMCPDispatchNotStarted, err) + } + hasQueue := false + for i := range s.lanes { + hasQueue = hasQueue || s.lanes[i].queued > 0 + } + if !hasQueue && s.eligibleLocked(class, session) { + if grant := s.grantLocked(class, session); grant != nil { + s.observeWaitLocked(class, mcpWaitAdmitted, time.Since(now)) + s.owner.mu.Unlock() + return grant, nil + } + } + wait := mcpEffectiveQueueWait(ctx, mcpDispatchQueueWaitForContext(ctx), now) + if s.lanes[class].queued >= mcpDispatchQueueFactor*cap(s.slots) { + s.observeWaitLocked(class, mcpWaitRefused, 0) + retry := s.retryAfterLocked(class) + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + return nil, &mcpDispatchBusyError{retryAfter: retry, lane: class} + } + waiter := &mcpDispatchWaiter{class: class, session: session, queuedAt: now, done: make(chan struct{})} + waiter.ctx = ctx + if wait > 0 { + waiter.expiresAt = now.Add(wait) + } + s.enqueueLocked(waiter) + s.scheduleLocked() + // Even with queuing off, a request may use capacity unavailable to the + // other queued classes (notably the mutation reservation). + if waiter.grant != nil { + s.owner.mu.Unlock() + return waiter.grant, nil + } + if waiter.err != nil { + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + return nil, waiter.err + } + if wait <= 0 { + s.removeLocked(waiter) + s.observeQueuedWaitLocked(waiter, mcpWaitRefused, now) + retry := s.retryAfterLocked(class) + s.scheduleLocked() + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + return nil, &mcpDispatchBusyError{retryAfter: retry, lane: class} + } + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + timer := time.NewTimer(max(0, time.Until(now.Add(wait)))) + defer timer.Stop() + select { + case <-waiter.done: + case <-timer.C: + case <-ctx.Done(): + } + s.owner.mu.Lock() + if err := ctx.Err(); err != nil { + if waiter.grant != nil { + waiter.grant.completeLocked(time.Now()) + } else { + s.removeLocked(waiter) + s.observeQueuedWaitLocked(waiter, mcpWaitCancelled, time.Now()) + s.scheduleLocked() + } + s.owner.mu.Unlock() + return nil, fmt.Errorf("%w: %w", errMCPDispatchNotStarted, err) + } + if waiter.grant != nil { + s.owner.mu.Unlock() + return waiter.grant, nil + } + if waiter.err != nil { + s.owner.mu.Unlock() + return nil, waiter.err + } + s.removeLocked(waiter) + s.observeQueuedWaitLocked(waiter, mcpWaitRefused, time.Now()) + retry := s.retryAfterLocked(class) + s.scheduleLocked() + s.owner.mu.Unlock() + return nil, &mcpDispatchBusyError{waited: time.Since(now), retryAfter: retry, lane: class} +} + +func (s *mcpDispatchScheduler) noteDrainLocked(class MCPDispatchClass, now time.Time) { + lane := &s.lanes[class] + if !lane.lastDrain.IsZero() { + interval := max(time.Millisecond, now.Sub(lane.lastDrain)) + if lane.drainInterval == 0 { + lane.drainInterval = interval + } else { + lane.drainInterval = (lane.drainInterval*7 + interval) / 8 + } + } + lane.lastDrain = now +} + +func (g *mcpScheduledGrant) dropActiveLocked(now time.Time) { + s := g.pool + select { + case <-s.slots: + default: + panic("MCP dispatch grant lost its capacity token") + } + s.active[g.class]-- + viewmetrics.AddGauge(viewmetrics.MCPDispatchInFlight, -1, g.class.String()) + if !g.protocol && g.class == MCPDispatchMutating && s.active[g.class] < s.reserve { + viewmetrics.AddGauge(viewmetrics.MCPDispatchReservedSlotsUsed, -1) + } + if !g.protocol { + if g.class != MCPDispatchMutating { + s.nonMutating-- + } + s.held[g.session]-- + if s.held[g.session] == 0 { + delete(s.held, g.session) + } + } + s.noteDrainLocked(g.class, now) +} + +func (c *mcpDispatchCoordinator) promoteCancelledLocked(now time.Time) { + for c.draining < mcpDispatchDrainLimit && len(c.blocked) > 0 { + grant := c.blocked[0] + c.blocked[0] = nil + c.blocked = c.blocked[1:] + if grant.state != mcpGrantActive || !grant.cancelled { + continue + } + grant.dropActiveLocked(now) + grant.state = mcpGrantDraining + grant.pool.draining[grant.class]++ + viewmetrics.AddGauge(viewmetrics.MCPDispatchCancelledHeld, -1, grant.class.String()) + viewmetrics.AddGauge(viewmetrics.MCPDispatchDraining, 1, grant.class.String()) + c.draining++ + grant.pool.scheduleLocked() + } +} + +func (g *mcpScheduledGrant) completeLocked(now time.Time) { + if g.state == mcpGrantReleased { + return + } + c := g.pool.owner + if g.state == mcpGrantActive { + g.dropActiveLocked(now) + } else { + c.draining-- + g.pool.draining[g.class]-- + viewmetrics.AddGauge(viewmetrics.MCPDispatchDraining, -1, g.class.String()) + } + g.state = mcpGrantReleased + // Remove a completed capacity-charged cancellation before promotion. + for i, grant := range c.blocked { + if grant == g { + viewmetrics.AddGauge(viewmetrics.MCPDispatchCancelledHeld, -1, g.class.String()) + copy(c.blocked[i:], c.blocked[i+1:]) + c.blocked[len(c.blocked)-1] = nil + c.blocked = c.blocked[:len(c.blocked)-1] + break + } + } + c.promoteCancelledLocked(now) + g.pool.scheduleLocked() +} + +func (g *mcpScheduledGrant) complete() { + g.pool.owner.mu.Lock() + defer g.pool.owner.mu.Unlock() + g.dispatchReturned = true + if g.retainedHandlers == 0 { + g.completeLocked(time.Now()) + } +} + +func (g *mcpScheduledGrant) retainCompletion() func() { + c := g.pool.owner + c.mu.Lock() + if g.state == mcpGrantReleased { + c.mu.Unlock() + return nil + } + g.retainedHandlers++ + c.mu.Unlock() + // The middleware stopped waiting, just like transport cancellation. + // Logical capacity returns, while unfinished work stays drain-bounded. + g.cancel() + return sync.OnceFunc(func() { + c.mu.Lock() + defer c.mu.Unlock() + g.retainedHandlers-- + if g.retainedHandlers == 0 && g.dispatchReturned { + g.completeLocked(time.Now()) + } + }) +} + +// cancel detaches at most four physically unfinished handlers process-wide. +// Once that allowance is full, canceled handlers stay capacity-charged until +// one drain completes or they return themselves. Thus live handler work is +// bounded by the configured cap plus four, never by the cancellation rate. +func (g *mcpScheduledGrant) cancel() { + c := g.pool.owner + c.mu.Lock() + defer c.mu.Unlock() + if g.state != mcpGrantActive || g.cancelled { + return + } + g.cancelled = true + if c.draining >= mcpDispatchDrainLimit { + c.blocked = append(c.blocked, g) + viewmetrics.AddGauge(viewmetrics.MCPDispatchCancelledHeld, 1, g.class.String()) + return + } + g.dropActiveLocked(time.Now()) + g.state = mcpGrantDraining + c.draining++ + g.pool.draining[g.class]++ + viewmetrics.AddGauge(viewmetrics.MCPDispatchDraining, 1, g.class.String()) + g.pool.scheduleLocked() +} + +func (g *mcpScheduledGrant) releaseEarly() bool { + c := g.pool.owner + c.mu.Lock() + defer c.mu.Unlock() + if g.state != mcpGrantActive || g.cancelled { + return false + } + g.completeLocked(time.Now()) + return true +} + +// Protocol initialization/notifications preserve their historical full-cap, +// unbudgeted wait. They are exempt from tool reservations/session limits, but +// their completion still wakes queued tools and shares the same physical cap. +func (s *mcpDispatchScheduler) acquireProtocol(ctx context.Context) (*mcpScheduledGrant, error) { + now := time.Now() + s.owner.mu.Lock() + s.protocolWaiting++ + s.owner.mu.Unlock() + select { + case s.slots <- struct{}{}: + s.owner.mu.Lock() + s.protocolWaiting-- + s.active[MCPDispatchFast]++ + viewmetrics.AddGauge(viewmetrics.MCPDispatchInFlight, 1, MCPDispatchFast.String()) + s.observeWaitLocked(MCPDispatchFast, mcpWaitAdmitted, time.Since(now)) + grant := &mcpScheduledGrant{pool: s, class: MCPDispatchFast, state: mcpGrantActive, protocol: true} + s.owner.mu.Unlock() + return grant, nil + case <-ctx.Done(): + s.owner.mu.Lock() + s.protocolWaiting-- + s.observeWaitLocked(MCPDispatchFast, mcpWaitCancelled, time.Since(now)) + s.owner.mu.Unlock() + signalMCPDispatchAdmission(ctx) + return nil, fmt.Errorf("%w: %w", errMCPDispatchNotStarted, ctx.Err()) + } +} diff --git a/internal/daemon/mcp_dispatch_scheduler_test.go b/internal/daemon/mcp_dispatch_scheduler_test.go new file mode 100644 index 000000000..09b9546b3 --- /dev/null +++ b/internal/daemon/mcp_dispatch_scheduler_test.go @@ -0,0 +1,305 @@ +package daemon + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +type scheduledReply struct { + grant *mcpScheduledGrant + err error +} + +func testDispatchPool(limit int) *mcpDispatchScheduler { + owner := &mcpDispatchCoordinator{} + return owner.pool(newMCPDispatchSlots(limit)) +} + +func takeScheduled(t *testing.T, pool *mcpDispatchScheduler, class MCPDispatchClass, session string) *mcpScheduledGrant { + t.Helper() + ctx := withMCPDispatchQueueWait(t.Context(), 0) + grant, err := pool.acquire(ctx, class, session) + if err != nil { + t.Fatalf("immediate %s/%s: %v", class, session, err) + } + t.Cleanup(grant.complete) + return grant +} + +func queueScheduled(t *testing.T, pool *mcpDispatchScheduler, class MCPDispatchClass, session string) <-chan scheduledReply { + t.Helper() + admission := newMCPDispatchAdmission() + ctx, cancel := context.WithCancel(withMCPDispatchAdmission(withMCPDispatchQueueWait(t.Context(), 5*time.Second), admission)) + result := make(chan scheduledReply, 1) + done := make(chan struct{}) + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Error("queued acquisition did not stop during cleanup") + return + } + select { + case reply := <-result: + if reply.grant != nil { + reply.grant.complete() + } + default: + } + }) + go func() { + defer close(done) + grant, err := pool.acquire(ctx, class, session) + result <- scheduledReply{grant, err} + }() + + select { + case <-admission.done: + case reply := <-result: + if reply.grant != nil { + reply.grant.complete() + } + t.Fatalf("request did not queue: %+v", reply) + case <-time.After(time.Second): + t.Fatal("request did not reach queue admission") + } + return result +} + +func receiveScheduled(t *testing.T, result <-chan scheduledReply) *mcpScheduledGrant { + t.Helper() + select { + case reply := <-result: + if reply.err != nil || reply.grant == nil { + t.Fatalf("queued admission: %+v", reply) + } + t.Cleanup(reply.grant.complete) + return reply.grant + case <-time.After(time.Second): + t.Fatal("queued dispatch not admitted") + return nil + } +} + +func notScheduled(t *testing.T, result <-chan scheduledReply) { + t.Helper() + select { + case reply := <-result: + if reply.grant != nil { + reply.grant.complete() + } + t.Fatalf("unexpected admission: %+v", reply) + default: + } +} + +func TestDispatchSchedulerSlowCapAndMutationReservation(t *testing.T) { + pool := testDispatchPool(8) + slow := []*mcpScheduledGrant{takeScheduled(t, pool, MCPDispatchSlow, "slow1"), takeScheduled(t, pool, MCPDispatchSlow, "slow2")} + thirdSlow := queueScheduled(t, pool, MCPDispatchSlow, "slow3") + for i := range 4 { + takeScheduled(t, pool, MCPDispatchFast, fmt.Sprintf("fast%d", i)) + } + fast := queueScheduled(t, pool, MCPDispatchFast, "extra-fast") + takeScheduled(t, pool, MCPDispatchMutating, "edit1") + takeScheduled(t, pool, MCPDispatchMutating, "edit2") + pool.owner.mu.Lock() + if pool.active[MCPDispatchSlow] != 2 || pool.active[MCPDispatchFast] != 4 || pool.active[MCPDispatchMutating] != 2 { + t.Errorf("active lanes=%v", pool.active) + } + pool.owner.mu.Unlock() + notScheduled(t, thirdSlow) + notScheduled(t, fast) + slow[0].complete() + // Fast waiters have priority over the third slow call. + receiveScheduled(t, fast).complete() + receiveScheduled(t, thirdSlow) +} + +func TestDispatchSchedulerPriorityAndSessionRoundRobin(t *testing.T) { + t.Run("class priority", func(t *testing.T) { + pool := testDispatchPool(8) + var holders []*mcpScheduledGrant + for range 8 { + holders = append(holders, takeScheduled(t, pool, MCPDispatchMutating, "holder")) + } + slow := queueScheduled(t, pool, MCPDispatchSlow, "slow") + fast := queueScheduled(t, pool, MCPDispatchFast, "fast") + edit := queueScheduled(t, pool, MCPDispatchMutating, "edit") + holders[0].complete() + receiveScheduled(t, edit) + notScheduled(t, fast) + notScheduled(t, slow) + holders[1].complete() + receiveScheduled(t, fast) + notScheduled(t, slow) + holders[2].complete() + receiveScheduled(t, slow) + }) + t.Run("round robin", func(t *testing.T) { + pool := testDispatchPool(1) + holder := takeScheduled(t, pool, MCPDispatchMutating, "holder") + a1 := queueScheduled(t, pool, MCPDispatchFast, "A") + a2 := queueScheduled(t, pool, MCPDispatchFast, "A") + a3 := queueScheduled(t, pool, MCPDispatchFast, "A") + b1 := queueScheduled(t, pool, MCPDispatchFast, "B") + b2 := queueScheduled(t, pool, MCPDispatchFast, "B") + holder.complete() + receiveScheduled(t, a1).complete() + notScheduled(t, a2) + receiveScheduled(t, b1).complete() + receiveScheduled(t, a2).complete() + notScheduled(t, a3) + receiveScheduled(t, b2).complete() + receiveScheduled(t, a3) + }) +} + +func TestDispatchSchedulerFourHeldLimitOnlyWhileOtherSessionsWait(t *testing.T) { + pool := testDispatchPool(8) + for range 4 { + takeScheduled(t, pool, MCPDispatchMutating, "A") + } + var holders []*mcpScheduledGrant + for range 4 { + holders = append(holders, takeScheduled(t, pool, MCPDispatchMutating, "holder")) + } + a := queueScheduled(t, pool, MCPDispatchMutating, "A") + b := queueScheduled(t, pool, MCPDispatchMutating, "B") + holders[0].complete() + receiveScheduled(t, b) + notScheduled(t, a) + // B has no backlog now, so A may borrow beyond four again. + holders[1].complete() + receiveScheduled(t, a) +} + +func TestDispatchSchedulerDeadlineBudgetAndSmallOverrides(t *testing.T) { + now := time.Now() + ctx, cancel := context.WithDeadline(t.Context(), now.Add(250*time.Millisecond)) + defer cancel() + if got := mcpEffectiveQueueWait(ctx, 5*time.Second, now); got != 150*time.Millisecond { + t.Errorf("deadline queue budget=%s", got) + } + if got := mcpEffectiveQueueWait(ctx, 50*time.Millisecond, now); got != 50*time.Millisecond { + t.Errorf("configured queue budget=%s", got) + } + if got := mcpEffectiveQueueWait(ctx, 5*time.Second, now.Add(200*time.Millisecond)); got != 0 { + t.Errorf("exhausted queue budget=%s", got) + } + for _, test := range []struct{ cap, reserve, slow int }{{1, 0, 1}, {2, 1, 1}, {3, 2, 1}, {8, 2, 2}, {64, 2, 2}} { + pool := testDispatchPool(test.cap) + if pool.reserve != test.reserve || pool.slowLimit != test.slow { + t.Errorf("cap%d reserve/slow=%d/%d", test.cap, pool.reserve, pool.slowLimit) + } + takeScheduled(t, pool, MCPDispatchFast, "reader") + } +} + +func TestDispatchSchedulerDrainingBoundAcrossPools(t *testing.T) { + owner := &mcpDispatchCoordinator{} + a, b := owner.pool(newMCPDispatchSlots(8)), owner.pool(newMCPDispatchSlots(8)) + var old []*mcpScheduledGrant + for range 4 { + old = append(old, takeScheduled(t, a, MCPDispatchMutating, "old")) + } + for _, grant := range old { + grant.cancel() + } + next := takeScheduled(t, b, MCPDispatchMutating, "next") + next.cancel() + owner.mu.Lock() + if owner.draining != 4 || len(b.slots) != 1 || next.state != mcpGrantActive || !next.cancelled { + t.Errorf("full drains=%d held=%d next=%+v", owner.draining, len(b.slots), next) + } + owner.mu.Unlock() + old[0].complete() + owner.mu.Lock() + if owner.draining != 4 || len(b.slots) != 0 || next.state != mcpGrantDraining { + t.Errorf("drain promotion: count=%d held=%d state=%d", owner.draining, len(b.slots), next.state) + } + owner.mu.Unlock() + next.complete() + for _, grant := range old { + grant.complete() + } + owner.mu.Lock() + if owner.draining != 0 || len(owner.blocked) != 0 { + t.Errorf("drains leaked=%d blocked=%d", owner.draining, len(owner.blocked)) + } + owner.mu.Unlock() +} + +func TestDispatchSchedulerRetryFollowsBoundedDrainRate(t *testing.T) { + pool := testDispatchPool(8) + pool.owner.mu.Lock() + defer pool.owner.mu.Unlock() + now := time.Now() + if got := pool.retryAfterLocked(MCPDispatchFast); got != time.Second { + t.Errorf("initial retry=%s", got) + } + pool.noteDrainLocked(MCPDispatchFast, now) + pool.noteDrainLocked(MCPDispatchFast, now.Add(20*time.Millisecond)) + if got := pool.retryAfterLocked(MCPDispatchFast); got != 100*time.Millisecond { + t.Errorf("fast drain retry=%s", got) + } + lane := &pool.lanes[MCPDispatchFast] + lane.drainInterval = 2 * time.Second + lane.queued = 4 + if got := pool.retryAfterLocked(MCPDispatchFast); got != 5*time.Second { + t.Errorf("slow frontier retry=%s", got) + } +} + +func TestDispatchSchedulerCannotGrantExpiredQueueBudgetAfterTimerWins(t *testing.T) { + pool := testDispatchPool(1) + holder := takeScheduled(t, pool, MCPDispatchMutating, "holder") + admission := newMCPDispatchAdmission() + ctx, cancel := context.WithCancel(withMCPDispatchAdmission(withMCPDispatchQueueWait(t.Context(), 20*time.Millisecond), admission)) + defer cancel() + result := make(chan scheduledReply, 1) + go func() { + grant, err := pool.acquire(ctx, MCPDispatchFast, "reader") + result <- scheduledReply{grant, err} + }() + select { + case <-admission.done: + case <-time.After(time.Second): + t.Fatal("reader did not queue") + } + func() { + pool.owner.mu.Lock() + defer pool.owner.mu.Unlock() + lane := &pool.lanes[MCPDispatchFast] + if len(lane.sessions) != 1 { + t.Fatalf("queued sessions=%d", len(lane.sessions)) + } + expires := lane.sessions[0].requests[0].expiresAt + // Hold the admission mutex across the actual queue deadline. This lets + // the acquisition timer fire but prevents its caller from removing the + // waiter, reproducing a late capacity-release race without a sleep oracle. + timer := time.NewTimer(max(0, time.Until(expires))) + defer timer.Stop() + <-timer.C + holder.completeLocked(time.Now()) + }() + select { + case reply := <-result: + if reply.grant != nil { + reply.grant.complete() + t.Fatal("expired queued request acquired late capacity") + } + if !errors.Is(reply.err, errMCPDispatchSaturated) { + t.Fatalf("expired queue error=%v", reply.err) + } + case <-time.After(time.Second): + t.Fatal("expired queue did not terminate") + } + if len(pool.slots) != 0 { + t.Fatalf("expired queue leaked %d capacity tokens", len(pool.slots)) + } +} diff --git a/internal/daemon/mcp_dispatch_slot_lease_test.go b/internal/daemon/mcp_dispatch_slot_lease_test.go index d076739e0..57c5179c1 100644 --- a/internal/daemon/mcp_dispatch_slot_lease_test.go +++ b/internal/daemon/mcp_dispatch_slot_lease_test.go @@ -203,6 +203,8 @@ func TestSessionDispatchSlotReleaseAcrossFollowerRetryFreesOnlyItsOwnSlot(t *tes oldCtx := withMCPDispatchSlotLease(oldParent) oldStarted := make(chan struct{}) releaseOld := make(chan struct{}) + unblockOld := sync.OnceFunc(func() { close(releaseOld) }) + t.Cleanup(unblockOld) oldHandlerReturned := make(chan struct{}) oldDone := make(chan error, 1) go func() { @@ -256,27 +258,35 @@ func TestSessionDispatchSlotReleaseAcrossFollowerRetryFreesOnlyItsOwnSlot(t *tes case <-time.After(time.Second): t.Fatal("new connection did not retry the cancelled flight") } - if retried.held != 2 || !retried.returned || retried.after != 1 { - t.Fatalf("retrying follower = %+v, want its own slot of two returned", retried) + if retried.held != 1 || !retried.returned || retried.after != 0 { + t.Fatalf("retrying follower = %+v, want only its own active slot returned", retried) } if err := <-newDone; err != nil { t.Fatalf("retried flight error = %v", err) } - if got := len(pool); got != 1 { - t.Fatalf("dispatch pool holds %d slots while the orphaned owner runs, want 1", got) + if got := len(pool); got != 0 { + t.Fatalf("dispatch pool holds %d slots while the orphaned owner drains, want 0", got) } + if got := mcpSchedulerFor(pool).snapshot().Lanes["fast"].Draining; got != 1 { + t.Fatalf("orphaned owner draining count = %d, want 1", got) + } + keeper := takeScheduled(t, mcpSchedulerFor(pool), MCPDispatchFast, "unrelated-keeper") - if !ReleaseMCPDispatchSlot(oldCtx) { - t.Fatal("orphaned owner's release returned no slot") + if ReleaseMCPDispatchSlot(oldCtx) { + t.Fatal("cancelled owner's late release claimed active capacity") } - if got := len(pool); got != 0 { - t.Fatalf("dispatch pool holds %d slots after the owner's release", got) + if got := len(pool); got != 1 { + t.Fatalf("owner's stale release changed keeper's slot count to %d", got) } if ReleaseMCPDispatchSlot(newCtx) { t.Fatal("follower's second release returned a slot") } - close(releaseOld) + unblockOld() awaitMCPTestSignal(t, oldHandlerReturned, "old orphan handler did not return") + if got := len(pool); got != 1 { + t.Fatalf("orphan completion changed keeper's slot count to %d", got) + } + keeper.complete() if got := len(pool); got != 0 { t.Fatalf("dispatch pool holds %d slots after the orphan returned", got) } diff --git a/internal/daemon/mcp_dispatch_status.go b/internal/daemon/mcp_dispatch_status.go new file mode 100644 index 000000000..2612ab744 --- /dev/null +++ b/internal/daemon/mcp_dispatch_status.go @@ -0,0 +1,109 @@ +package daemon + +import ( + "time" + + "github.com/zzet/gortex/internal/viewmetrics" +) + +const ( + mcpWaitAdmitted = iota + mcpWaitRefused + mcpWaitCancelled +) + +var mcpWaitBoundsMS = [...]int64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000} +var mcpWaitBucketLabels = [...]string{"1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1000ms", "2500ms", "5000ms", "10000ms", "30000ms", "inf"} +var mcpWaitOutcomeLabels = [...]string{"admitted", "refused", "cancelled"} + +type mcpDispatchWaitStats struct { + buckets [14]uint64 + count uint64 + total time.Duration +} + +// MCPDispatchWaitHistogram uses disjoint upper-bound buckets in milliseconds. +// The final bucket is >the last bound. Zero-wait admissions are included. +type MCPDispatchWaitHistogram struct { + BoundsMS []int64 `json:"bounds_ms"` + Buckets []uint64 `json:"buckets"` + Count uint64 `json:"count"` + TotalMS int64 `json:"total_ms"` +} + +type MCPDispatchLaneStatus struct { + InFlight int `json:"in_flight"` + QueueDepth int `json:"queue_depth"` + Draining int `json:"draining"` + CancelledHeld int `json:"cancelled_held"` + Refusals uint64 `json:"refusals"` + QueueWait map[string]MCPDispatchWaitHistogram `json:"queue_wait"` +} + +// MCPDispatchStatus is a bounded, identity-free admission snapshot. Protocol +// initialization/notifications share total capacity but are exempt from tool +// reservations and waiter caps; their unbudgeted waiters are reported apart. +type MCPDispatchStatus struct { + Limit int `json:"limit"` + SlowLimit int `json:"slow_limit"` + MutationReserve int `json:"mutation_reserve"` + ReservedSlotsUsed int `json:"reserved_slots_used"` + ReservedAdmissions uint64 `json:"reserved_admissions"` + DrainingLimit int `json:"draining_limit"` + DrainingTotal int `json:"draining_total"` + ProtocolWaiters int `json:"protocol_waiters"` + Lanes map[string]MCPDispatchLaneStatus `json:"lanes"` +} + +func (s *mcpDispatchScheduler) observeWaitLocked(class MCPDispatchClass, outcome int, waited time.Duration) { + waited = max(0, waited) + lane := &s.lanes[class] + stat := &lane.wait[outcome] + bucket := len(mcpWaitBoundsMS) + for i, bound := range mcpWaitBoundsMS { + if waited <= time.Duration(bound)*time.Millisecond { + bucket = i + break + } + } + stat.buckets[bucket]++ + stat.count++ + stat.total += waited + if outcome == mcpWaitRefused { + lane.refusals++ + viewmetrics.Count(viewmetrics.MCPDispatchRefusalsTotal, class.String()) + } + viewmetrics.Count(viewmetrics.MCPDispatchQueueWaitBucketsTotal, class.String(), mcpWaitOutcomeLabels[outcome], mcpWaitBucketLabels[bucket]) + viewmetrics.Observe(viewmetrics.MCPDispatchQueueWaitSeconds, waited, class.String(), mcpWaitOutcomeLabels[outcome]) +} +func (s *mcpDispatchScheduler) observeQueuedWaitLocked(waiter *mcpDispatchWaiter, outcome int, now time.Time) { + if waiter.observed { + return + } + waiter.observed = true + s.observeWaitLocked(waiter.class, outcome, now.Sub(waiter.queuedAt)) +} +func (s *mcpDispatchScheduler) snapshot() MCPDispatchStatus { + s.owner.mu.Lock() + defer s.owner.mu.Unlock() + out := MCPDispatchStatus{Limit: cap(s.slots), SlowLimit: s.slowLimit, MutationReserve: s.reserve, ReservedSlotsUsed: min(s.reserve, s.active[MCPDispatchMutating]), ReservedAdmissions: s.reservedAdmissions, DrainingLimit: mcpDispatchDrainLimit, DrainingTotal: s.owner.draining, ProtocolWaiters: s.protocolWaiting, Lanes: make(map[string]MCPDispatchLaneStatus, 3)} + for _, class := range [...]MCPDispatchClass{MCPDispatchMutating, MCPDispatchFast, MCPDispatchSlow} { + lane := &s.lanes[class] + row := MCPDispatchLaneStatus{InFlight: s.active[class], QueueDepth: lane.queued, Draining: s.draining[class], Refusals: lane.refusals, QueueWait: make(map[string]MCPDispatchWaitHistogram, 3)} + for _, blocked := range s.owner.blocked { + if blocked.pool == s && blocked.class == class { + row.CancelledHeld++ + } + } + for outcome, label := range mcpWaitOutcomeLabels { + stat := lane.wait[outcome] + row.QueueWait[label] = MCPDispatchWaitHistogram{BoundsMS: append([]int64(nil), mcpWaitBoundsMS[:]...), Buckets: append([]uint64(nil), stat.buckets[:]...), Count: stat.count, TotalMS: stat.total.Milliseconds()} + } + out.Lanes[class.String()] = row + } + return out +} + +// MCPDispatchSnapshot is the process pool's live status, independent of index +// readiness and without catalog/store work. It does not expose session IDs. +func MCPDispatchSnapshot() MCPDispatchStatus { return mcpSchedulerFor(mcpDispatchSlots).snapshot() } diff --git a/internal/daemon/mcp_dispatch_status_test.go b/internal/daemon/mcp_dispatch_status_test.go new file mode 100644 index 000000000..fe923261d --- /dev/null +++ b/internal/daemon/mcp_dispatch_status_test.go @@ -0,0 +1,86 @@ +package daemon + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/zzet/gortex/internal/viewmetrics" +) + +func TestDispatchQueueWaitMetricBucketsMatchStatusBounds(t *testing.T) { + declared := viewmetrics.LabelVocabularies(viewmetrics.MCPDispatchQueueWaitBucketsTotal)["bucket_ms"] + if !slices.Equal(declared, mcpWaitBucketLabels[:]) { + t.Fatalf("emitted buckets=%v declared=%v", mcpWaitBucketLabels, declared) + } + for i, bound := range mcpWaitBoundsMS { + if want := fmt.Sprintf("%dms", bound); mcpWaitBucketLabels[i] != want { + t.Fatalf("bucket %d label=%q status bound=%dms", i, mcpWaitBucketLabels[i], bound) + } + } + if mcpWaitBucketLabels[len(mcpWaitBoundsMS)] != "inf" { + t.Fatal("final metric bucket must retain values above the last status bound") + } +} + +func TestDispatchStatusCountsAdmittedAndRefusedQueueWaits(t *testing.T) { + pool := testDispatchPool(1) + before := viewmetrics.Read() + holder := takeScheduled(t, pool, MCPDispatchFast, "holder") + ctx := withMCPDispatchQueueWait(t.Context(), 0) + _, err := pool.acquire(ctx, MCPDispatchFast, "refused") + if !errors.Is(err, errMCPDispatchSaturated) { + t.Fatalf("saturated result=%v", err) + } + status := pool.snapshot() + lane := status.Lanes["fast"] + if lane.InFlight != 1 || lane.QueueDepth != 0 || lane.Refusals != 1 || lane.QueueWait["admitted"].Count != 1 || lane.QueueWait["refused"].Count != 1 { + t.Fatalf("lane status=%+v", lane) + } + for _, outcome := range []string{"admitted", "refused"} { + histogram := lane.QueueWait[outcome] + total := uint64(0) + for _, count := range histogram.Buckets { + total += count + } + if total != histogram.Count || len(histogram.BoundsMS)+1 != len(histogram.Buckets) { + t.Fatalf("incomplete %s histogram=%+v", outcome, histogram) + } + } + key := viewmetrics.MCPDispatchRefusalsTotal + "{lane=fast}" + if delta := viewmetrics.Read().Counters[key] - before.Counters[key]; delta != 1 { + t.Errorf("metrics refusal delta=%d", delta) + } + lane.QueueWait["admitted"].Buckets[0] = 999 + if pool.snapshot().Lanes["fast"].QueueWait["admitted"].Buckets[0] == 999 { + t.Fatal("status exposes mutable histogram storage") + } + holder.complete() + if pool.snapshot().Lanes["fast"].InFlight != 0 { + t.Fatal("completed dispatch still in status") + } +} + +func TestDispatchStatusTracksReservedUseAcrossSharedMutators(t *testing.T) { + pool := testDispatchPool(3) + a := takeScheduled(t, pool, MCPDispatchMutating, "editor") + b := takeScheduled(t, pool, MCPDispatchMutating, "editor") + c := takeScheduled(t, pool, MCPDispatchMutating, "editor") + status := pool.snapshot() + if status.ReservedSlotsUsed != 2 || status.ReservedAdmissions != 2 || status.Lanes["mutating"].InFlight != 3 { + t.Fatalf("reserved/shared snapshot=%+v", status) + } + a.complete() + if pool.snapshot().ReservedSlotsUsed != 2 { + t.Fatal("remaining shared mutators should still occupy both reservations") + } + b.complete() + if pool.snapshot().ReservedSlotsUsed != 1 { + t.Fatal("reservation did not follow remaining mutator") + } + c.complete() + if pool.snapshot().ReservedSlotsUsed != 0 { + t.Fatal("reserved capacity leaked") + } +} diff --git a/internal/daemon/mcp_dispatch_timing.go b/internal/daemon/mcp_dispatch_timing.go new file mode 100644 index 000000000..fa7d1e2a0 --- /dev/null +++ b/internal/daemon/mcp_dispatch_timing.go @@ -0,0 +1,64 @@ +package daemon + +import ( + "context" + "time" +) + +// MCPDispatchTiming is a value copy of one request's original bounded +// admission. It is not a response/transport measurement. Releasing capacity, +// attaching another grant or later slow-lane admission cannot change it. +type MCPDispatchTiming struct { + Entered time.Time + Granted time.Time +} + +// MCPDispatchTimingForContext returns unavailable unless this context's +// existing lease witnessed both boundaries of its first admission attempt. +// Cache hits, replay followers, unbounded and non-daemon callers have no such +// observation. Reading the snapshot neither samples a clock nor resets it. +func MCPDispatchTimingForContext(ctx context.Context) (MCPDispatchTiming, bool) { + if ctx == nil || isMCPDispatchUnbounded(ctx) { + return MCPDispatchTiming{}, false + } + lease := mcpDispatchSlotLeaseForContext(ctx) + if lease == nil { + return MCPDispatchTiming{}, false + } + lease.mu.Lock() + defer lease.mu.Unlock() + value := lease.timing + return value, !value.Entered.IsZero() && !value.Granted.IsZero() +} + +// This stack token fences the first attempt without a new context, allocation +// or registry. Nil-lease primitives retain their clock-free path. Requests +// with an already allocated lease pay two clock samples on successful bounded +// admission, including reads; that overhead requires measurement. +type mcpDispatchTimingAttempt struct { + lease *mcpDispatchSlotLease +} + +func (l *mcpDispatchSlotLease) beginTiming() mcpDispatchTimingAttempt { + if l == nil { + return mcpDispatchTimingAttempt{} + } + l.mu.Lock() + defer l.mu.Unlock() + if !l.timing.Entered.IsZero() { + return mcpDispatchTimingAttempt{} + } + l.timing.Entered = time.Now() + return mcpDispatchTimingAttempt{lease: l} +} + +func (a mcpDispatchTimingAttempt) granted() { + if a.lease == nil { + return + } + a.lease.mu.Lock() + if a.lease.timing.Granted.IsZero() { + a.lease.timing.Granted = time.Now() + } + a.lease.mu.Unlock() +} diff --git a/internal/daemon/mcp_dispatch_timing_test.go b/internal/daemon/mcp_dispatch_timing_test.go new file mode 100644 index 000000000..571517098 --- /dev/null +++ b/internal/daemon/mcp_dispatch_timing_test.go @@ -0,0 +1,189 @@ +package daemon + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMCPDispatchTimingActualGrantAndLeaseReplacement(t *testing.T) { + ctx := withMCPDispatchSlots(withMCPDispatchSlotLease(t.Context()), newMCPDispatchSlots(1)) + var inside MCPDispatchTiming + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { + var ok bool + inside, ok = MCPDispatchTimingForContext(ctx) + if !ok { + return nil, context.Canceled // actual handler must see the completed grant + } + return []byte("done"), nil + }) + require.NoError(t, err) + require.False(t, inside.Granted.Before(inside.Entered)) + lease := mcpDispatchSlotLeaseForContext(ctx) + first := lease.attach(func() {}) + second := lease.attach(func() {}) + lease.detach(first) // stale detach must not replace the observation either + require.True(t, ReleaseMCPDispatchSlot(ctx)) + lease.detach(second) + _, err = dispatchMCPWithContext(ctx, func() ([]byte, error) { return nil, nil }) + require.NoError(t, err) + after, ok := MCPDispatchTimingForContext(ctx) + require.True(t, ok) + require.Equal(t, inside, after, "later admission/release/attach cannot become the original grant") + after.Entered = time.Time{} + copyValue, _ := MCPDispatchTimingForContext(ctx) + require.Equal(t, inside, copyValue) +} + +func TestMCPDispatchTimingQueuedGrantIsNotEarlyAdmissionSignal(t *testing.T) { + parent, cancel := context.WithCancel(t.Context()) + slots := newMCPDispatchSlots(1) + holder, err := mcpSchedulerFor(slots).acquire(t.Context(), MCPDispatchFast, "holder") + require.NoError(t, err) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(holder.complete) } + admission := newMCPDispatchAdmission() + ctx := withMCPDispatchQueueWait(withMCPDispatchSlots(withMCPDispatchSlotLease(withMCPDispatchAdmission(parent, admission)), slots), time.Minute) + done := make(chan error, 1) + finished := make(chan struct{}) + t.Cleanup(func() { + release() + cancel() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("owned queued dispatcher did not finish") + } + }) + go func() { + defer close(finished) + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { return []byte("done"), nil }) + done <- err + }() + select { + case <-admission.done: + case <-time.After(5 * time.Second): + t.Fatal("queue did not release the connection reader") + } + queued, available := MCPDispatchTimingForContext(ctx) + require.False(t, available, "admission.done is not the actual channel grant") + require.False(t, queued.Entered.IsZero()) + // The held slot proves queue ordering; a real later clock reading also + // proves the strict measured interval on coarse Windows clocks. + require.Eventually(t, func() bool { return time.Now().After(queued.Entered) }, 5*time.Second, time.Millisecond, + "monotonic clock did not advance while dispatch capacity was held") + release() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("queued handler did not finish after capacity release") + } + timing, available := MCPDispatchTimingForContext(ctx) + require.True(t, available) + require.True(t, timing.Granted.After(timing.Entered)) +} + +func TestMCPDispatchTimingRefusalDoesNotBorrowLaterAttempt(t *testing.T) { + slots := newMCPDispatchSlots(1) + slots <- struct{}{} + ctx := withMCPDispatchQueueWait(withMCPDispatchSlots(withMCPDispatchSlotLease(t.Context()), slots), 0) + _, err := dispatchMCPWithContext(ctx, func() ([]byte, error) { + t.Error("saturated handler must not run") + return nil, nil + }) + require.ErrorIs(t, err, errMCPDispatchSaturated) + _, ok := MCPDispatchTimingForContext(ctx) + require.False(t, ok) + <-slots + _, err = dispatchMCPWithContext(ctx, func() ([]byte, error) { return nil, nil }) + require.NoError(t, err) + _, ok = MCPDispatchTimingForContext(ctx) + require.False(t, ok, "a later grant cannot complete a refused attempt's timeline") + + slots <- struct{}{} + parent, cancel := context.WithCancel(t.Context()) + cancel() + cancelled := withMCPDispatchSlots(withMCPDispatchSlotLease(parent), slots) + _, err = dispatchMCPWithContext(cancelled, func() ([]byte, error) { + t.Error("cancelled queued handler must not run") + return nil, nil + }) + require.ErrorIs(t, err, context.Canceled) + _, ok = MCPDispatchTimingForContext(cancelled) + require.False(t, ok, "a cancelled queue has no invented grant") + <-slots +} + +func TestMCPDispatchTimingReplayAndUnboundedRemainUnavailable(t *testing.T) { + sess := &Session{LogicalSessionID: "timing-session"} + request := []byte(`{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"write_file"}}`) + first := withMCPDispatchSlots(withMCPDispatchSlotLease(t.Context()), newMCPDispatchSlots(1)) + _, replayed, err := sess.dispatchMCPOnceContext(first, request, func() ([]byte, error) { + return []byte(`{"jsonrpc":"2.0","id":1,"result":{}}`), nil + }) + require.NoError(t, err) + require.False(t, replayed) + _, ok := MCPDispatchTimingForContext(first) + require.True(t, ok) + replay := withMCPDispatchSlotLease(t.Context()) + _, replayed, err = sess.dispatchMCPOnceContext(replay, request, func() ([]byte, error) { + t.Error("cached response must not execute") + return nil, nil + }) + require.NoError(t, err) + require.True(t, replayed) + _, ok = MCPDispatchTimingForContext(replay) + require.False(t, ok) + unbounded := withMCPDispatchUnbounded(withMCPDispatchSlotLease(t.Context())) + _, err = dispatchMCPWithContext(unbounded, func() ([]byte, error) { return nil, nil }) + require.NoError(t, err) + _, ok = MCPDispatchTimingForContext(unbounded) + require.False(t, ok) + _, ok = MCPDispatchTimingForContext(withMCPDispatchUnbounded(first)) + require.False(t, ok, "an unbounded descendant cannot advertise its parent's bounded grant") +} + +func TestMCPDispatchTimingNilPrimitiveDoesNotAllocate(t *testing.T) { + var lease *mcpDispatchSlotLease + ctx := t.Context() + allocations := testing.AllocsPerRun(100, func() { + lease.beginTiming().granted() + _, _ = MCPDispatchTimingForContext(ctx) + }) + require.Zero(t, allocations) + _, ok := MCPDispatchTimingForContext(nil) //nolint:staticcheck // Intentional nil-context abstention control. + require.False(t, ok) +} + +// Primitive lower bound only: this isolates the two clock samples and existing +// lease mutexes, not JSON dispatch, queueing, syscall or end-to-end read cost. +func BenchmarkMCPDispatchTimingLease(b *testing.B) { + b.Run("nil-lease", func(b *testing.B) { + var lease *mcpDispatchSlotLease + b.ReportAllocs() + for b.Loop() { + lease.beginTiming().granted() + } + }) + b.Run("lease-mutex-control", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + lease := mcpDispatchSlotLease{} + lease.mu.Lock() + lease.mu.Unlock() //nolint:staticcheck // Intentional empty lock baseline isolates existing lease mutex costs. + lease.mu.Lock() + lease.mu.Unlock() //nolint:staticcheck // Intentional empty lock baseline isolates existing lease mutex costs. + } + }) + b.Run("observed-lease", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + lease := mcpDispatchSlotLease{} + lease.beginTiming().granted() + } + }) +} diff --git a/internal/daemon/mcp_request_identity.go b/internal/daemon/mcp_request_identity.go new file mode 100644 index 000000000..629626c41 --- /dev/null +++ b/internal/daemon/mcp_request_identity.go @@ -0,0 +1,21 @@ +package daemon + +import "context" + +// MCPRequestIDKeyForContext returns the canonical wire-ID key already validated +// by this connection's request registry. Embedded and untracked callers have +// no such observation. Reading it adds no clock, parsing, allocation or registry. +// +// A client can reuse an ID after a request finishes. This is an attribution +// cause, not a publisher identity: pair it with the actual session and admitted +// coordinator generation when relating work to a request. +func MCPRequestIDKeyForContext(ctx context.Context) (string, bool) { + if ctx == nil { + return "", false + } + lease := mcpDispatchSlotLeaseForContext(ctx) + if lease == nil || lease.requestIDKey == "" { + return "", false + } + return lease.requestIDKey, true +} diff --git a/internal/daemon/mcp_request_identity_test.go b/internal/daemon/mcp_request_identity_test.go new file mode 100644 index 000000000..4aa36f3de --- /dev/null +++ b/internal/daemon/mcp_request_identity_test.go @@ -0,0 +1,70 @@ +package daemon + +import ( + "context" + "encoding/json" + "testing" + "time" +) + +func TestMCPRequestIDKeyFollowsActualWireTypes(t *testing.T) { + conn, reader, _ := startMCPConnectionTest(t, time.Second, func(ctx context.Context, frame []byte) ([]byte, error) { + var envelope mcpRequestEnvelope + if err := json.Unmarshal(frame, &envelope); err != nil { + return nil, err + } + key, available := MCPRequestIDKeyForContext(ctx) + return json.Marshal(map[string]any{"jsonrpc": "2.0", "id": envelope.ID, "result": map[string]any{"key": key, "available": available}}) + }) + for _, item := range []struct{ frame, key string }{ + {`{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"read"}}`, "n:7"}, + {`{"jsonrpc":"2.0","id":"7","method":"tools/call","params":{"name":"read"}}`, "s:7"}, + } { + writeMCPTestFrame(t, conn, item.frame) + response := readMCPTestResponse(t, conn, reader) + if response.Error != nil || response.Result["key"] != item.key || response.Result["available"] != true { + t.Fatalf("wire identity: got %+v, want %s", response, item.key) + } + } +} + +func TestMCPRequestIDKeyIsNotAPublisherIdentity(t *testing.T) { + first := withMCPDispatchSlotLeaseForRequest(t.Context(), "n:7") + second := withMCPDispatchSlotLeaseForRequest(t.Context(), "n:7") + if mcpDispatchSlotLeaseForContext(first) == mcpDispatchSlotLeaseForContext(second) { + t.Fatal("separate request leases were shared") + } + firstKey, firstOK := MCPRequestIDKeyForContext(first) + secondKey, secondOK := MCPRequestIDKeyForContext(second) + if !firstOK || !secondOK || firstKey != secondKey { + t.Fatal("reused wire key was mistaken for a unique physical owner") + } +} + +func TestMCPRequestIDKeySurvivesGrantReplacementAndCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(withMCPDispatchSlotLeaseForRequest(t.Context(), "s:read-source")) + defer cancel() + lease := mcpDispatchSlotLeaseForContext(ctx) + first := lease.attach(func() {}) + second := lease.attach(func() {}) + lease.detach(first) + if !ReleaseMCPDispatchSlot(ctx) { + t.Fatal("current grant was not released") + } + lease.detach(second) + cancel() + if key, available := MCPRequestIDKeyForContext(ctx); !available || key != "s:read-source" { + t.Fatalf("request identity replaced by grant lifetime: %q %v", key, available) + } + if got := testing.AllocsPerRun(100, func() { _, _ = MCPRequestIDKeyForContext(ctx) }); got != 0 { + t.Fatalf("identity lookup allocates: %v", got) + } +} + +func TestMCPRequestIDKeyIsUnavailableWithoutTrackedRequest(t *testing.T) { + for _, ctx := range []context.Context{nil, context.Background(), withMCPDispatchSlotLease(t.Context())} { + if key, available := MCPRequestIDKeyForContext(ctx); available || key != "" { + t.Fatalf("untracked request borrowed identity: %q %v", key, available) + } + } +} diff --git a/internal/daemon/mcp_request_lifetime.go b/internal/daemon/mcp_request_lifetime.go index 0188575bb..661fbbf40 100644 --- a/internal/daemon/mcp_request_lifetime.go +++ b/internal/daemon/mcp_request_lifetime.go @@ -13,7 +13,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "time" ) @@ -31,6 +30,8 @@ type mcpRequestEnvelope struct { Method string `json:"method"` Params struct { RequestID json.RawMessage `json:"requestId"` + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` } `json:"params"` } @@ -230,28 +231,27 @@ func mcpDispatchQueueWait(raw string) time.Duration { var mcpDispatchQueueWaitDefault = mcpDispatchQueueWait(os.Getenv(mcpDispatchQueueWaitEnv)) -// mcpDispatchQueueWaiterCounts maps each dispatch pool to the number of -// requests waiting for one of its slots. Pools are few and never removed. -var mcpDispatchQueueWaiterCounts sync.Map // chan struct{} -> *atomic.Int32 - -func mcpDispatchQueueWaiterCount(slots chan struct{}) *atomic.Int32 { - waiters, _ := mcpDispatchQueueWaiterCounts.LoadOrStore(slots, new(atomic.Int32)) - return waiters.(*atomic.Int32) -} - +// mcpDispatchQueueWaiters retains the existing isolated-pool census seam. func mcpDispatchQueueWaiters(slots chan struct{}) int { - waiters, ok := mcpDispatchQueueWaiterCounts.Load(slots) - if !ok { - return 0 + pool := mcpSchedulerFor(slots) + pool.owner.mu.Lock() + defer pool.owner.mu.Unlock() + count := 0 + for i := range pool.lanes { + count += pool.lanes[i].queued } - return int(waiters.(*atomic.Int32).Load()) + return count } var errMCPDispatchSaturated = errors.New("MCP dispatcher capacity is saturated") // mcpDispatchBusyError is a busy refusal from the dispatch queue: the budget // ran out, or the queue was full and the request did not wait. -type mcpDispatchBusyError struct{ waited time.Duration } +type mcpDispatchBusyError struct { + waited time.Duration + retryAfter time.Duration + lane MCPDispatchClass +} func (e *mcpDispatchBusyError) Error() string { return errMCPDispatchSaturated.Error() } func (e *mcpDispatchBusyError) Unwrap() error { return errMCPDispatchSaturated } @@ -274,18 +274,27 @@ type mcpDispatchSlotsContextKey struct{} type mcpDispatchQueueWaitContextKey struct{} type mcpDispatchUnboundedContextKey struct{} type mcpDispatchSlotLeaseContextKey struct{} +type mcpDispatchCompletionOwnerContextKey struct{} + +type mcpDispatchCompletionOwner struct { + lease *mcpDispatchSlotLease + grant *mcpDispatchSlotGrant +} // mcpDispatchSlotLease lets the handler running a dispatch give its slot back // before it returns. type mcpDispatchSlotLease struct { - mu sync.Mutex - grant *mcpDispatchSlotGrant // the attached dispatch's release; nil when none + mu sync.Mutex + grant *mcpDispatchSlotGrant // the attached dispatch's release; nil when none + timing MCPDispatchTiming // immutable original admission, guarded by mu + requestIDKey string // immutable validated wire key, assigned before dispatch } // mcpDispatchSlotGrant carries one dispatch's slot release. Its address // identifies that dispatch, so a stale detach cannot clear a later attach. type mcpDispatchSlotGrant struct { - release func() + release func() + scheduled *mcpScheduledGrant // guarded by the owning lease's mutex } func isLongLivedMCPRequest(method string) bool { @@ -327,7 +336,11 @@ func mcpDispatchSlotsForContext(ctx context.Context) chan struct{} { // withMCPDispatchSlotLease gives one request an empty lease. A dispatch that // takes a slot attaches its release to the lease while its handler runs. func withMCPDispatchSlotLease(ctx context.Context) context.Context { - return context.WithValue(ctx, mcpDispatchSlotLeaseContextKey{}, &mcpDispatchSlotLease{}) + return withMCPDispatchSlotLeaseForRequest(ctx, "") +} + +func withMCPDispatchSlotLeaseForRequest(ctx context.Context, requestIDKey string) context.Context { + return context.WithValue(ctx, mcpDispatchSlotLeaseContextKey{}, &mcpDispatchSlotLease{requestIDKey: requestIDKey}) } func mcpDispatchSlotLeaseForContext(ctx context.Context) *mcpDispatchSlotLease { @@ -374,14 +387,145 @@ func ReleaseMCPDispatchSlot(ctx context.Context) bool { lease.mu.Lock() grant := lease.grant lease.grant = nil + var scheduled *mcpScheduledGrant + if grant != nil { + scheduled = grant.scheduled + } lease.mu.Unlock() if grant == nil { return false } + if scheduled != nil { + return scheduled.releaseEarly() + } grant.release() return true } +// MoveMCPDispatchToSlowLane returns mutation capacity before joining the +// bounded slow lane for a passive post-commit freshness wait. done closes when +// that wait is over, and a wait that is over needs no slow slot: one already +// over keeps its capacity for the handler's last steps, and a slow admission +// still queued when it ends is withdrawn and reported as success without +// capacity. On refusal or cancellation the committed mutation must report its +// existing pending receipt, not a retryable busy response. Outside daemon +// dispatch this is a no-op. +func MoveMCPDispatchToSlowLane(ctx context.Context, done <-chan struct{}) error { + lease := mcpDispatchSlotLeaseForContext(ctx) + if lease == nil || closedMCPSignal(done) { + return nil + } + lease.mu.Lock() + grant := lease.grant + if grant == nil || grant.scheduled == nil { + lease.mu.Unlock() + return nil + } + old := grant.scheduled + if old.class == MCPDispatchSlow { + lease.mu.Unlock() + return nil + } + released := old.releaseEarly() + lease.mu.Unlock() + if !released { + return fmt.Errorf("%w: dispatch no longer owns mutation capacity", errMCPDispatchNotStarted) + } + // Never hold the lease mutex through admission: caller cancellation must + // still release/cancel the current lease without waiting for this queue. + // The queued admission alone also stops when the wait ends; the caller's + // own context still decides whether an admitted grant is cancelled below. + admission, withdraw := context.WithCancel(ctx) + defer withdraw() + if done != nil { + go func() { + select { + case <-done: + withdraw() + case <-admission.Done(): + } + }() + } + next, err := old.pool.acquire(admission, MCPDispatchSlow, old.session) + if err != nil { + if ctx.Err() == nil && closedMCPSignal(done) { + return nil + } + return err + } + lease.mu.Lock() + if lease.grant != grant { + lease.mu.Unlock() + next.complete() + return fmt.Errorf("%w: dispatch lease changed during slow admission", errMCPDispatchNotStarted) + } + grant.scheduled = next + lease.mu.Unlock() + if ctx.Err() != nil { + next.cancel() + } + return nil +} + +// closedMCPSignal reports whether signal has closed; a nil signal never does. +func closedMCPSignal(signal <-chan struct{}) bool { + select { + case <-signal: + return true + default: + return false + } +} + +func (l *mcpDispatchSlotLease) scheduledGrant(grant *mcpDispatchSlotGrant, fallback *mcpScheduledGrant) *mcpScheduledGrant { + if l == nil || grant == nil { + return fallback + } + l.mu.Lock() + defer l.mu.Unlock() + return grant.scheduled +} + +// WithMCPDispatchCompletionOwner captures the dispatch identity before a +// middleware firewall forks its handler. Nested firewalls inherit that identity +// even after the outer dispatcher returns and detaches its live lease. A later +// attach can never adopt the abandoned handler's completion. +func WithMCPDispatchCompletionOwner(ctx context.Context) context.Context { + if _, ok := ctx.Value(mcpDispatchCompletionOwnerContextKey{}).(mcpDispatchCompletionOwner); ok { + return ctx + } + lease := mcpDispatchSlotLeaseForContext(ctx) + if lease == nil { + return ctx + } + lease.mu.Lock() + grant := lease.grant + lease.mu.Unlock() + if grant == nil { + return ctx + } + return context.WithValue(ctx, mcpDispatchCompletionOwnerContextKey{}, mcpDispatchCompletionOwner{lease: lease, grant: grant}) +} + +// RetainMCPDispatchCompletion joins an abandoned inner handler to its captured +// dispatch's physical drain lifetime. A middleware firewall calls this before +// returning without its handler, then calls the returned idempotent release +// only when that handler actually returns. It releases logical admission but +// preserves the process drain bound. Nil means no bounded dispatch remains; +// an explicitly released passive wait must never reacquire its old capacity. +func RetainMCPDispatchCompletion(ctx context.Context) func() { + owner, ok := ctx.Value(mcpDispatchCompletionOwnerContextKey{}).(mcpDispatchCompletionOwner) + if !ok { + return nil + } + owner.lease.mu.Lock() + defer owner.lease.mu.Unlock() + if owner.grant.scheduled == nil { + return nil + } + return owner.grant.scheduled.retainCompletion() +} + func withMCPDispatchQueueWait(ctx context.Context, wait time.Duration) context.Context { return context.WithValue(ctx, mcpDispatchQueueWaitContextKey{}, wait) } @@ -408,89 +552,59 @@ func dispatchMCPWithContext(ctx context.Context, dispatch func() ([]byte, error) signalMCPDispatchAdmission(ctx) return dispatch() } - slots := mcpDispatchSlotsForContext(ctx) - if wait, _ := ctx.Value(mcpDispatchWaitContextKey{}).(bool); wait { - select { - case slots <- struct{}{}: - case <-ctx.Done(): - signalMCPDispatchAdmission(ctx) - return nil, ctx.Err() - } + lease := mcpDispatchSlotLeaseForContext(ctx) + timing := lease.beginTiming() + pool := mcpSchedulerFor(mcpDispatchSlotsForContext(ctx)) + info := mcpDispatchRequestForContext(ctx) + var scheduled *mcpScheduledGrant + var err error + wait, _ := ctx.Value(mcpDispatchWaitContextKey{}).(bool) + if wait || info.protocol { + scheduled, err = pool.acquireProtocol(ctx) } else { - select { - case slots <- struct{}{}: - default: - if err := awaitMCPDispatchSlot(ctx, slots); err != nil { - return nil, err - } - } + scheduled, err = pool.acquire(ctx, info.class, info.session) + } + if err != nil { + return nil, err + } + timing.granted() + if err := ctx.Err(); err != nil { + scheduled.complete() + signalMCPDispatchAdmission(ctx) + return nil, fmt.Errorf("%w: %w", errMCPDispatchNotStarted, err) } - result := make(chan mcpDispatchResult, 1) started := make(chan struct{}) - lease := mcpDispatchSlotLeaseForContext(ctx) + var grant *mcpDispatchSlotGrant go func() { - var releaseOnce sync.Once - release := func() { releaseOnce.Do(func() { <-slots }) } - defer release() // Retain the slot until an uncooperative handler returns, including panic unwinding, unless the handler hands it back early through ReleaseMCPDispatchSlot. - grant := lease.attach(release) - defer lease.detach(grant) // Registered after release, so it runs first while unwinding. + grant = lease.attach(func() { scheduled.releaseEarly() }) + if grant != nil { + lease.mu.Lock() + grant.scheduled = scheduled + lease.mu.Unlock() + } + complete := func() { lease.scheduledGrant(grant, scheduled).complete() } + defer complete() + defer lease.detach(grant) close(started) reply, err := dispatch() - lease.detach(grant) // No release handle outlives the response. - release() // A terminal response must not overtake its dispatch capacity release. + lease.detach(grant) + complete() // A terminal reply cannot overtake its capacity release. result <- mcpDispatchResult{reply: reply, err: err} }() <-started signalMCPDispatchAdmission(ctx) - select { case completed := <-result: return completed.reply, completed.err case <-ctx.Done(): + // Synchronously return lane capacity (or retain its charge only when the + // process-wide drain allowance is full) before publishing cancellation. + lease.scheduledGrant(grant, scheduled).cancel() return nil, ctx.Err() } } -// awaitMCPDispatchSlot waits for a slot of a saturated pool for at most the -// queue budget. Blocked senders are served in arrival order. The slot, the -// budget and cancellation share one select, so a request that takes a slot -// never also reports busy or not started. -func awaitMCPDispatchSlot(ctx context.Context, slots chan struct{}) error { - wait := mcpDispatchQueueWaitForContext(ctx) - if wait <= 0 { - signalMCPDispatchAdmission(ctx) - return errMCPDispatchSaturated - } - waiters := mcpDispatchQueueWaiterCount(slots) - waiterCap := int32(mcpDispatchQueueFactor * cap(slots)) - for { - queued := waiters.Load() - if queued >= waiterCap { - signalMCPDispatchAdmission(ctx) - return &mcpDispatchBusyError{} - } - if waiters.CompareAndSwap(queued, queued+1) { - break - } - } - defer waiters.Add(-1) - // Count the waiter before freeing the reader so the next frame observes it. - signalMCPDispatchAdmission(ctx) - - queuedAt := time.Now() - budget := time.NewTimer(wait) - defer budget.Stop() - select { - case slots <- struct{}{}: - return nil - case <-budget.C: - return &mcpDispatchBusyError{waited: time.Since(queuedAt)} - case <-ctx.Done(): - return fmt.Errorf("%w: %w", errMCPDispatchNotStarted, ctx.Err()) - } -} - func (s *Server) mcpToolCallTimeout() time.Duration { if s != nil && s.MCPToolCallTimeout > 0 { return s.MCPToolCallTimeout @@ -505,6 +619,8 @@ func serveMCPConnection(conn net.Conn, reader *bufio.Reader, timeout time.Durati type mcpConnectionHooks struct { beforeResponse func(json.RawMessage) dispatchSlots chan struct{} + sessionID string + classify MCPDispatchClassifier // dispatchQueueWait, if set, overrides the dispatch queue budget for // this connection; zero disables the queue. dispatchQueueWait *time.Duration @@ -521,6 +637,14 @@ func serveMCPConnectionWithHooks(conn net.Conn, reader *bufio.Reader, timeout ti timeout = DefaultMCPToolCallTimeout } connectionCtx, connectionCancel := context.WithCancel(context.Background()) + sessionID := "connection-" + strconv.FormatUint(mcpDispatchConnectionSequence.Add(1), 10) + var classifier MCPDispatchClassifier + if hooks != nil { + if hooks.sessionID != "" { + sessionID = hooks.sessionID + } + classifier = hooks.classify + } if hooks != nil && hooks.dispatchSlots != nil { connectionCtx = withMCPDispatchSlots(connectionCtx, hooks.dispatchSlots) } @@ -570,7 +694,7 @@ func serveMCPConnectionWithHooks(conn net.Conn, reader *bufio.Reader, timeout ti // The supplied dispatcher owns the single process-wide admission. // Notifications wait for that slot under the connection context so // state transitions are not silently dropped under load. - notificationCtx := withMCPDispatchWait(connectionCtx) + notificationCtx := withMCPDispatchWait(withMCPDispatchRequest(connectionCtx, mcpDispatchRequestInfo{session: sessionID, protocol: true})) _, _ = dispatch(notificationCtx, line) }() return done @@ -629,7 +753,8 @@ func serveMCPConnectionWithHooks(conn net.Conn, reader *bufio.Reader, timeout ti requestLine := append([]byte(nil), line...) requestID := append(json.RawMessage(nil), responseID...) admission := newMCPDispatchAdmission() - dispatchCtx := withMCPDispatchSlotLease(withMCPDispatchAdmission(requestCtx, admission)) + requestCtx = prepareMCPDispatchContext(requestCtx, sessionID, envelope, classifier) + dispatchCtx := withMCPDispatchSlotLeaseForRequest(withMCPDispatchAdmission(requestCtx, admission), entry.key) requests.Add(1) go func() { defer requests.Done() @@ -691,19 +816,7 @@ func serveMCPConnectionWithHooks(conn net.Conn, reader *bufio.Reader, timeout ti } reply = mcpErrorResponse(requestID, code, message, data) } else if errors.Is(dispatchErr, errMCPDispatchSaturated) { - data := map[string]any{ - "outcome": "server_busy", - "retryable": true, - } - if envelope.Method == "tools/call" { - data["delivery_unknown"] = false - } - var busy *mcpDispatchBusyError - if errors.As(dispatchErr, &busy) { - data["queue_wait_ms"] = busy.waited.Milliseconds() - data["retry_after_ms"] = mcpDispatchBusyRetryAfter.Milliseconds() - } - reply = mcpErrorResponse(requestID, -32002, "MCP dispatcher is busy", data) + reply = mcpDispatchBusyResponse(requestID, envelope.Method, dispatchErr) } else if dispatchErr != nil { reply = mcpErrorResponse(requestID, -32603, "request dispatch failed", map[string]any{ "outcome": "dispatch_error", diff --git a/internal/daemon/mcp_request_lifetime_regression_test.go b/internal/daemon/mcp_request_lifetime_regression_test.go index a133e773a..7d7c519b1 100644 --- a/internal/daemon/mcp_request_lifetime_regression_test.go +++ b/internal/daemon/mcp_request_lifetime_regression_test.go @@ -424,10 +424,14 @@ saturated: if response.Error == nil || response.Error.Code != -32002 || response.Error.Data["outcome"] != "server_busy" { t.Fatalf("busy response %s = %#v", id, response) } - for _, field := range []string{"queue_wait_ms", "retry_after_ms"} { - if value, present := response.Error.Data[field]; present { - t.Fatalf("busy response %s reported %s = %#v with the queue disabled", id, field, value) - } + // Lane admission supplies bounded retry guidance even with queueing off; + // this remains a not-started refusal and does not relax lifetime safety. + retry, ok := response.Error.Data["retry_after_ms"].(float64) + if !ok || retry < 100 || retry > 5000 { + t.Fatalf("busy response %s retry=%#v", id, response.Error.Data["retry_after_ms"]) + } + if waited, ok := response.Error.Data["queue_wait_ms"].(float64); !ok || waited != 0 { + t.Fatalf("queue-off response %s wait=%#v", id, response.Error.Data["queue_wait_ms"]) } } if responses["1"].Error.Data["delivery_unknown"] != false || responses["1"].Error.Data["retryable"] != true { diff --git a/internal/daemon/proto.go b/internal/daemon/proto.go index d80f1ea03..0350841b7 100644 --- a/internal/daemon/proto.go +++ b/internal/daemon/proto.go @@ -323,6 +323,9 @@ const ( // RepoStateLoading: configured, and the daemon has not registered it // since it started. Graph calls answer repo_loading until it is ready. RepoStateLoading = "loading" + // RepoStateIdentityPending means configured identities are unresolved; it + // does not claim that the requested path belongs to a repository. + RepoStateIdentityPending = "identity_pending" // RepoStateReady: registered; graph calls answer. RepoStateReady = "ready" // RepoStateFailed: its load ended without registering it. @@ -334,6 +337,7 @@ const ( // StatusResponse is the payload returned under Result on a successful // ControlStatus call. type StatusResponse struct { + MCPDispatch *MCPDispatchStatus `json:"mcp_dispatch,omitempty"` Version string `json:"version"` PID int `json:"pid"` UptimeSeconds int64 `json:"uptime_seconds"` @@ -551,6 +555,15 @@ type RowCounterStatus struct { // the time the read gate was closed for a reset; ReaderWait is the time new // reads spent waiting at it. type WALReclaimStatus struct { + // Same since-store-open observations exposed by graph_stats. The peak is + // a sampled lower bound and remains null until actually observed; reads + // and WAL resets never clear it or the cancellation counters. + CancelledWriterParked int64 `json:"cancelled_writer_parked"` + CancelledWriteIntent int64 `json:"cancelled_write_intent"` + CancelledOther int64 `json:"cancelled_other"` + ObservedPeakWALBytes *int64 `json:"observed_peak_wal_bytes"` + WALPeakObserved bool `json:"wal_peak_observed"` + ThresholdBytes int64 `json:"threshold_bytes"` Attempts int64 `json:"attempts"` Resets int64 `json:"resets"` @@ -1024,6 +1037,8 @@ type FileCoverageResult struct { // Graph calls for such a repo answer repo_loading, so a coverage verdict // read from its persisted rows must not turn into a deny of the native // tool: the agent would have no read path at all. Repo names it. + // RepoStateIdentityPending instead reports unresolved configured identities + // without asserting ownership of Path; Repo is empty and no graph is read. RepoState string `json:"repo_state,omitempty"` Repo string `json:"repo,omitempty"` // MutationState is read_only while the current checkout's exact route has diff --git a/internal/daemon/server.go b/internal/daemon/server.go index b4aabf44e..e45815cc1 100644 --- a/internal/daemon/server.go +++ b/internal/daemon/server.go @@ -652,7 +652,10 @@ func (s *Server) serveMCP(conn net.Conn, reader *bufio.Reader, sess *Session) { return } + classifier, _ := s.MCPDispatcher.(MCPDispatchClassifier) hooks := &mcpConnectionHooks{ + sessionID: sess.ID, + classify: classifier, onReady: func(write func([]byte) error) { if hook, ok := s.MCPDispatcher.(SessionStartedHook); ok && hook != nil { hook.SessionStarted(sess, write) @@ -660,13 +663,7 @@ func (s *Server) serveMCP(conn net.Conn, reader *bufio.Reader, sess *Session) { }, } serveMCPConnectionWithHooks(conn, reader, s.mcpToolCallTimeout(), func(ctx context.Context, line []byte) ([]byte, error) { - reply, _, err := sess.dispatchMCPOnceContext(ctx, line, func() ([]byte, error) { - reply, err := s.MCPDispatcher.Dispatch(ctx, sess, line) - if ctxErr := ctx.Err(); ctxErr != nil && (!isMCPDispatchUnbounded(ctx) || err != nil || len(reply) == 0) { - return nil, ctxErr - } - return reply, err - }) + reply, err := DispatchMCPFrame(ctx, sess, line, s.MCPDispatcher) if err != nil && s.Logger != nil { if ctxErr := ctx.Err(); ctxErr != nil { outcome := "cancelled" @@ -887,6 +884,8 @@ func (s *Server) handleControl(ctx context.Context, _ *Session, req ControlReque } // Binary-drift self-report: did the on-disk daemon image get // replaced under this running process? See populateBinaryStatus. + dispatch := MCPDispatchSnapshot() + st.MCPDispatch = &dispatch s.populateBinaryStatus(&st) buf, _ := json.Marshal(st) return ControlResponse{OK: true, Result: buf} diff --git a/internal/graph/graph.go b/internal/graph/graph.go index d39ebbc10..3f930448b 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -465,10 +465,8 @@ func removeEdgeFromBucket(bucket map[string][]*Edge, keys map[string][]edgeHash, // constructed against this Graph. Per-shard locks protect individual // node/edge writes, but resolution phases (ResolveAll, ResolveFile) // iterate every shard and mutate edge targets in place — two -// resolvers racing on the same edge produced the data race that -// MultiIndexer.indexMultiRepo triggered when its per-repo goroutines -// each created their own Resolver. Routing every resolver through -// this single mutex serialises those phases without blocking +// per-repository resolvers can race on the same shared edge. Routing every +// resolver through this single mutex serialises those phases without blocking // ordinary shard-scoped reads and writes. type Graph struct { // shards holds the lock-sharded node/edge maps. Its length is diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index fe5ebad36..3fec85127 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -46,11 +46,12 @@ import ( type generationBulkCheckpointLease uint64 type backgroundCheckpointAttempt struct { - ctx context.Context - cancel context.CancelCauseFunc - timeoutCancel context.CancelFunc - done chan struct{} - finishOnce sync.Once + reclaimInterruption atomic.Uint32 // first consumed cause; counted once by reclaimWALOnce + ctx context.Context + cancel context.CancelCauseFunc + timeoutCancel context.CancelFunc + done chan struct{} + finishOnce sync.Once // copy is the attempt's paced copy, set before the lane watcher starts // (its passes pause only once the attempt marks it pausable). copy *walCopyAttempt @@ -1100,6 +1101,10 @@ var errGenerationBulkCheckpointCoordination = errors.New("store_sqlite: generati // cancelled with that cause when a cycle takes the lane (see // checkpoint_cycle_yield.go). func (s *Store) beginBackgroundCheckpointAttempt(policy checkpointCyclePolicy) (*backgroundCheckpointAttempt, error) { + return s.beginCheckpointAttempt(policy, false) +} + +func (s *Store) beginCheckpointAttempt(policy checkpointCyclePolicy, reclaim bool) (*backgroundCheckpointAttempt, error) { yieldPolicy := policy == checkpointYieldsToCycle || policy == checkpointOverridesLease yields := yieldPolicy && s.cycleYieldEnabled() overridesLease := policy == checkpointOverridesLease || policy == checkpointOverridesLeaseAndCycle @@ -1119,6 +1124,10 @@ func (s *Store) beginBackgroundCheckpointAttempt(policy checkpointCyclePolicy) ( copy: &walCopyAttempt{}, } + if reclaim { + attempt.ctx = context.WithValue(ctx, walReclaimAttemptKey{}, attempt) + } + coordination := &s.backgroundCheckpoint coordination.mu.Lock() if (coordination.generationLease != 0 && !overridesLease) || coordination.active != nil { diff --git a/internal/graph/store_sqlite/store_compact.go b/internal/graph/store_sqlite/store_compact.go index a114768ab..e1a6959ab 100644 --- a/internal/graph/store_sqlite/store_compact.go +++ b/internal/graph/store_sqlite/store_compact.go @@ -27,7 +27,7 @@ import ( // Store.PublishPayloadGeneration, whose tail schedules the // planner-statistics refresh the publish owes; the direct // publish+route API asks below its own route flip instead. -// VACUUM cmd/gortex/daemon_state.go -> maybeCompactStore -> Compact. +// VACUUM cmd/gortex's owned post-warmup task -> CompactContext. // TRUNCATE internal/indexer/multi.go's read boundary -> CheckpointWAL, // and Compact's own tail through the ungated inner path. // @@ -111,8 +111,7 @@ func (j maintenanceJob) priority() bool { return !j.preemptible() } // token itself. Generous on purpose — the point of the wait is that // maintenance yields to real work, and a job that gives up simply runs at the // next boundary — but finite in every direction, because Compact runs on the -// daemon's warmup path with context.Background() and must never be able to -// stall a boot behind another job. +// daemon's post-warmup task and must yield when maintenance remains busy. // // A var, not a const, only so the in-package cases can shorten the budget // instead of spending it; production never assigns it. @@ -810,7 +809,17 @@ func (s *Store) CompactStats() (freeBytes, totalBytes int64) { // same lane hold, through the ungated inner path, because re-entering the lane // from inside it would deadlock on its own token. func (s *Store) Compact() error { - return s.runMaintenance(context.Background(), maintenanceVacuum, true, func(ctx context.Context) error { + return s.CompactContext(context.Background()) +} + +// CompactContext retains Compact's maintenance admission and actual completion +// boundary while allowing its owner to cancel lane waits and the VACUUM SQL. +// Returning an error never certifies that the file was compacted. +func (s *Store) CompactContext(ctx context.Context) error { + if ctx == nil { + return errors.New("compact: nil context") + } + return s.runMaintenance(ctx, maintenanceVacuum, true, func(ctx context.Context) error { if err := s.vacuum(ctx); err != nil { return err } @@ -833,8 +842,14 @@ func (s *Store) vacuum(ctx context.Context) error { if s.bulkConn != nil { return fmt.Errorf("%w: %s: bulk writer pinned", ErrMaintenanceBusy, maintenanceVacuum) } - _, err := s.writerDB.ExecContext(ctx, `VACUUM`) - return err + if isMemoryPath(s.dbPath) { + _, err := s.writerDB.ExecContext(ctx, `VACUUM`) + return err + } + return s.withVacuumConnection(ctx, func(conn *sql.Conn) error { + _, err := conn.ExecContext(ctx, `VACUUM`) + return err + }) } // errWALResidueHandedOff reports that a residue drain found the log held by a diff --git a/internal/graph/store_sqlite/store_dbstat.go b/internal/graph/store_sqlite/store_dbstat.go index 5c5bbffdc..c2c5774a1 100644 --- a/internal/graph/store_sqlite/store_dbstat.go +++ b/internal/graph/store_sqlite/store_dbstat.go @@ -16,6 +16,9 @@ func (s *Store) DBStats() (dbBytes, walBytes int64) { } if fi, err := os.Stat(s.dbPath + "-wal"); err == nil { walBytes = fi.Size() + s.noteObservedWALBytes(walBytes) + } else if os.IsNotExist(err) { + s.noteObservedWALBytes(0) } return dbBytes, walBytes } diff --git a/internal/graph/store_sqlite/vacuum_connection.go b/internal/graph/store_sqlite/vacuum_connection.go new file mode 100644 index 000000000..d09bf6a25 --- /dev/null +++ b/internal/graph/store_sqlite/vacuum_connection.go @@ -0,0 +1,120 @@ +package store_sqlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" +) + +// SQLite's deprecated temp_store_directory pragma changes a process-global +// pointer. Serialize our dedicated FILE connections, including restoration and +// physical close. Ordinary store connections retain temp_store(MEMORY). +// This is separate from each store's write gate: two open stores can compact +// concurrently otherwise. Waiting for this guard remains cancelable. +var vacuumTempDirectoryGate sqliteWriteGate + +// ErrVacuumTempDirectory means no VACUUM ran because its scratch volume could +// not be confirmed. Callers must skip instead of assuming the system temp +// volume has the headroom checked for the store directory. +var ErrVacuumTempDirectory = errors.New("sqlite vacuum: temporary directory unconfirmed") + +func sqliteVacuumDSN(path string) string { + return strings.Replace(sqliteWriterDSN(path), "temp_store(MEMORY)", "temp_store(FILE)", 1) +} + +// withVacuumConnection is called under the maintenance lane and write gate. +// The callback is also the test seam for inspecting the actual dedicated +// connection; it is not used to change the pooled reader/writer policy. +func (s *Store) withVacuumConnection(ctx context.Context, run func(*sql.Conn) error) (result error) { + if err := vacuumTempDirectoryGate.LockContext(ctx); err != nil { + return err + } + defer vacuumTempDirectoryGate.Unlock() + + // Ask the existing physical database, rather than interpreting a possibly + // escaped file: URI or guessing a default path. A memory database has no + // main filename and is handled separately by vacuum. + rows, err := s.writerDB.QueryContext(ctx, `PRAGMA database_list`) + if err != nil { + return fmt.Errorf("%w: main filename: %w", ErrVacuumTempDirectory, err) + } + var path string + for rows.Next() { + var sequence int + var name, file string + if err := rows.Scan(&sequence, &name, &file); err != nil { + return errors.Join(fmt.Errorf("%w: main filename: %w", ErrVacuumTempDirectory, err), rows.Close()) + } + if name == "main" { + path = file + } + } + err = errors.Join(rows.Err(), rows.Close()) + if err != nil || path == "" { + return errors.Join(fmt.Errorf("%w: main filename unavailable", ErrVacuumTempDirectory), err) + } + + db, err := openSQLiteStoreDB(sqliteVacuumDSN(path)) + if err != nil { + return err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + defer func() { result = errors.Join(result, db.Close()) }() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer func() { result = errors.Join(result, conn.Close()) }() + + var mode int + if err := conn.QueryRowContext(ctx, `PRAGMA temp_store`).Scan(&mode); err != nil || mode != 1 { + return errors.Join(fmt.Errorf("%w: temp_store=%d, want FILE", ErrVacuumTempDirectory, mode), err) + } + previous, err := vacuumTempDirectory(ctx, conn) + if err != nil { + return fmt.Errorf("%w: read previous directory: %w", ErrVacuumTempDirectory, err) + } + // Restoration is owned cleanup, not canceled work. Join it and physical + // close before releasing the process-wide guard, even when VACUUM fails or + // ctx is canceled. No bounded-cleanup claim is made for this SQLite call. + defer func() { + result = errors.Join(result, setVacuumTempDirectory(context.Background(), conn, previous)) + }() + if err := setVacuumTempDirectory(ctx, conn, filepath.Dir(path)); err != nil { + return fmt.Errorf("%w: %w", ErrVacuumTempDirectory, err) + } + if err := ctx.Err(); err != nil { + return err + } + return run(conn) +} + +func vacuumTempDirectory(ctx context.Context, conn *sql.Conn) (string, error) { + var directory string + err := conn.QueryRowContext(ctx, `PRAGMA temp_store_directory`).Scan(&directory) + if errors.Is(err, sql.ErrNoRows) { + return "", nil // SQLite reports no row for its unset global directory. + } + return directory, err +} + +func setVacuumTempDirectory(ctx context.Context, conn *sql.Conn, directory string) error { + // PRAGMA assignment does not accept bound parameters. Quote the exact + // directory as a SQLite string, including embedded single quotes. + statement := `PRAGMA temp_store_directory='` + strings.ReplaceAll(directory, "'", "''") + `'` + if _, err := conn.ExecContext(ctx, statement); err != nil { + return fmt.Errorf("set vacuum temp directory: %w", err) + } + actual, err := vacuumTempDirectory(ctx, conn) + if err != nil { + return fmt.Errorf("verify vacuum temp directory: %w", err) + } + if actual != directory { + return fmt.Errorf("verify vacuum temp directory: got %q, want %q", actual, directory) + } + return nil +} diff --git a/internal/graph/store_sqlite/vacuum_connection_test.go b/internal/graph/store_sqlite/vacuum_connection_test.go new file mode 100644 index 000000000..b3d39bc9a --- /dev/null +++ b/internal/graph/store_sqlite/vacuum_connection_test.go @@ -0,0 +1,238 @@ +package store_sqlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestVacuumDedicatedConnectionKeepsPoolsInMemory(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "quote's store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + ctx := t.Context() + var before string + writer, err := s.writerDB.Conn(ctx) + require.NoError(t, err) + before, err = vacuumTempDirectory(ctx, writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + require.NoError(t, s.withVacuumConnection(ctx, func(conn *sql.Conn) error { + var mode, pooledWriter, pooledReader int + require.NoError(t, conn.QueryRowContext(ctx, `PRAGMA temp_store`).Scan(&mode)) + require.Equal(t, 1, mode, "only the dedicated connection uses FILE") + require.NoError(t, s.writerDB.QueryRowContext(ctx, `PRAGMA temp_store`).Scan(&pooledWriter)) + require.NoError(t, s.db.QueryRowContext(ctx, `PRAGMA temp_store`).Scan(&pooledReader)) + require.Equal(t, 2, pooledWriter) + require.Equal(t, 2, pooledReader) + directory, err := vacuumTempDirectory(ctx, conn) + require.NoError(t, err) + actual, err := filepath.EvalSymlinks(directory) + require.NoError(t, err) + want, err := filepath.EvalSymlinks(filepath.Dir(s.Path())) + require.NoError(t, err) + require.Equal(t, want, actual, "SQLite may resolve a /tmp alias in its main filename") + requireNoCloseCheckpoint(t, conn) + return nil + })) + writer, err = s.writerDB.Conn(ctx) + require.NoError(t, err) + defer writer.Close() + after, err := vacuumTempDirectory(ctx, writer) + require.NoError(t, err) + require.Equal(t, before, after, "process-global temp directory restored") +} + +func TestVacuumDirectoryRestoredAfterCanceledSQL(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + conn, err := s.writerDB.Conn(t.Context()) + require.NoError(t, err) + before, err := vacuumTempDirectory(t.Context(), conn) + require.NoError(t, err) + require.NoError(t, conn.Close()) + ctx, cancel := context.WithCancel(t.Context()) + err = s.withVacuumConnection(ctx, func(conn *sql.Conn) error { + cancel() + _, err := conn.ExecContext(ctx, `VACUUM`) + return err + }) + require.ErrorIs(t, err, context.Canceled) + conn, err = s.writerDB.Conn(t.Context()) + require.NoError(t, err) + defer conn.Close() + after, err := vacuumTempDirectory(t.Context(), conn) + require.NoError(t, err) + require.Equal(t, before, after) +} + +func TestCompactContextCancelsHeldWriterAdmission(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + s.writeMu.Lock() + defer s.writeMu.Unlock() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { done <- s.CompactContext(ctx) }() + deadline := time.After(5 * time.Second) + for s.writeMu.waiting() == 0 { + select { + case <-deadline: + t.Fatal("compaction did not reach held write gate") + case <-time.After(time.Millisecond): + } + } + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, err, ErrMaintenanceBusy) + case <-time.After(5 * time.Second): + t.Fatal("compaction ignored cancellation at write gate") + } +} + +func TestVacuumDirectoryGuardCancelsWithoutRunningCallback(t *testing.T) { + s, err := Open(filepath.Join(t.TempDir(), "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + vacuumTempDirectoryGate.Lock() + defer vacuumTempDirectoryGate.Unlock() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + done <- s.withVacuumConnection(ctx, func(*sql.Conn) error { + return errors.New("unexpected callback after canceled guard wait") + }) + }() + deadline := time.After(5 * time.Second) + for vacuumTempDirectoryGate.waiting() == 0 { + select { + case <-deadline: + t.Fatal("compaction did not reach directory guard") + case <-time.After(time.Millisecond): + } + } + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("directory guard ignored cancellation") + } +} + +func TestVacuumUnconfirmedDirectoryDoesNotRun(t *testing.T) { + s, err := Open(":memory:") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + called := false + err = s.withVacuumConnection(t.Context(), func(*sql.Conn) error { called = true; return nil }) + require.ErrorIs(t, err, ErrVacuumTempDirectory) + require.False(t, called) + require.NoError(t, s.CompactContext(t.Context()), "memory compatibility path does not open a second database") + require.Error(t, s.CompactContext(nil)) //nolint:staticcheck // SA1012: Verify explicit nil-context rejection. +} + +func TestVacuumTwoStoresKeepDistinctDirectories(t *testing.T) { + first, err := Open(filepath.Join(t.TempDir(), "first.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.Close()) }) + second, err := Open(filepath.Join(t.TempDir(), "second.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, second.Close()) }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + writer, err := first.writerDB.Conn(ctx) + require.NoError(t, err) + previous, err := vacuumTempDirectory(ctx, writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + firstDone, secondDone := make(chan error, 1), make(chan error, 1) + checkAndVacuum := func(conn *sql.Conn, path string) error { + directory, err := vacuumTempDirectory(ctx, conn) + if err != nil { + return err + } + actual, err := filepath.EvalSymlinks(directory) + if err != nil { + return err + } + want, err := filepath.EvalSymlinks(filepath.Dir(path)) + if err != nil { + return err + } + if actual != want { + return fmt.Errorf("vacuum directory=%q, want own physical store %q", actual, want) + } + _, err = conn.ExecContext(ctx, `VACUUM`) + return err + } + go func() { + firstDone <- first.withVacuumConnection(ctx, func(conn *sql.Conn) error { + close(entered) + <-release + return checkAndVacuum(conn, first.Path()) + }) + }() + var firstResult, secondResult error + var joinFirstOnce, joinSecondOnce sync.Once + joinFirst := func() { joinFirstOnce.Do(func() { firstResult = <-firstDone }) } + joinSecond := func() { joinSecondOnce.Do(func() { secondResult = <-secondDone }) } + t.Cleanup(func() { cancel(); unblock(); joinFirst() }) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("first actual dedicated connection did not enter") + } + go func() { + secondDone <- second.withVacuumConnection(ctx, func(conn *sql.Conn) error { + return checkAndVacuum(conn, second.Path()) + }) + }() + t.Cleanup(func() { cancel(); unblock(); joinSecond() }) + deadline := time.After(5 * time.Second) + for vacuumTempDirectoryGate.waiting() == 0 { + select { + case <-deadline: + t.Fatal("second actual store did not wait for the first temp-directory owner") + case <-time.After(time.Millisecond): + } + } + writer, err = first.writerDB.Conn(ctx) + require.NoError(t, err) + during, err := vacuumTempDirectory(ctx, writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + actual, err := filepath.EvalSymlinks(during) + require.NoError(t, err) + want, err := filepath.EvalSymlinks(filepath.Dir(first.Path())) + require.NoError(t, err) + require.Equal(t, want, actual, "second store cannot redirect first store's scratch") + unblock() + joinFirst() + joinSecond() + require.NoError(t, firstResult) + require.NoError(t, secondResult) + writer, err = first.writerDB.Conn(ctx) + require.NoError(t, err) + after, err := vacuumTempDirectory(ctx, writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + require.Equal(t, previous, after) +} diff --git a/internal/graph/store_sqlite/vacuum_temp_open_darwin_test.go b/internal/graph/store_sqlite/vacuum_temp_open_darwin_test.go new file mode 100644 index 000000000..43da137aa --- /dev/null +++ b/internal/graph/store_sqlite/vacuum_temp_open_darwin_test.go @@ -0,0 +1,5 @@ +//go:build darwin + +package store_sqlite + +func vacuumOpenIsAnonymous(int32) bool { return false } diff --git a/internal/graph/store_sqlite/vacuum_temp_open_linux_test.go b/internal/graph/store_sqlite/vacuum_temp_open_linux_test.go new file mode 100644 index 000000000..a0ee1578b --- /dev/null +++ b/internal/graph/store_sqlite/vacuum_temp_open_linux_test.go @@ -0,0 +1,46 @@ +//go:build linux + +package store_sqlite + +import ( + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func vacuumOpenIsAnonymous(flags int32) bool { + // O_TMPFILE contains the directory flag; equality avoids treating an ordinary + // directory open as an anonymous file. Its value is architecture-specific. + return flags&int32(unix.O_TMPFILE) == int32(unix.O_TMPFILE) +} + +func TestVacuumPhysicalProbeObservesRealAnonymousLinuxDescriptor(t *testing.T) { + directory, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + flags := unix.O_RDWR | unix.O_EXCL | unix.O_TMPFILE + fd, err := unix.Open(directory, flags, 0600) + if errors.Is(err, unix.EOPNOTSUPP) || errors.Is(err, unix.EINVAL) { + t.Skipf("this filesystem/kernel does not support native O_TMPFILE; named fallback still requires physical VACUUM proof: %v", err) + } + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, unix.Close(fd)) }) + observed, recognized, err := inspectVacuumNativeOpen(int32(fd), directory, int32(flags), directory) + require.NoError(t, err) + require.True(t, recognized) + require.True(t, observed.anonymous) + require.Equal(t, directory, observed.directory) + require.Zero(t, observed.links) + var actual unix.Stat_t + require.NoError(t, unix.Fstat(fd, &actual)) + require.Equal(t, uint64(actual.Dev), observed.device) + require.Equal(t, uint64(actual.Ino), observed.inode) + require.False(t, vacuumOpenIsAnonymous(int32(unix.O_RDONLY|unix.O_DIRECTORY)), "ordinary directory flags must not be mistaken for O_TMPFILE") + wrong, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + _, recognized, err = inspectVacuumNativeOpen(int32(fd), directory, int32(flags), wrong) + require.True(t, recognized) + require.Error(t, err, "an anonymous descriptor on another directory is not accepted") +} diff --git a/internal/graph/store_sqlite/vacuum_temp_open_unix_test.go b/internal/graph/store_sqlite/vacuum_temp_open_unix_test.go new file mode 100644 index 000000000..11cd1ecff --- /dev/null +++ b/internal/graph/store_sqlite/vacuum_temp_open_unix_test.go @@ -0,0 +1,310 @@ +//go:build darwin || linux + +package store_sqlite + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + "unsafe" + + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" + "modernc.org/libc" + sqlite3 "modernc.org/sqlite/lib" +) + +// Observe the real native open, not a directory listing: SQLite unlinks its +// temporary files immediately after open. SQLite's supported VFS system-call +// API exposes the generated filename before that unlink, on Unix. No driver +// struct offsets or production hooks are added. Other OSes retain the portable +// FILE/MEMORY policy tests; this is actual Unix file-placement evidence only. +// _sqlite3RunVacuum copies the main schema cache_size into vacuum_db; +// _sqlite3PagerOpen delays opening a temporary file until its first OsWrite. +// Our real policy is 32MiB, so the fixture must exceed that cache to prove +// actual FILE spill rather than accepting a successful entirely cached copy. +const vacuumTempFixtureRows = 16384 // 64MiB payload at the production cache policy + +var vacuumOpenTestMu sync.Mutex +var vacuumOpenOriginal atomic.Uintptr +var vacuumOpenObservation atomic.Pointer[vacuumOpenProbe] + +type vacuumOpenProbe struct { + directory string + mu sync.Mutex + opens []vacuumObservedOpen + err error + entered chan struct{} + release <-chan struct{} + once sync.Once +} + +type vacuumObservedOpen struct { + path, directory string + anonymous bool + device, inode, links uint64 + mode uint32 +} + +// Inspect the actual successful OS descriptor while SQLite still owns it. +// Never wrap it in os.File: its finalizer must not close SQLite's descriptor. +func inspectVacuumNativeOpen(fd int32, path string, flags int32, wanted string) (opened vacuumObservedOpen, recognized bool, err error) { + opened.path, opened.anonymous = path, vacuumOpenIsAnonymous(flags) + if !opened.anonymous && !strings.HasPrefix(filepath.Base(path), "etilqs_") { + return opened, false, nil + } + parent := filepath.Dir(path) + if opened.anonymous { + parent = path // O_TMPFILE opens an anonymous regular inode in this directory. + } + opened.directory, err = filepath.EvalSymlinks(parent) + if err != nil { + return opened, true, err + } + if opened.directory != wanted { + return opened, true, fmt.Errorf("temporary open directory=%q, want %q", opened.directory, wanted) + } + var actual, directory unix.Stat_t + if err = unix.Fstat(int(fd), &actual); err != nil { + return opened, true, err + } + opened.device, opened.inode, opened.links, opened.mode = uint64(actual.Dev), uint64(actual.Ino), uint64(actual.Nlink), uint32(actual.Mode) + if actual.Mode&unix.S_IFMT != unix.S_IFREG { + return opened, true, fmt.Errorf("temporary descriptor is not a regular file: %s", path) + } + if err = unix.Stat(wanted, &directory); err != nil { + return opened, true, err + } + if uint64(actual.Dev) != uint64(directory.Dev) { + return opened, true, errors.New("temporary descriptor is on a different device from the store directory") + } + if opened.anonymous { + if actual.Nlink != 0 { + return opened, true, errors.New("anonymous temporary descriptor has a linked name") + } + } else { + var named unix.Stat_t + if err = unix.Stat(path, &named); err != nil { + return opened, true, err + } + if actual.Dev != named.Dev || actual.Ino != named.Ino { + return opened, true, errors.New("named temporary path does not match the actual descriptor") + } + } + return opened, true, nil +} + +var observeVacuumNativeOpen = func(tls *libc.TLS, name uintptr, flags, mode int32) int32 { + original := vacuumOpenOriginal.Load() + fd := (*(*func(*libc.TLS, uintptr, int32, int32) int32)(unsafe.Pointer(&struct{ uintptr }{original})))(tls, name, flags, mode) + probe := vacuumOpenObservation.Load() + if probe == nil || fd < 0 { + return fd + } + opened, recognized, err := inspectVacuumNativeOpen(fd, libc.GoString(name), flags, probe.directory) + if !recognized { + return fd + } + probe.mu.Lock() + probe.opens = append(probe.opens, opened) + probe.err = errors.Join(probe.err, err) + probe.mu.Unlock() + if err == nil && probe.entered != nil { + probe.once.Do(func() { close(probe.entered); <-probe.release }) + } + return fd +} + +func TestVacuumPhysicallyOpensTempOnStoreVolume(t *testing.T) { + vacuumOpenTestMu.Lock() + defer vacuumOpenTestMu.Unlock() + s, err := Open(filepath.Join(t.TempDir(), "physical.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + _, err = s.writerDB.Exec(fmt.Sprintf(`CREATE TABLE vacuum_fixture (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<%d) + INSERT INTO vacuum_fixture SELECT n, randomblob(4096) FROM ids`, vacuumTempFixtureRows)) + require.NoError(t, err) + directory, err := filepath.EvalSymlinks(filepath.Dir(s.Path())) + require.NoError(t, err) + probe := &vacuumOpenProbe{directory: directory} + require.NoError(t, withVacuumOpenObservation(probe, func() error { return s.CompactContext(t.Context()) })) + probe.mu.Lock() + defer probe.mu.Unlock() + require.NoError(t, probe.err) + require.NotEmpty(t, probe.opens, "a successful pragma alone is not physical placement proof") + want, err := filepath.EvalSymlinks(filepath.Dir(s.Path())) + require.NoError(t, err) + var parent unix.Stat_t + require.NoError(t, unix.Stat(want, &parent)) + for _, opened := range probe.opens { + require.Equal(t, want, opened.directory) + require.Equal(t, uint64(parent.Dev), opened.device) + require.Equal(t, uint32(unix.S_IFREG), opened.mode&unix.S_IFMT) + if opened.anonymous { + require.Zero(t, opened.links) + } else { + directory, err := filepath.EvalSymlinks(filepath.Dir(opened.path)) + require.NoError(t, err) + require.Equal(t, want, directory) + } + t.Logf("VACUUM actual native temp open: path=%s anonymous=%v device=%d inode=%d links=%d", opened.path, opened.anonymous, opened.device, opened.inode, opened.links) + } + var count int + require.NoError(t, s.db.QueryRow(`SELECT count(*) FROM vacuum_fixture`).Scan(&count)) + require.Equal(t, vacuumTempFixtureRows, count) +} + +// SQLite's native VFS open signature is documented in the pinned generated +// _robust_open: (TLS, pathname, flags, mode). Restore its exact original +// pointer; no sandbox/environment/filename substitution is performed. +func withVacuumOpenObservation(probe *vacuumOpenProbe, run func() error) (result error) { + tls := libc.NewTLS() + defer tls.Close() + vfs := sqlite3.Xsqlite3_vfs_find(tls, 0) + if vfs == 0 || vfsAt(vfs).FxGetSystemCall == 0 || vfsAt(vfs).FxSetSystemCall == 0 { + return errors.New("vacuum physical test: native VFS system-call API unavailable") + } + name, err := libc.CString("open") + if err != nil { + return err + } + defer libc.Xfree(tls, name) + get, set := vfsAt(vfs).FxGetSystemCall, vfsAt(vfs).FxSetSystemCall + original := (*(*func(*libc.TLS, uintptr, uintptr) uintptr)(unsafe.Pointer(&struct{ uintptr }{get})))(tls, vfs, name) + if original == 0 { + return errors.New("vacuum physical test: native open unavailable") + } + setOpen := func(pointer uintptr) int32 { + return (*(*func(*libc.TLS, uintptr, uintptr, uintptr) int32)(unsafe.Pointer(&struct{ uintptr }{set})))(tls, vfs, name, pointer) + } + vacuumOpenOriginal.Store(original) + vacuumOpenObservation.Store(probe) + defer vacuumOpenObservation.Store(nil) + if rc := setOpen(funcPointer(observeVacuumNativeOpen)); rc != sqlite3.SQLITE_OK { + return fmt.Errorf("vacuum physical test: set native open rc=%d", rc) + } + defer func() { + if rc := setOpen(original); rc != sqlite3.SQLITE_OK { + result = errors.Join(result, fmt.Errorf("restore native open rc=%d", rc)) + } + }() + return run() +} + +func TestCompactContextInterruptsInFlightVacuumAndRestoresDirectory(t *testing.T) { + vacuumOpenTestMu.Lock() + defer vacuumOpenTestMu.Unlock() + path := filepath.Join(t.TempDir(), "cancel.sqlite") + s, err := Open(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, s.Close()) }) + _, err = s.writerDB.Exec(fmt.Sprintf(`CREATE TABLE vacuum_fixture (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE ids(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM ids WHERE n<%d) + INSERT INTO vacuum_fixture SELECT n, randomblob(4096) FROM ids`, vacuumTempFixtureRows)) + require.NoError(t, err) + require.NoError(t, s.CheckpointWAL()) + var firstPayload string + require.NoError(t, s.db.QueryRow(`SELECT hex(payload) FROM vacuum_fixture WHERE id=1`).Scan(&firstPayload)) + writer, err := s.writerDB.Conn(t.Context()) + require.NoError(t, err) + previous, err := vacuumTempDirectory(t.Context(), writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + directory, err := filepath.EvalSymlinks(filepath.Dir(path)) + require.NoError(t, err) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + defer unblock() + probe := &vacuumOpenProbe{directory: directory, entered: make(chan struct{}), release: release} + done := make(chan error, 1) + go func() { done <- withVacuumOpenObservation(probe, func() error { return s.CompactContext(ctx) }) }() + // Cleanup joins the operation before store.Close, even on an assertion + // failure. This barrier is the actual successful native temp-file open + // inside VACUUM; no cancellation sleep or canceled-before-Exec shortcut. + var result error + var joinOnce sync.Once + join := func() { joinOnce.Do(func() { cancel(); unblock(); result = <-done }) } + t.Cleanup(join) + select { + case <-probe.entered: + case early := <-done: + joinOnce.Do(func() { cancel(); unblock(); result = early }) + probe.mu.Lock() + observed, observerErr := append([]vacuumObservedOpen(nil), probe.opens...), probe.err + probe.mu.Unlock() + t.Fatalf("VACUUM ended before a verified physical temporary open: result=%v observer=%v opens=%+v", early, observerErr, observed) + case <-time.After(10 * time.Second): + t.Fatal("VACUUM never reached its physical temporary open") + } + join() + require.ErrorIs(t, result, context.Canceled) + probe.mu.Lock() + require.NoError(t, probe.err) + require.NotEmpty(t, probe.opens) + probe.mu.Unlock() + writer, err = s.writerDB.Conn(t.Context()) + require.NoError(t, err) + after, err := vacuumTempDirectory(t.Context(), writer) + require.NoError(t, err) + require.NoError(t, writer.Close()) + require.Equal(t, previous, after) + require.NoError(t, s.Close()) + reopened, err := Open(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + var check, payload string + var count, bytes int + require.NoError(t, reopened.db.QueryRow(`PRAGMA quick_check`).Scan(&check)) + require.Equal(t, "ok", check) + require.NoError(t, reopened.db.QueryRow(`SELECT count(*), sum(length(payload)) FROM vacuum_fixture`).Scan(&count, &bytes)) + require.Equal(t, vacuumTempFixtureRows, count) + require.Equal(t, vacuumTempFixtureRows*4096, bytes) + require.NoError(t, reopened.db.QueryRow(`SELECT hex(payload) FROM vacuum_fixture WHERE id=1`).Scan(&payload)) + require.Equal(t, firstPayload, payload, "canceled VACUUM retained durable survivor bytes") +} + +func TestVacuumPhysicalProbeRequiresMatchingRegularDescriptorAndDirectory(t *testing.T) { + directory, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + path := filepath.Join(directory, "etilqs_probe") + file, err := os.Create(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, file.Close()) }) + observed, recognized, err := inspectVacuumNativeOpen(int32(file.Fd()), path, 0, directory) + require.NoError(t, err) + require.True(t, recognized) + require.False(t, observed.anonymous) + require.Equal(t, directory, observed.directory) + var actual unix.Stat_t + require.NoError(t, unix.Fstat(int(file.Fd()), &actual)) + require.Equal(t, uint64(actual.Dev), observed.device) + require.Equal(t, uint64(actual.Ino), observed.inode) + wrong, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + _, recognized, err = inspectVacuumNativeOpen(int32(file.Fd()), path, 0, wrong) + require.True(t, recognized) + require.ErrorContains(t, err, "temporary open directory") + parent, err := os.Open(directory) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, parent.Close()) }) + _, recognized, err = inspectVacuumNativeOpen(int32(parent.Fd()), path, 0, directory) + require.True(t, recognized) + require.ErrorContains(t, err, "not a regular file") + _, recognized, err = inspectVacuumNativeOpen(-1, path, 0, directory) + require.True(t, recognized) + require.Error(t, err, "invalid actual descriptor cannot become placement evidence") + _, recognized, err = inspectVacuumNativeOpen(int32(file.Fd()), filepath.Join(directory, "ordinary.sqlite"), 0, directory) + require.NoError(t, err) + require.False(t, recognized, "an unrelated regular open is not VACUUM temporary evidence") +} diff --git a/internal/graph/store_sqlite/wal_reclaim.go b/internal/graph/store_sqlite/wal_reclaim.go index 5543ea4cf..40ed9afbe 100644 --- a/internal/graph/store_sqlite/wal_reclaim.go +++ b/internal/graph/store_sqlite/wal_reclaim.go @@ -254,10 +254,17 @@ func resolveWALReclaimConfig() walReclaimConfig { // read gate was closed per attempt that closed it; ReaderWait is what readers // actually spent held at the gate (per held read), measured on their side. type WALReclaimStats struct { - ThresholdBytes int64 - Attempts int64 // attempts that passed the size check and the lease - Resets int64 // TRUNCATE checkpoints that reset the log - OpenGateResets int64 // resets reached without closing the read gate + // One first observed cause per final non-reset reclaim attempt, including + // writer-demand refusal. Parked writer precedes intent if both are seen. + CancelledWriterParked, CancelledWriteIntent, CancelledOther int64 + // Lower-bound sampled high-water since this store opened, not an exact + // instantaneous or warm-run peak. Status reads never reset it. + ObservedPeakWALBytes int64 + WALPeakObserved bool + ThresholdBytes int64 + Attempts int64 // attempts that passed the size check and the lease + Resets int64 // TRUNCATE checkpoints that reset the log + OpenGateResets int64 // resets reached without closing the read gate // LastResortRuns counts attempts at walReclaimLastResortBytes (each may // hold the writer up to walReclaimMaxWriterHold). LastResortRuns int64 @@ -318,8 +325,10 @@ type WALReclaimStats struct { } type walReclaimState struct { - mu sync.Mutex - stats WALReclaimStats + observedPeak atomic.Int64 + peakObserved atomic.Bool + mu sync.Mutex + stats WALReclaimStats // cycle is the build-lane yield shared by the reclaim and the PASSIVE // loop (checkpoint_cycle_yield.go). cycle checkpointCycleYield @@ -333,6 +342,8 @@ func (s *Store) WALReclaimStats() WALReclaimStats { s.walReclaim.mu.Lock() out := s.walReclaim.stats s.walReclaim.mu.Unlock() + out.WALPeakObserved = s.walReclaim.peakObserved.Load() + out.ObservedPeakWALBytes = s.walReclaim.observedPeak.Load() cycle := &s.walReclaim.cycle out.CycleDeferrals = cycle.deferrals.Load() out.CycleRefusals = cycle.refusals.Load() @@ -398,6 +409,7 @@ type walReclaimResetReaders struct { } type walReclaimResult struct { + attempt *backgroundCheckpointAttempt // bounded cause witness, never a global registry outcome walReclaimOutcome reason string frames int @@ -536,13 +548,14 @@ func (s *Store) reclaimWALOnce(cfg walReclaimConfig, ckptDB *sql.DB, walPath str } if res.outcome == walReclaimSkipped && res.bytesBefore == 0 { // A refusal returns before measuring the log; the skip line reports it. - res.bytesBefore = walFileSize(walPath) + res.bytesBefore = s.observedWALFileSize(walPath) } if res.outcome == walReclaimReset { res.progressed = true } s.walReclaim.update(func(st *WALReclaimStats) { st.ThresholdBytes = cfg.thresholdBytes + st.noteInterruption(res) st.LastOutcome = res.outcome.String() st.LastReason = res.reason if res.adaptiveUsed { @@ -596,7 +609,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath laneBusy := s.cycleYieldEnabled() && s.buildLaneBusy() hardCap, pressure := false, false if cfg.ceilingBytes > 0 { - size := walFileSize(walPath) + size := s.observedWALFileSize(walPath) if laneBusy && size >= walReclaimHardCapFactor*cfg.ceilingBytes && s.hardCapDue(time.Now()) { policy, hardCap = checkpointIgnoresCycle, true s.walReclaim.cycle.ceiling.Add(1) @@ -609,7 +622,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath policy, pressure = checkpointIgnoresCycle, true } } - attempt, berr := s.beginBackgroundCheckpointAttempt(policy) + attempt, berr := s.beginCheckpointAttempt(policy, true) if errors.Is(berr, errWALCheckpointYieldedToCycle) { s.walReclaim.cycle.refusals.Add(1) return walReclaimResult{outcome: walReclaimSkipped, reason: "build_lane_busy"} @@ -624,7 +637,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath // ceiling one attempt runs anyway — bounded by the writer-hold cap, // its writer step taking the write gate between the window's own // transactions — at most once per walReclaimLeaseOverrideSpacing. - size := walFileSize(walPath) + size := s.observedWALFileSize(walPath) if cfg.ceilingBytes <= 0 || size < cfg.ceilingBytes || !s.leaseOverrideDue(time.Now()) { return walReclaimResult{outcome: walReclaimSkipped, reason: "checkpoint_lease"} } @@ -632,7 +645,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath if hardCap { leasePolicy = checkpointOverridesLeaseAndCycle } - attempt, berr = s.beginBackgroundCheckpointAttempt(leasePolicy) + attempt, berr = s.beginCheckpointAttempt(leasePolicy, true) if errors.Is(berr, errWALCheckpointYieldedToCycle) { s.walReclaim.cycle.refusals.Add(1) return walReclaimResult{outcome: walReclaimSkipped, reason: "build_lane_busy"} @@ -671,7 +684,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath return walReclaimResult{outcome: walReclaimSkipped, reason: "copy_budget"} } - res := walReclaimResult{bytesBefore: walFileSize(walPath), leaseOverride: leaseOverride, began: true, copy: attempt.copy, pressure: pressure, hardCap: hardCap, bulkCompletion: leaseOverride} + res := walReclaimResult{attempt: attempt, bytesBefore: s.observedWALFileSize(walPath), leaseOverride: leaseOverride, began: true, copy: attempt.copy, pressure: pressure, hardCap: hardCap, bulkCompletion: leaseOverride} if mark := walReclaimLastResortBytes(cfg); mark > 0 && res.bytesBefore >= mark { res.lastResort = true } @@ -685,7 +698,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath initial, initialErr := s.pacedPassive(attempt.ctx, ckptDB, attempt) if s.walAttemptHandsOverToPressure(attempt.ctx, cfg, walPath, &res, initial, initialErr) { res.outcome, res.reason = walReclaimSkipped, "pressure_mark" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } // Step 2: without the writer and with the gate OPEN, wait for a log no @@ -729,7 +742,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath pcancel() if s.walAttemptHandsOverToPressure(attempt.ctx, cfg, walPath, &res, result, perr) { res.outcome, res.reason = walReclaimSkipped, "pressure_mark" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } if s.walAttemptHandsOverToRequest(&res) { @@ -810,23 +823,23 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath } if !res.resetWriterFree && s.walAttemptHandsOverToRequest(&res) { res.outcome, res.reason = walReclaimSkipped, "reclaim_request" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } if res.resetWriterFree { res.openGate = true res.outcome = walReclaimReset - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } - if cfg.thresholdBytes > 0 && walFileSize(walPath) >= walReclaimUrgentFactor*cfg.thresholdBytes { + if cfg.thresholdBytes > 0 && s.observedWALFileSize(walPath) >= walReclaimUrgentFactor*cfg.thresholdBytes { res.urgent = true } // An edit began meanwhile: the copy so far is kept (every pass that // completed moved nBackfill); the writer is not taken during an edit. if attempt.copy != nil && attempt.copy.pausable.Load() && !attempt.copy.pressure && s.buildLaneBusy() { res.outcome, res.reason = walReclaimSkipped, "build_lane_busy" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } // Before the writer: converge the backfill so the capped writer step @@ -847,22 +860,23 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath } if attempt.copy != nil && attempt.copy.pausable.Load() && !attempt.copy.pressure && s.buildLaneBusy() { res.outcome, res.reason = walReclaimSkipped, "build_lane_busy" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } if s.walAttemptHandsOverToRequest(&res) { res.outcome, res.reason = walReclaimSkipped, "reclaim_request" - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) return res } if pressure && s.buildLaneBusy() { // Inside the busy lane: the reset alone, under its own short cap. err := s.reclaimWALPressureResetWithDrain(attempt.ctx, ckptDB, &res, cfg.drainDeadline) - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) if err == nil { res.outcome = walReclaimReset return res } + noteWALReclaimCancellation(attempt, err) if res.outcome == 0 || res.outcome == walReclaimReset { res.outcome = walReclaimDeferred } @@ -881,7 +895,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath return s.reclaimWALInLane(ctx, cfg, ckptDB, &res) }) } - res.bytesAfter = walFileSize(walPath) + res.bytesAfter = s.observedWALFileSize(walPath) if err == nil { res.outcome = walReclaimReset return res @@ -895,6 +909,7 @@ func (s *Store) reclaimWALAttempt(cfg walReclaimConfig, ckptDB *sql.DB, walPath return res } if attempt.ctx.Err() != nil { + noteWALReclaimCancellation(attempt, attempt.ctx.Err()) res.outcome, res.reason = walReclaimSkipped, "shutdown" return res } @@ -927,7 +942,7 @@ func (s *Store) walAttemptHandsOverToPressure(ctx context.Context, cfg walReclai return false } mark := walPressureMark(cfg) - return mark > 0 && walFileSize(walPath) >= mark + return mark > 0 && s.observedWALFileSize(walPath) >= mark } // walAttemptHandsOverToRequest reports an attempt that should end so the loop @@ -955,9 +970,10 @@ func (s *Store) reclaimWALInLane(ctx context.Context, cfg walReclaimConfig, ckpt urgent := res.urgent // A mutation already waiting for the writer wins outright (unless the log // is urgent; the hold stays capped). - if !urgent && s.writeWanted() { + if demand := s.writeDemandCause(); !urgent && demand != nil { + noteWALReclaimDemand(ctx, demand) res.outcome, res.reason = walReclaimSkipped, "writer_waiting" - return errWALReclaimWriterWaiting + return demand } if !res.lastResort && !res.bulkCompletion && !walReclaimSkipQuiescence && s.readGate != nil && !walReclaimSkipOpenGate && cfg.readerWait > 0 { return s.reclaimWALInLaneOpenGate(ctx, cfg, ckptDB, res) @@ -969,7 +985,7 @@ func (s *Store) reclaimWALInLane(ctx context.Context, cfg walReclaimConfig, ckpt }() } if res.lastResort { - at := walFileSize(s.dbPath + "-wal") + at := s.observedWALFileSize(s.dbPath + "-wal") defer func() { log.Printf("store_sqlite: wal reclaim last resort wal_bytes=%d mark=%d writer_hold=%s reset=%t reason=%q", at, walReclaimLastResortBytes(cfg), res.writerHold.Round(time.Millisecond), res.openGate, res.reason) @@ -1015,7 +1031,7 @@ func (s *Store) reclaimWALInLane(ctx context.Context, cfg walReclaimConfig, ckpt } if expected := res.lastResortPromotion; expected != nil { current, ok := readWALReclaimFrontier(s.dbPath) - if !ok || current.salt != expected.salt || current.mx < expected.mx || current.backfill < expected.backfill || walFileSize(s.dbPath+"-wal") < walReclaimLastResortBytes(cfg) { + if !ok || current.salt != expected.salt || current.mx < expected.mx || current.backfill < expected.backfill || s.observedWALFileSize(s.dbPath+"-wal") < walReclaimLastResortBytes(cfg) { res.reason = "last_resort_frontier_changed" return errWALReclaimReadersInFlight } @@ -1031,7 +1047,14 @@ func (s *Store) reclaimWALInLane(ctx context.Context, cfg walReclaimConfig, ckpt yctx, stopYield = s.yieldToWriters(hctx) } defer stopYield() - yielded := func() bool { return errors.Is(context.Cause(yctx), errWALReclaimWriterWaiting) } + yielded := func() bool { + cause := context.Cause(yctx) + if !errors.Is(cause, errWALReclaimWriterWaiting) { + return false + } + noteWALReclaimDemand(ctx, cause) + return true + } giveUp := func(reason string, cause error) error { if yielded() { res.outcome, res.reason = walReclaimSkipped, "writer_waiting" @@ -1204,8 +1227,8 @@ func (s *Store) yieldToWriters(parent context.Context) (context.Context, func()) case <-ctx.Done(): return case <-ticker.C: - if s.writeWanted() { - cancel(errWALReclaimWriterWaiting) + if demand := s.writeDemandCause(); demand != nil { + cancel(demand) return } } @@ -1427,7 +1450,7 @@ func (s *Store) runWALReclaimLoop(cfg walReclaimConfig, walPath string, poll tim } now = time.Now() } - if !schedule.ready(now) || walFileSize(walPath) <= cfg.thresholdBytes { + if !schedule.ready(now) || s.observedWALFileSize(walPath) <= cfg.thresholdBytes { continue } if ckptDB == nil { @@ -1557,7 +1580,7 @@ func (s *Store) reclaimWALInLaneOpenGate(ctx context.Context, cfg walReclaimConf if err := ctx.Err(); err != nil { return err } - if mark > 0 && walFileSize(s.dbPath+"-wal") >= mark { + if mark > 0 && s.observedWALFileSize(s.dbPath+"-wal") >= mark { current, ok := readWALReclaimFrontier(s.dbPath) if !entryOK || !ok || current.salt != entry.salt || current.mx < entry.mx || current.backfill < entry.backfill { res.reason = "last_resort_frontier_changed" @@ -1568,7 +1591,7 @@ func (s *Store) reclaimWALInLaneOpenGate(ctx context.Context, cfg walReclaimConf return errWALCheckpointYieldedToCycle } res.lastResort, res.lastResortPromotion = true, ¤t - if cfg.thresholdBytes > 0 && walFileSize(s.dbPath+"-wal") >= walReclaimUrgentFactor*cfg.thresholdBytes { + if cfg.thresholdBytes > 0 && s.observedWALFileSize(s.dbPath+"-wal") >= walReclaimUrgentFactor*cfg.thresholdBytes { res.urgent = true } // Keep this operation's deadline and charge every preceding short @@ -1594,7 +1617,7 @@ func (s *Store) reclaimWALInLaneOpenGate(ctx context.Context, cfg walReclaimConf } if res.resetReaders != nil { epoch, older, werr := s.waitWALRoundReaders(waitCtx, res, deadline) - if werr != nil && ctx.Err() == nil && waitCtx.Err() != nil && mark > 0 && walFileSize(s.dbPath+"-wal") >= mark { + if werr != nil && ctx.Err() == nil && waitCtx.Err() != nil && mark > 0 && s.observedWALFileSize(s.dbPath+"-wal") >= mark { continue // recheck current identity/mark at the round boundary } if werr != nil { @@ -1621,7 +1644,7 @@ func (s *Store) reclaimWALInLaneOpenGate(ctx context.Context, cfg walReclaimConf if observer != nil { observer("lane_post_copy_full_wait_end", epoch, older, werr) } - if werr != nil && ctx.Err() == nil && waitCtx.Err() != nil && mark > 0 && walFileSize(s.dbPath+"-wal") >= mark { + if werr != nil && ctx.Err() == nil && waitCtx.Err() != nil && mark > 0 && s.observedWALFileSize(s.dbPath+"-wal") >= mark { continue // a growing WAL now qualifies the existing completion class } if werr != nil { @@ -1739,9 +1762,10 @@ func (s *Store) reclaimWALResetHold(ctx context.Context, cfg walReclaimConfig, c func (s *Store) reclaimWALResetHoldOnce(ctx context.Context, cfg walReclaimConfig, ckptDB *sql.DB, res *walReclaimResult, first, yieldsToLane bool, budget time.Duration) (done bool, err error) { res.resetReaders = nil - if !res.urgent && s.writeWanted() { + if demand := s.writeDemandCause(); !res.urgent && demand != nil { + noteWALReclaimDemand(ctx, demand) res.outcome, res.reason = walReclaimSkipped, "writer_waiting" - return false, errWALReclaimWriterWaiting + return false, demand } wctx, wcancel := context.WithTimeout(ctx, walReclaimWriterWait) err = s.writeMu.LockContext(wctx) @@ -1778,7 +1802,14 @@ func (s *Store) reclaimWALResetHoldOnce(ctx context.Context, cfg walReclaimConfi yctx, stopYield = s.yieldToWriters(hctx) } defer stopYield() - yielded := func() bool { return errors.Is(context.Cause(yctx), errWALReclaimWriterWaiting) } + yielded := func() bool { + cause := context.Cause(yctx) + if !errors.Is(cause, errWALReclaimWriterWaiting) { + return false + } + noteWALReclaimDemand(ctx, cause) + return true + } // Ordinary holds copy only a fitted small remainder. A positively witnessed // adaptive copy is admitted by its remaining bounded time credit instead. snap, ok := readWALIndexSnapshot(s.dbPath) @@ -1887,7 +1918,7 @@ func (s *Store) watchWALMark(parent context.Context, walPath string, markBytes i case <-ctx.Done(): return case <-ticker.C: - if walFileSize(walPath) >= markBytes { + if s.observedWALFileSize(walPath) >= markBytes { cancel() return } diff --git a/internal/graph/store_sqlite/wal_reclaim_observations.go b/internal/graph/store_sqlite/wal_reclaim_observations.go new file mode 100644 index 000000000..a0438f3d2 --- /dev/null +++ b/internal/graph/store_sqlite/wal_reclaim_observations.go @@ -0,0 +1,98 @@ +package store_sqlite + +import ( + "context" + "errors" + "strings" +) + +// Observation precedence, not exclusive causality: a positively observed +// writer parked on writeMu wins over a simultaneously observed announced intent. +type walReclaimInterruption uint32 + +const ( + walReclaimUninterrupted walReclaimInterruption = iota + walReclaimWriterParked + walReclaimWriteIntent + walReclaimOtherCancellation +) + +type walReclaimDemandCause struct{ kind walReclaimInterruption } + +func (c *walReclaimDemandCause) Error() string { return errWALReclaimWriterWaiting.Error() } +func (c *walReclaimDemandCause) Unwrap() error { return errWALReclaimWriterWaiting } + +var ( + walParkedDemand = &walReclaimDemandCause{walReclaimWriterParked} + walIntentDemand = &walReclaimDemandCause{walReclaimWriteIntent} +) + +type walReclaimAttemptKey struct{} + +func (s *Store) writeDemandCause() error { + if s.writeMu.waiting() > 0 { + return walParkedDemand + } + if s.writeIntents.Load() > 0 { + return walIntentDemand + } + return nil +} + +func noteWALReclaimDemand(ctx context.Context, cause error) { + attempt, _ := ctx.Value(walReclaimAttemptKey{}).(*backgroundCheckpointAttempt) + if attempt == nil { + return + } + var demand *walReclaimDemandCause + if errors.As(cause, &demand) { + attempt.reclaimInterruption.CompareAndSwap(uint32(walReclaimUninterrupted), uint32(demand.kind)) + } +} + +func noteWALReclaimCancellation(attempt *backgroundCheckpointAttempt, err error) { + // Child budget expiration, cycle/bulk withdrawal and finish/stopYield + // cleanup are not terminal caller/shutdown cancellation. Observe only a + // consumed error while the ORIGINAL operation is cancelled by that cause. + if errors.Is(err, context.Canceled) && errors.Is(context.Cause(attempt.ctx), context.Canceled) { + attempt.reclaimInterruption.CompareAndSwap(uint32(walReclaimUninterrupted), uint32(walReclaimOtherCancellation)) + } +} + +func (st *WALReclaimStats) noteInterruption(res walReclaimResult) { + if res.outcome == walReclaimReset || res.attempt == nil { + return + } + switch walReclaimInterruption(res.attempt.reclaimInterruption.Load()) { + case walReclaimWriterParked: + st.CancelledWriterParked++ + case walReclaimWriteIntent: + st.CancelledWriteIntent++ + case walReclaimOtherCancellation: + st.CancelledOther++ + } +} + +// noteObservedWALBytes consumes an EXISTING successful size observation, never +// performs I/O, and never resets on truncate/shrink or a status read. +func (s *Store) noteObservedWALBytes(bytes int64) { + if s.coreless() || s.dbPath == "" || strings.Contains(s.dbPath, ":memory:") || bytes < 0 { + return + } + for old := s.walReclaim.observedPeak.Load(); bytes > old; old = s.walReclaim.observedPeak.Load() { + if s.walReclaim.observedPeak.CompareAndSwap(old, bytes) { + break + } + } + s.walReclaim.peakObserved.Store(true) +} + +func (s *Store) observedWALFileSize(path string) int64 { + bytes := walFileSize(path) + // The legacy helper collapses absence/I/O errors to0. A positive size + // proves an observation; DBStats separately handles proven empty/absent WAL. + if bytes > 0 { + s.noteObservedWALBytes(bytes) + } + return bytes +} diff --git a/internal/graph/store_sqlite/wal_reclaim_observations_test.go b/internal/graph/store_sqlite/wal_reclaim_observations_test.go new file mode 100644 index 000000000..93a81c9d7 --- /dev/null +++ b/internal/graph/store_sqlite/wal_reclaim_observations_test.go @@ -0,0 +1,233 @@ +package store_sqlite + +import ( + "context" + "database/sql" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestWALReclaimFinalCauseSurvivesDisappearingDemand(t *testing.T) { + for _, kind := range []string{"intent", "parked", "both", "shutdown"} { + t.Run(kind, func(t *testing.T) { + t.Setenv("GORTEX_SQLITE_WAL_RECLAIM_MB", "0") + s, path := openWALReclaimStore(t) + defer func() { require.NoError(t, s.Close()) }() + seedWALChurnTable(t, s) + growWAL(t, s, 4) + ckpt, err := openWALReclaimCheckpointDB(path) + require.NoError(t, err) + defer ckpt.Close() + before := s.WALReclaimStats() + oldHook := walResetCheckpointConnObserver + defer func() { walResetCheckpointConnObserver = oldHook }() + entered := make(chan context.Context, 1) + beforeReturn := make(chan func(), 1) + var resetCalls atomic.Int32 + walResetCheckpointConnObserver = func(ctx context.Context, _ *sql.Conn) { + if resetCalls.Add(1) != 1 { + // Record unexpected reentry without parking on a second + // test token. The completion oracle below must reject it. + return + } + entered <- ctx + <-ctx.Done() + // Remove real demand BEFORE the attempt consumes the cancelled + // query result/updates counters. Attribution must use its cause. + cleanup := <-beforeReturn + cleanup() + } + done := make(chan walReclaimResult, 1) + joined := false + defer func() { + if joined { + return + } + s.backgroundCheckpoint.mu.Lock() + attempt := s.backgroundCheckpoint.active + s.backgroundCheckpoint.mu.Unlock() + if attempt != nil { + attempt.cancel(context.Canceled) + } + select { + case beforeReturn <- func() {}: + default: + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("owned reclaim did not join") + } + }() + go func() { + // readerWait=0 selects the supported terminal closed-gate + // path. An ordinary open-gate round may legitimately retry + // after the demand disappears, even within one attempt. + done <- s.reclaimWALOnce(walReclaimConfig{thresholdBytes: 1 << 30, + drainDeadline: time.Second, truncateBudget: walReclaimTruncateBudget}, ckpt, path+"-wal") + }() + var resetCtx context.Context + select { + case resetCtx = <-entered: + case result := <-done: + joined = true + t.Fatalf("real reset boundary was not reached: %+v", result) + case <-time.After(5 * time.Second): + t.Fatal("real reset did not start") + } + var cleanups []func() + if kind == "parked" || kind == "both" { + writerCtx, cancel := context.WithCancel(t.Context()) + writerDone := make(chan error, 1) + go func() { + err := s.writeMu.LockContext(writerCtx) + if err == nil { + s.writeMu.Unlock() + } + writerDone <- err + }() + var once sync.Once + stopWriter := func() { once.Do(func() { cancel(); <-writerDone }) } + defer stopWriter() + cleanups = append(cleanups, stopWriter) + require.Eventually(t, func() bool { return s.writeMu.waiting() > 0 }, time.Second, time.Millisecond, + "an actual foreground writer must park behind the held reclaim gate") + } + if kind == "intent" || kind == "both" { + release := s.AnnounceWrite() + defer release() + cleanups = append(cleanups, release) + if kind == "both" { + require.Same(t, walParkedDemand, s.writeDemandCause(), "observed parked demand takes precedence when both are positive") + } + } + if kind == "shutdown" { + attempt := resetCtx.Value(walReclaimAttemptKey{}).(*backgroundCheckpointAttempt) + attempt.cancel(context.Canceled) + } + beforeReturn <- func() { + for _, cleanup := range cleanups { + cleanup() + } + } + var result walReclaimResult + select { + case result = <-done: + joined = true + case <-time.After(5 * time.Second): + t.Fatal("cancelled reclaim did not finish") + } + require.EqualValues(t, 1, resetCalls.Load(), "unexpected reset reentry is not a terminal cancelled attempt") + require.NotEqual(t, walReclaimReset, result.outcome) + require.Zero(t, s.writeIntents.Load()) + require.Zero(t, s.writeMu.waiting()) + after := s.WALReclaimStats() + parked := after.CancelledWriterParked - before.CancelledWriterParked + intent := after.CancelledWriteIntent - before.CancelledWriteIntent + other := after.CancelledOther - before.CancelledOther + require.Equal(t, int64(1), parked+intent+other, "one final attempt, not polling/child/cleanup counts") + switch kind { + case "parked", "both": + require.ErrorIs(t, context.Cause(resetCtx), errWALReclaimWriterWaiting) + require.Equal(t, int64(1), parked) + case "intent": + require.ErrorIs(t, context.Cause(resetCtx), errWALReclaimWriterWaiting) + require.Equal(t, int64(1), intent) + case "shutdown": + require.Equal(t, int64(1), other) + } + }) + } +} + +func TestWALReclaimDemandConsumersRetainOneAttemptCause(t *testing.T) { + attempt := &backgroundCheckpointAttempt{} + ctx := context.WithValue(t.Context(), walReclaimAttemptKey{}, attempt) + noteWALReclaimDemand(ctx, walParkedDemand) + noteWALReclaimDemand(ctx, walParkedDemand) + noteWALReclaimDemand(ctx, walIntentDemand) + require.Equal(t, uint32(walReclaimWriterParked), attempt.reclaimInterruption.Load()) + var stats WALReclaimStats + stats.noteInterruption(walReclaimResult{attempt: attempt}) + require.Equal(t, int64(1), stats.CancelledWriterParked) + require.Zero(t, stats.CancelledWriteIntent+stats.CancelledOther) +} + +func TestWALReclaimHelperCleanupAndResetAreNotExtraAttempts(t *testing.T) { + s, _ := openWALReclaimStore(t) + defer s.Close() + attempt, err := s.beginCheckpointAttempt(checkpointIgnoresCycle, true) + require.NoError(t, err) + ctx, stop := s.yieldToWriters(attempt.ctx) + stop() // unconditional helper cleanup: not an operation cancellation + require.ErrorIs(t, context.Cause(ctx), context.Canceled) + require.Zero(t, attempt.reclaimInterruption.Load()) + noteWALReclaimCancellation(attempt, context.DeadlineExceeded) + require.Zero(t, attempt.reclaimInterruption.Load(), "a child budget is not a caller/shutdown cancellation") + s.finishBackgroundCheckpointAttempt(attempt) + require.Zero(t, attempt.reclaimInterruption.Load(), "normal attempt cleanup must not increment causes") + var stats WALReclaimStats + attempt.reclaimInterruption.Store(uint32(walReclaimWriteIntent)) + stats.noteInterruption(walReclaimResult{attempt: attempt, outcome: walReclaimReset}) + require.Zero(t, stats.CancelledWriteIntent, "a successful final reset is not a cancelled attempt") + var cause *walReclaimDemandCause + require.True(t, errors.As(walParkedDemand, &cause)) + require.ErrorIs(t, walParkedDemand, errWALReclaimWriterWaiting) +} + +func TestObservedWALPeakIsSharedPerStoreAndSurvivesReset(t *testing.T) { + t.Setenv("GORTEX_SQLITE_WAL_RECLAIM_MB", "0") + s, _ := openWALReclaimStore(t) + defer s.Close() + other, _ := openWALReclaimStore(t) + defer other.Close() + otherBefore := other.WALReclaimStats() + seedWALChurnTable(t, s) + growWAL(t, s, 4) + _, wal := s.DBStats() + require.Positive(t, wal) + before := s.WALReclaimStats() + require.True(t, before.WALPeakObserved) + require.GreaterOrEqual(t, before.ObservedPeakWALBytes, wal) + require.Equal(t, before.ObservedPeakWALBytes, s.AtGeneration(7).WALReclaimStats().ObservedPeakWALBytes) + func() { + s.writeMu.Lock() + defer s.writeMu.Unlock() + _, err := s.resetWALForReclaim(t.Context()) + require.NoError(t, err) + }() + _, afterWAL := s.DBStats() + require.Less(t, afterWAL, wal) + require.Equal(t, before.ObservedPeakWALBytes, s.WALReclaimStats().ObservedPeakWALBytes, + "TRUNCATE/shrink/status reads cannot reset high-water evidence") + require.Equal(t, otherBefore.ObservedPeakWALBytes, other.WALReclaimStats().ObservedPeakWALBytes) + require.Equal(t, otherBefore.WALPeakObserved, other.WALReclaimStats().WALPeakObserved) + var empty Store + require.False(t, empty.WALReclaimStats().WALPeakObserved) +} + +func TestWALReclaimSuccessfulFinalResetDoesNotCountCleanup(t *testing.T) { + t.Setenv("GORTEX_SQLITE_WAL_RECLAIM_MB", "0") + s, path := openWALReclaimStore(t) + defer s.Close() + seedWALChurnTable(t, s) + growWAL(t, s, 4) + ckpt, err := openWALReclaimCheckpointDB(path) + require.NoError(t, err) + defer ckpt.Close() + res := s.reclaimWALOnce(walReclaimConfig{thresholdBytes: 1 << 30, drainDeadline: time.Second, truncateBudget: walReclaimTruncateBudget, readerWait: time.Second}, ckpt, path+"-wal") + require.Equal(t, walReclaimReset, res.outcome) + stats := s.WALReclaimStats() + require.Zero(t, stats.CancelledWriterParked+stats.CancelledWriteIntent+stats.CancelledOther, "normal operation/yield-context cleanup is not a cancelled attempt") + memory, err := Open(":memory:") + require.NoError(t, err) + defer memory.Close() + memory.DBStats() + memory.noteObservedWALBytes(64) + require.False(t, memory.WALReclaimStats().WALPeakObserved, "in-memory stores have no physical WAL observation") +} diff --git a/internal/graph/store_sqlite/wal_reclaim_pressure.go b/internal/graph/store_sqlite/wal_reclaim_pressure.go index 07b6759a0..8e2b2f6c6 100644 --- a/internal/graph/store_sqlite/wal_reclaim_pressure.go +++ b/internal/graph/store_sqlite/wal_reclaim_pressure.go @@ -186,9 +186,10 @@ func (s *Store) reclaimWALPressureResetOnce(ctx context.Context, ckptDB *sql.DB, res.recordWriterCredit(writer, adaptiveCopy) }() if preemptible { - if s.writeWanted() { + if demand := s.writeDemandCause(); demand != nil { + noteWALReclaimDemand(ctx, demand) res.reason = "pressure_foreground_writer" - return errWALReclaimWriterWaiting + return demand } } if s.bulkConn != nil && !res.leaseOverride { @@ -240,9 +241,10 @@ func (s *Store) reclaimWALPressureResetOnce(ctx context.Context, ckptDB *sql.DB, return errWALPressureHold } // Any slow-copy readmission has finished; no own waiter is visible here. - if preemptible && s.writeWanted() { + if demand := s.writeDemandCause(); preemptible && demand != nil { + noteWALReclaimDemand(ctx, demand) res.reason = "pressure_foreground_writer" - return errWALReclaimWriterWaiting + return demand } resetCtx := writer.resetContext(hctx) if preemptible { @@ -255,6 +257,7 @@ func (s *Store) reclaimWALPressureResetOnce(ctx context.Context, ckptDB *sql.DB, } result, err := s.resetWALForPressure(resetCtx) if preemptible && err != nil && errors.Is(context.Cause(resetCtx), errWALReclaimWriterWaiting) { + noteWALReclaimDemand(ctx, context.Cause(resetCtx)) err = errWALReclaimWriterWaiting } res.pressureResetBusy = result.Busy > 0 && errors.Is(err, errSQLiteCheckpointIncomplete) @@ -274,9 +277,10 @@ func (s *Store) reclaimWALPressureResetOnce(ctx context.Context, ckptDB *sql.DB, // share the configured drain allowance; the writer still gets at most its // existing short credit and remaining aggregate allowance. func (s *Store) reclaimWALPressureResetAfterReaderDrain(ctx context.Context, ckptDB *sql.DB, res *walReclaimResult, allowance time.Duration) error { - if s.writeWanted() { + if demand := s.writeDemandCause(); demand != nil { + noteWALReclaimDemand(ctx, demand) res.reason = "pressure_foreground_writer" - return errWALReclaimWriterWaiting + return demand } started := time.Now() pauseCtx, cancel := context.WithDeadline(ctx, started.Add(allowance)) @@ -292,6 +296,7 @@ func (s *Store) reclaimWALPressureResetAfterReaderDrain(ctx context.Context, ckp res.pauseClosed = true } if errors.Is(context.Cause(yieldCtx), errWALReclaimWriterWaiting) { + noteWALReclaimDemand(ctx, context.Cause(yieldCtx)) err = errWALReclaimWriterWaiting } res.reason = fmt.Sprintf("pressure_reader_drain readers=%d error=%v", inFlight, err) @@ -314,14 +319,16 @@ func (s *Store) reclaimWALPressureResetAfterReaderDrain(ctx context.Context, ckp }() if err := yieldCtx.Err(); err != nil { if errors.Is(context.Cause(yieldCtx), errWALReclaimWriterWaiting) { + noteWALReclaimDemand(ctx, context.Cause(yieldCtx)) err = errWALReclaimWriterWaiting } res.reason = fmt.Sprintf("pressure_reader_drain error=%v", err) return err } - if s.writeWanted() { + if demand := s.writeDemandCause(); demand != nil { + noteWALReclaimDemand(ctx, demand) res.reason = "pressure_foreground_writer" - return errWALReclaimWriterWaiting + return demand } credit := min(s.walPressureHold(), walReclaimMaxWriterHold-res.writerSpent, time.Until(started.Add(allowance))) if credit <= 0 { diff --git a/internal/graph/store_sqlite/wal_reclaim_request_test.go b/internal/graph/store_sqlite/wal_reclaim_request_test.go index 48369aa14..5e87fce3c 100644 --- a/internal/graph/store_sqlite/wal_reclaim_request_test.go +++ b/internal/graph/store_sqlite/wal_reclaim_request_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "os" + "sync" "sync/atomic" "testing" "time" @@ -342,6 +343,14 @@ func TestARefusedFoldGetsItsResetInBoundedTimeWithTheDaemonDefaults(t *testing.T _ = churnWriteOnce(s, k) } }() + var stopWriterOnce sync.Once + stopWriter := func() { + stopWriterOnce.Do(func() { + close(stop) + <-writerDone + }) + } + defer stopWriter() // Also join when an assertion exits the fold loop. resets := s.WALReclaimStats().Resets var requested time.Time var reset time.Duration @@ -361,8 +370,7 @@ func TestARefusedFoldGetsItsResetInBoundedTimeWithTheDaemonDefaults(t *testing.T } time.Sleep(20 * time.Millisecond) } - close(stop) - <-writerDone + stopWriter() cp := s.WALCopyStats() t.Logf("fold refused at log=%.0fMiB; reset %s after the request; pressure_runs=%d lane_resets=%d give_ups=%d", float64(chainFoldWALMark+(64<<20))/(1<<20), reset.Round(time.Millisecond), cp.PressureRuns, cp.PressureLaneResets, cp.PressureGiveUps) diff --git a/internal/graph/store_sqlite/wal_reclaim_writer_credit.go b/internal/graph/store_sqlite/wal_reclaim_writer_credit.go index 7da30645d..3315c067e 100644 --- a/internal/graph/store_sqlite/wal_reclaim_writer_credit.go +++ b/internal/graph/store_sqlite/wal_reclaim_writer_credit.go @@ -155,8 +155,9 @@ func (w *walReclaimWriterCredit) passive(ctx, operationCtx context.Context, db * if w.readmitted || unused <= 0 || operationCtx.Err() != nil || !errors.Is(ctx.Err(), context.DeadlineExceeded) || !errors.Is(context.Cause(ctx), context.DeadlineExceeded) { return result, ctx.Err() } - if w.yieldToWriters && w.store.writeWanted() { - return result, errWALReclaimWriterWaiting + if demand := w.store.writeDemandCause(); w.yieldToWriters && demand != nil { + noteWALReclaimDemand(operationCtx, demand) + return result, demand } limit := time.Now().Add(unused) if operationLimit, ok := opCtx.Deadline(); ok && operationLimit.Before(limit) { diff --git a/internal/graph/store_sqlite/wal_shrink.go b/internal/graph/store_sqlite/wal_shrink.go index 86102c5e0..f2f7d2f0d 100644 --- a/internal/graph/store_sqlite/wal_shrink.go +++ b/internal/graph/store_sqlite/wal_shrink.go @@ -101,7 +101,7 @@ func (s *Store) resetWALForReclaimWithBusyTimeout(ctx context.Context, busyMilli return walCheckpointResult{}, err } walPath := s.dbPath + "-wal" - if walFileSize(walPath) < walShrinkInPlaceBytes { + if s.observedWALFileSize(walPath) < walShrinkInPlaceBytes { var result walCheckpointResult attempted := false err := s.withWriterResetConnWithBusyTimeout(ctx, busyMillis, func(conn *sql.Conn) error { @@ -354,7 +354,7 @@ func (s *Store) shrinkWALSliceOnce(ctx context.Context, ckptDB *sql.DB, slice in } used := int64(32) + int64(snap.MxFrame)*(24+pageSize) floor := max(used, walShrinkKeepBytes) - out.before = walFileSize(walPath) + out.before = s.observedWALFileSize(walPath) out.after = out.before if out.before <= floor { out.done = true @@ -416,7 +416,7 @@ func (s *Store) shrinkWAL(ctx context.Context, ckptDB *sql.DB) (slices int, shru // pager setting fails after the file reaches its floor. s.walShrinkPending.Store(true) walPath := s.dbPath + "-wal" - startBytes := walFileSize(walPath) + startBytes := s.observedWALFileSize(walPath) started := time.Now() slice := min(walShrinkFirstSlice, walShrinkSliceCap) var maxHold time.Duration @@ -435,7 +435,7 @@ func (s *Store) shrinkWAL(ctx context.Context, ckptDB *sql.DB) (slices int, shru } step, serr := s.shrinkWALSliceOnce(ctx, ckptDB, slice) if serr != nil { - log.Printf("store_sqlite: wal shrink paused after %d slices error=%q wal_bytes=%d", slices, serr, walFileSize(walPath)) + log.Printf("store_sqlite: wal shrink paused after %d slices error=%q wal_bytes=%d", slices, serr, s.observedWALFileSize(walPath)) return slices, shrunk, serr } if step.after < step.before { @@ -472,7 +472,7 @@ func (s *Store) shrinkWAL(ctx context.Context, ckptDB *sql.DB) (slices int, shru } if slices > 0 { log.Printf("store_sqlite: wal shrunk bytes_before=%d bytes_after=%d slices=%d max_slice_hold=%s elapsed=%s", - startBytes, walFileSize(walPath), slices, maxHold.Round(time.Millisecond), time.Since(started).Round(time.Millisecond)) + startBytes, s.observedWALFileSize(walPath), slices, maxHold.Round(time.Millisecond), time.Since(started).Round(time.Millisecond)) } return slices, shrunk, nil } diff --git a/internal/hooks/pretooluse.go b/internal/hooks/pretooluse.go index 665c2dee5..987760724 100644 --- a/internal/hooks/pretooluse.go +++ b/internal/hooks/pretooluse.go @@ -1205,7 +1205,7 @@ func readOnlyNativeAllowance(writing bool) enrichResult { // hook lets it through because the path's repository is still loading. func repoLoadingNativeAllowance(repo string) string { if repo == "" { - repo = "this repository" + return "[Gortex] Graph admission for this path is still resolving and is not yet established (repo_loading, identity_pending). Native tools are allowed while this temporary state resolves. Retry shortly without changing tracking configuration.\n" } return fmt.Sprintf("[Gortex] %s is still loading after a restart; graph tools return repo_loading until it is ready. "+ "Native reads are allowed until then.\n", repo) @@ -1314,7 +1314,7 @@ func fileIndexStatusFromCoverage(result daemon.FileCoverageResult) fileIndexStat st.ProbeOK, st.ReadOnly, st.MutationKnown, st.MutationReason = true, true, true, result.MutationReason } st.MutationKnown = st.MutationKnown || result.MutationState == "ready" - if result.RepoState == daemon.RepoStateLoading { + if result.RepoState == daemon.RepoStateLoading || result.RepoState == daemon.RepoStateIdentityPending { // Coverage read from a loading repository's persisted rows is not a // reason to deny: report it as loading, never as indexed. st.ProbeOK, st.Loading, st.LoadingRepo = true, true, result.Repo @@ -1640,7 +1640,7 @@ var dirCoverageFn = dirCoverageViaDaemon // Only a completed walk that claimed nothing proves a scope holds no source. // From the graph alone a scope mid-walk looks exactly like an excluded one. func scopeTrackedFromCoverage(result daemon.DirCoverageResult) (hasSource, probeOK bool) { - if result.RepoState == daemon.RepoStateLoading { + if result.RepoState == daemon.RepoStateLoading || result.RepoState == daemon.RepoStateIdentityPending { // A loading repository's scope holds nothing the graph will answer // for yet, whatever its persisted rows say (see fileIndexStatus). return false, true diff --git a/internal/hooks/repo_loading_test.go b/internal/hooks/repo_loading_test.go index a3554a72d..21a6725e3 100644 --- a/internal/hooks/repo_loading_test.go +++ b/internal/hooks/repo_loading_test.go @@ -55,3 +55,31 @@ func TestLoadingRepo_SearchScopeAllowed(t *testing.T) { t.Fatalf("a loading repository's scope must read as holding nothing the graph answers for: hasSource=%v probeOK=%v", hasSource, probeOK) } } + +func TestIdentityPending_NativeDoorsDoNotClaimOwnership(t *testing.T) { + st := fileIndexStatusFromCoverage(daemon.FileCoverageResult{RepoState: daemon.RepoStateIdentityPending}) + if !st.Loading || st.Indexed || st.Tracked || st.LoadingRepo != "" { + t.Fatalf("uncertain identity is not an indexed or tracked verdict: %#v", st) + } + stubFileIndexScope(t, st) + for name, res := range map[string]enrichResult{ + "Read": enrichRead(map[string]any{"file_path": "internal/graph/graph.go"}, "/repo"), + "Edit": enrichEdit(map[string]any{"file_path": "internal/graph/graph.go"}, "/repo"), + "Bash cat": enrichBash(map[string]any{"command": "cat internal/graph/graph.go"}, "/repo"), + } { + if res.deny { + t.Fatalf("%s denied while graph admission is unresolved: %#v", name, res) + } + if strings.Contains(res.context, "this repository is still loading") || strings.Contains(res.context, "already configured") { + t.Fatalf("%s invented repository ownership: %s", name, res.context) + } + } + read := enrichRead(map[string]any{"file_path": "internal/graph/graph.go"}, "/repo") + if !strings.Contains(read.context, "identity_pending") { + t.Fatalf("missing honest reason: %s", read.context) + } + hasSource, ok := scopeTrackedFromCoverage(daemon.DirCoverageResult{RepoState: daemon.RepoStateIdentityPending}) + if hasSource || !ok { + t.Fatalf("pending directory must not redirect to graph: source=%v known=%v", hasSource, ok) + } +} diff --git a/internal/indexer/affected_by_bound_test.go b/internal/indexer/affected_by_bound_test.go index 812e5518c..95bcd9fb4 100644 --- a/internal/indexer/affected_by_bound_test.go +++ b/internal/indexer/affected_by_bound_test.go @@ -640,7 +640,7 @@ func enqueueAffectedByPointMutationWith( // // Revert-red: remove the derived-fan-out window from // patchGraphWithReceiptStateRawModern (or stop threading it into -// completeMutationWaitersWithFanout, or drop the field from MutationResult) +// completeMutationWaitersObserved, or drop the field from MutationResult) // and a knowingly incomplete mutation reports Observed=false — the shape in // which the fact reached the receipt and then reached no consumer at all. func TestAdmittedPointMutationReportsTheDerivedFanoutCut(t *testing.T) { @@ -843,7 +843,7 @@ func TestFailedPointMutationPublishesNoDerivedFanoutVerdict(t *testing.T) { t.Run("executor fails after the bounded pass ran", func(t *testing.T) { result, observed, _ := enqueueAffectedByPointMutationWith(t, newSqliteGraph(t), 3, 2, func(idx *Indexer, w *Watcher) { - w.pointReindexRaw = func(path string) (*IndexResult, error) { + w.pointReindexRaw = func(path string, _ *PublicationStamps) (*IndexResult, error) { // Run the real executor first so the bounded pass records a // genuine cut into this mutation's window, then fail. _, _ = idx.incrementalPointWatcherPath(idx.rootPath, path) @@ -864,7 +864,7 @@ func TestFailedPointMutationPublishesNoDerivedFanoutVerdict(t *testing.T) { t.Run("patch fails after the executor returned", func(t *testing.T) { result, observed, _ := enqueueAffectedByPointMutationWith(t, newSqliteGraph(t), 3, 2, func(idx *Indexer, w *Watcher) { - w.pointReindexRaw = func(path string) (*IndexResult, error) { + w.pointReindexRaw = func(path string, _ *PublicationStamps) (*IndexResult, error) { out, err := idx.incrementalPointWatcherPath(idx.rootPath, path) if out != nil { out.FailedFiles = append(out.FailedFiles, path) @@ -968,7 +968,7 @@ func TestPatchGraphObservingFanoutStampsNothingOnAFailedPatch(t *testing.T) { idx.SetRootPath(dir) w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 5}, zap.NewNop()) require.NoError(t, err) - w.pointReindexRaw = func(path string) (*IndexResult, error) { + w.pointReindexRaw = func(path string, _ *PublicationStamps) (*IndexResult, error) { // The bounded pass runs for real and is cut, so a verdict exists. _, _ = idx.incrementalPointWatcherPath(idx.rootPath, path) return nil, errors.New("watcher-test: executor failed after the bounded pass") diff --git a/internal/indexer/canonical_admission.go b/internal/indexer/canonical_admission.go new file mode 100644 index 000000000..b8f4641d7 --- /dev/null +++ b/internal/indexer/canonical_admission.go @@ -0,0 +1,154 @@ +package indexer + +import "time" + +// Two fixed admission envelopes, not a tracing vocabulary or interval history. +// Both precede the executor and remain separate from its deep patch laps. +type canonicalAdmissionKind uint8 + +const ( + canonicalCohortAdmission canonicalAdmissionKind = iota + canonicalBatchAdmission + canonicalAdmissionCount +) + +var canonicalAdmissionNames = [...]string{"cohort_admission", "batch_admission"} + +type canonicalAdmissionPair struct { + started, finished time.Time + invalid bool +} + +type canonicalAdmissionSpan struct { + stamps *PublicationStamps + kind canonicalAdmissionKind +} + +type canonicalAdmissionEvidence struct { + pairs [canonicalAdmissionCount]struct { + startNS, endNS int64 + started, finished, invalid bool + } +} + +func (s *PublicationStamps) beginCanonicalAdmission(kind canonicalAdmissionKind) canonicalAdmissionSpan { + if s == nil || s.patchLaps == nil { + return canonicalAdmissionSpan{} + } + return s.beginCanonicalAdmissionAt(kind, time.Now()) +} + +func (s *PublicationStamps) beginCanonicalAdmissionAt(kind canonicalAdmissionKind, at time.Time) canonicalAdmissionSpan { + s.mu.Lock() + defer s.mu.Unlock() + pair := &s.admissions[kind] + if at.IsZero() || !pair.started.IsZero() { + pair.invalid = true + } else { + pair.started = at + } + s.patchLaps.beginObservedOuterLocked(patchCohortAdmission+canonicalPatchLap(kind), at) + return canonicalAdmissionSpan{stamps: s, kind: kind} +} + +func (span *canonicalAdmissionSpan) end() { + if span.stamps == nil { + return + } + span.endAt(time.Now()) +} + +func (span *canonicalAdmissionSpan) endAt(at time.Time) { + s := span.stamps + if s == nil { + return + } + span.stamps = nil + s.mu.Lock() + defer s.mu.Unlock() + pair := &s.admissions[span.kind] + if at.IsZero() || pair.started.IsZero() || at.Before(pair.started) || !pair.finished.IsZero() { + pair.invalid = true + return + } + pair.finished = at + s.patchLaps.endObservedOuterLocked(patchCohortAdmission+canonicalPatchLap(span.kind), pair.started, at) +} + +func (s *PublicationStamps) freezeCanonicalAdmissions(origin time.Time) canonicalAdmissionEvidence { + var out canonicalAdmissionEvidence + for i, pair := range s.admissions { + v := &out.pairs[i] + v.started, v.finished, v.invalid = !pair.started.IsZero(), !pair.finished.IsZero(), pair.invalid + if v.started { + v.startNS = pair.started.Sub(origin).Nanoseconds() + } + if v.finished { + v.endNS = pair.finished.Sub(origin).Nanoseconds() + } + } + return out +} + +func (p canonicalAdmissionEvidence) project(offsets map[PublicationPhase]int64, terminal bool, lane *canonicalLaneWaitEvidence, out *EditPhaseTimings) { + enqueue, hasEnqueue := offsets[PublicationTicketEnqueued] + cycle, hasCycle := offsets[PublicationCycleStarted] + executor, hasExecutor := offsets[PublicationPatchExecutionStarted] + invalid := false + for _, pair := range p.pairs { + if pair.invalid || pair.started && pair.startNS < 0 || pair.finished && (!pair.started || pair.endNS < pair.startNS) { + invalid = true + } + } + cohort, batch := p.pairs[canonicalCohortAdmission], p.pairs[canonicalBatchAdmission] + if cohort.finished && batch.started && cohort.endNS > batch.startNS { + invalid = true + } + if lane != nil && lane.started { + for _, pair := range p.pairs { + if pair.finished && pair.endNS > lane.startNS { + invalid = true + } + } + } + for i, name := range canonicalAdmissionNames { + pair := p.pairs[i] + out.Milliseconds[name], out.States[name] = nil, "not_observed" + if invalid { + out.States[name] = "invalid_order" + continue + } + if !pair.started { + continue + } + if !pair.finished { + if !terminal { + out.States[name] = "pending" + } + continue + } + if !hasEnqueue || !hasCycle || !hasExecutor { + continue + } + if enqueue < 0 || cycle < enqueue || pair.startNS < cycle || pair.endNS > executor { + out.States[name] = "invalid_order" + continue + } + if terminal { + last := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + last = PublicationTicketFailed + } + end, known := offsets[last] + if !known { + continue + } + if pair.endNS > end { + out.States[name] = "invalid_order" + continue + } + } + ms := float64(pair.endNS-pair.startNS) / 1e6 + out.Milliseconds[name], out.States[name] = &ms, "measured" + } +} diff --git a/internal/indexer/canonical_admission_test.go b/internal/indexer/canonical_admission_test.go new file mode 100644 index 000000000..1ffdb12ed --- /dev/null +++ b/internal/indexer/canonical_admission_test.go @@ -0,0 +1,173 @@ +package indexer + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCanonicalAdmissionObservesRealCohortGrant(t *testing.T) { + path, _, w := pointPublicationFixture(t) + slots := w.mutationSlots() + for i := 0; i < cap(slots); i++ { + slots <- struct{}{} + } + t.Cleanup(func() { + if !w.mutationAdmissionStopped() { + require.NoError(t, w.Stop()) + } + for len(slots) > 0 { + <-slots + } + }) + entered := make(chan struct{}, 1) + w.mutationAdmissionWaitFn = func() time.Duration { + select { + case entered <- struct{}{}: + default: + } + return 5 * time.Second + } + record := newPointPublication(t, "cohort-admission") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + awaitPublicationSignal(t, entered) + require.Nil(t, record.Snapshot().admissions, "blocked unvalidated attempt cannot publish") + grantedAfter := time.Now() + <-slots // actual occupied semaphore cannot grant before this release + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + snapshot := record.Snapshot() + require.NotNil(t, snapshot.admissions) + pair := snapshot.admissions.pairs[canonicalCohortAdmission] + require.True(t, pair.started) + require.True(t, pair.finished) + require.GreaterOrEqual(t, pair.endNS, grantedAfter.Sub(record.origin).Nanoseconds()) + require.Equal(t, "measured", snapshot.EditTimings().States["cohort_admission"]) + require.NotNil(t, snapshot.EditTimings().Milliseconds["cohort_admission"]) +} + +func TestCanonicalAdmissionObservesRealBatchLockBeforeLane(t *testing.T) { + record := newPointPublication(t, "batch-admission") + require.True(t, record.bindCanonicalTicket(1, time.Now())) + stamps := record.newCanonicalPatchStamps() + stamps.markAt(PublicationCycleStarted, time.Now()) + ctx, cancel := context.WithCancel(context.WithValue(t.Context(), publicationStampsKey{}, stamps)) + var gate sync.RWMutex + gate.Lock() + var once sync.Once + release := func() { once.Do(gate.Unlock) } + coordinator := newRepositoryMutationCoordinator(nil) + coordinator.batchMutationGate = &gate + done := make(chan error, 1) + finished := make(chan struct{}) + go func() { + defer close(finished) + done <- coordinator.runExclusive(ctx, func() error { + stamps.markAt(PublicationPatchExecutionStarted, time.Now()) + stamps.markAt(PublicationPatchExecutionFinished, time.Now()) + return nil + }) + }() + t.Cleanup(func() { + cancel() + release() + select { + case <-finished: + case <-time.After(2 * time.Second): + t.Error("batch admission did not join") + } + }) + require.Eventually(t, func() bool { + stamps.mu.Lock() + defer stamps.mu.Unlock() + return !stamps.admissions[canonicalBatchAdmission].started.IsZero() + }, 2*time.Second, time.Millisecond) + stamps.mu.Lock() + unfinished := stamps.admissions[canonicalBatchAdmission].finished.IsZero() + laneNotStarted := stamps.laneWait.started.IsZero() + stamps.mu.Unlock() + require.True(t, unfinished) + require.True(t, laneNotStarted, "batch lock must not be charged to lane_wait") + grantedAfter := time.Now() + release() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("batch lock did not grant") + } + record.Absorb(stamps) + record.Mark(PublicationTicketCompleted) + snapshot := record.Snapshot() + pair := snapshot.admissions.pairs[canonicalBatchAdmission] + require.GreaterOrEqual(t, pair.endNS, grantedAfter.Sub(record.origin).Nanoseconds()) + require.LessOrEqual(t, pair.endNS, snapshot.laneWait.startNS) + require.Equal(t, "measured", snapshot.EditTimings().States["batch_admission"]) +} + +func TestCanonicalAdmissionProjectionRejectsOverlapAndMissingOwners(t *testing.T) { + for _, name := range []string{"measured", "zero", "cohort_batch_overlap", "batch_lane_overlap", "before_cycle", "after_executor", "missing_executor", "missing_terminal", "outside_terminal", "incomplete", "negative", "repeated"} { + t.Run(name, func(t *testing.T) { + var p canonicalAdmissionEvidence + p.pairs[0].started, p.pairs[0].finished = true, true + p.pairs[0].startNS, p.pairs[0].endNS = 2, 3 + p.pairs[1].started, p.pairs[1].finished = true, true + p.pairs[1].startNS, p.pairs[1].endNS = 4, 5 + lane := canonicalLaneWaitEvidence{started: true, acquired: true, startNS: 6, acquiredNS: 7} + offsets := map[PublicationPhase]int64{PublicationTicketEnqueued: 0, PublicationCycleStarted: 1, PublicationPatchExecutionStarted: 8, PublicationTicketCompleted: 9} + terminal := true + want := "invalid_order" + switch name { + case "measured": + want = "measured" + case "zero": + p.pairs[0].endNS = 2 + want = "measured" + case "cohort_batch_overlap": + p.pairs[0].endNS = 5 + case "batch_lane_overlap": + p.pairs[1].endNS = 7 + case "before_cycle": + p.pairs[0].startNS = 0 + case "after_executor": + offsets[PublicationPatchExecutionStarted] = 2 + case "missing_executor": + delete(offsets, PublicationPatchExecutionStarted) + want = "not_observed" + case "missing_terminal": + delete(offsets, PublicationTicketCompleted) + want = "not_observed" + case "outside_terminal": + offsets[PublicationTicketCompleted] = 2 + case "incomplete": + p.pairs[0].finished = false + terminal = false + want = "pending" + case "negative": + p.pairs[0].startNS = -1 + case "repeated": + p.pairs[0].invalid = true + } + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + p.project(offsets, terminal, &lane, &out) + require.Equal(t, want, out.States["cohort_admission"]) + if want == "measured" { + require.NotNil(t, out.Milliseconds["cohort_admission"]) + } else { + require.Nil(t, out.Milliseconds["cohort_admission"]) + } + }) + } +} + +func TestCanonicalAdmissionNilAndForeignObserversDoNotAllocate(t *testing.T) { + for _, stamps := range []*PublicationStamps{nil, {}} { + var span canonicalAdmissionSpan + allocations := testing.AllocsPerRun(100, func() { span = stamps.beginCanonicalAdmission(canonicalCohortAdmission); span.end() }) + require.Nil(t, span.stamps) + require.Zero(t, allocations) + } +} diff --git a/internal/indexer/canonical_attempt_accounting.go b/internal/indexer/canonical_attempt_accounting.go new file mode 100644 index 000000000..081e7d5f4 --- /dev/null +++ b/internal/indexer/canonical_attempt_accounting.go @@ -0,0 +1,272 @@ +package indexer + +import ( + "time" + + "go.uber.org/zap/zapcore" +) + +var canonicalAttemptLapNames = [...]string{ + "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", "patch_analysis", + "attempt_setup", "patch_inspection", "executor_control", "attempt_finalize", "cohort_admission", "batch_admission", "lane_wait", +} + +type canonicalAttemptEvidence struct { + laps [canonicalPatchLapCount]canonicalLapEvidence + startNS, endNS int64 + observed, invalid bool + nativeMtimeDuplicate bool + executorStart, executorEnd, derivedStart, derivedEnd int64 + hasExecutorStart, hasExecutorEnd, hasDerivedStart, hasDerivedEnd bool +} + +// New immutable evidence is copied at actual return, never into an already +// exposed early snapshot. Caller holds the collector lock. +func (s *PublicationStamps) freezeAttemptEvidence(origin time.Time) canonicalAttemptEvidence { + p := s.patchLaps + out := canonicalAttemptEvidence{observed: p.controlObserved && p.controlFinished, invalid: p.invalid || p.active != 0, nativeMtimeDuplicate: p.nativeMtimeDuplicate} + if !p.attemptStarted.IsZero() { + out.startNS = p.attemptStarted.Sub(origin).Nanoseconds() + } + if !p.attemptFinished.IsZero() { + out.endNS = p.attemptFinished.Sub(origin).Nanoseconds() + } + for i, v := range p.laps { + out.laps[i] = canonicalLapEvidence{totalNS: v.total.Nanoseconds(), begun: v.begun, done: v.done, incomplete: v.incomplete} + if v.begun > 0 { + out.laps[i].firstNS = v.first.Sub(origin).Nanoseconds() + out.laps[i].lastNS = v.last.Sub(origin).Nanoseconds() + } + } + for _, mark := range s.marks { + at := mark.at.Sub(origin).Nanoseconds() + switch mark.phase { + case PublicationPatchExecutionStarted: + if !out.hasExecutorStart { + out.executorStart, out.hasExecutorStart = at, true + } + case PublicationPatchExecutionFinished: + if !out.hasExecutorEnd { + out.executorEnd, out.hasExecutorEnd = at, true + } + case PublicationPatchDerivedStarted: + if !out.hasDerivedStart { + out.derivedStart, out.hasDerivedStart = at, true + } + case PublicationPatchDerivedFinished: + if !out.hasDerivedEnd { + out.derivedEnd, out.hasDerivedEnd = at, true + } + } + } + return out +} + +func (p *canonicalAttemptEvidence) agreesWithPair(kind canonicalPatchLap, start, end int64, started, finished, invalid bool) bool { + v := p.laps[kind] + if invalid || started != (v.begun > 0) || finished != (v.done > 0) || v.begun > 1 || v.done > 1 { + return false + } + if !started { + return true + } + return v.firstNS == start && (!finished || v.lastNS == end && v.totalNS == end-start) +} + +func (p *canonicalAttemptEvidence) agreesWithAdmissions(admissions canonicalAdmissionEvidence, lane canonicalLaneWaitEvidence) bool { + for i, pair := range admissions.pairs { + if !p.agreesWithPair(patchCohortAdmission+canonicalPatchLap(i), pair.startNS, pair.endNS, pair.started, pair.finished, pair.invalid) { + return false + } + } + return p.agreesWithPair(patchRepositoryLane, lane.startNS, lane.acquiredNS, lane.started, lane.acquired, lane.invalid) +} + +type canonicalAttemptProjection struct { + states [canonicalPatchLapCount]string + values [canonicalPatchLapCount]float64 + sumNS int64 + state string + complete bool +} + +func requiredAttemptLap(kind canonicalPatchLap) bool { + // Missing requested deep evidence is not made applicable/complete by a broad + // control interval. Analysis and a nonexistent batch lock/hook are optional. + return kind < patchBookkeeping || isAttemptControl(kind) || kind == patchCohortAdmission || kind == patchRepositoryLane +} + +func (w canonicalAttemptWindow) evaluateAccounting() canonicalAttemptProjection { + return w.evaluateAccountingFor(false) +} + +// The two existing owners share ordered scalar validation, not their scope. +// Read repair begins after admission/lane acquisition and cannot borrow those +// P intervals. Its one actual control interval surrounds the named operations. +func (w canonicalAttemptWindow) evaluateAccountingFor(readRepair bool) canonicalAttemptProjection { + p := w.accounting + out := canonicalAttemptProjection{state: "not_observed"} + for i := range out.states { + out.states[i] = "not_observed" + } + if !p.observed { + return out + } + if w.startNS < 0 || w.endNS < w.startNS { + out.state = "invalid_order" + for i := range out.states { + out.states[i] = "invalid_order" + } + return out + } + invalid := p.invalid || p.startNS != w.startNS || p.endNS != w.endNS || w.startNS < 0 || w.endNS < w.startNS + invalid = invalid || readRepair && p.nativeMtimeDuplicate + wall := w.endNS - w.startNS + missing := !p.hasExecutorStart || !p.hasExecutorEnd || !p.hasDerivedStart || !p.hasDerivedEnd + if p.nativeMtimeDuplicate { + missing = false + // A native duplicate never entered the executor. Any execution evidence + // contradicts that disposition; it cannot be reclassified as zero work. + invalid = invalid || p.hasExecutorStart || p.hasExecutorEnd || p.hasDerivedStart || p.hasDerivedEnd + } + if p.hasExecutorStart && (p.executorStart < w.startNS || p.executorStart > w.endNS) || p.hasExecutorEnd && (p.executorEnd < p.executorStart || p.executorEnd > w.endNS) { + invalid = true + } + for i, v := range p.laps { + kind := canonicalPatchLap(i) + if readRepair && kind >= canonicalDeepLapCount && kind != patchExecutorControl { + if v != (canonicalLapEvidence{}) { + invalid = true + } + continue + } + if p.nativeMtimeDuplicate && (kind < canonicalDeepLapCount || kind == patchExecutorControl) { + if v != (canonicalLapEvidence{}) { + invalid = true + } + out.states[i] = "not_applicable" + continue + } + required := requiredAttemptLap(kind) + if readRepair { + required = kind < patchBookkeeping || kind == patchExecutorControl + } + if required && (v.begun == 0 || v.begun != v.done || v.incomplete) { + missing = true + } + if v.incomplete || v.begun != v.done { + missing = true + } + if v.begun == 0 { + continue + } + if v.totalNS < 0 || v.firstNS < w.startNS || v.lastNS < v.firstNS || v.lastNS > w.endNS || v.totalNS > v.lastNS-v.firstNS || v.done > v.begun { + invalid = true + } + if kind < canonicalDeepLapCount && (p.hasExecutorStart && v.firstNS < p.executorStart || p.hasExecutorEnd && v.lastNS > p.executorEnd) { + invalid = true + } + if v.begun != v.done || v.incomplete { + continue + } + if v.totalNS < 0 || v.totalNS > wall || out.sumNS > wall-v.totalNS { + invalid = true + continue + } + out.sumNS += v.totalNS + out.states[i] = "measured" + } + derived := p.laps[patchDerived] + if p.hasDerivedStart || p.hasDerivedEnd { + if !p.hasDerivedStart || !p.hasDerivedEnd || derived.begun != 1 || derived.done != 1 || derived.firstNS != p.derivedStart || derived.lastNS != p.derivedEnd || derived.totalNS != p.derivedEnd-p.derivedStart { + invalid = true + } + } + if invalid { + out.state = "invalid_order" + for i := range out.states { + out.states[i] = "invalid_order" + } + return out + } + for i := range out.values { + if out.states[i] == "measured" { + out.values[i] = float64(p.laps[i].totalNS) / 1e6 + } + } + switch { + case missing: + out.state = "missing_laps" + case out.sumNS != wall: + out.state = "unclassified_gap" + default: + out.state, out.complete = "partial", true + } + return out +} + +// EditPatchAccounting describes only one accepted winning P, not quiet waits, +// backoff/discarded attempts, receipt lifetime, ACK or whole-edit background. +// Even an observed complete partition is not the actual100-edit/5% acceptance. +type EditPatchAccounting struct { + ExecutionDisposition string `json:"execution_disposition"` + AttemptTicket uint64 `json:"attempt_ticket"` + AttemptRetry int `json:"attempt_retry"` + Scope string `json:"scope"` + BackgroundCoverage string `json:"background_coverage"` + SumState string `json:"sum_state"` + PartitionComplete bool `json:"observed_partition_complete"` + PhasesMS map[string]*float64 `json:"phases_ms"` + PhaseStates map[string]string `json:"phase_states"` + SubtotalMS *float64 `json:"observed_subtotal_ms"` + UnattributedMS *float64 `json:"unattributed_ms"` +} + +func (w canonicalAttemptWindow) projectAccounting() *EditPatchAccounting { + p := w.evaluateAccounting() + out := &EditPatchAccounting{AttemptTicket: w.ticket, AttemptRetry: w.retry, Scope: "accepted_winning_physical_attempt", BackgroundCoverage: "not_established", SumState: p.state, PartitionComplete: p.complete, + PhasesMS: make(map[string]*float64, canonicalPatchLapCount), PhaseStates: make(map[string]string, canonicalPatchLapCount)} + out.ExecutionDisposition = w.executionDisposition() + for i, name := range canonicalAttemptLapNames { + out.PhasesMS[name], out.PhaseStates[name] = nil, p.states[i] + if p.states[i] == "measured" { + ms := p.values[i] + out.PhasesMS[name] = &ms + } + } + if p.state != "not_observed" && p.state != "invalid_order" { + subtotal := float64(p.sumNS) / 1e6 + residual := float64(w.endNS-w.startNS-p.sumNS) / 1e6 + out.SubtotalMS, out.UnattributedMS = &subtotal, &residual + } + return out +} + +func (w canonicalAttemptWindow) MarshalLogObject(enc zapcore.ObjectEncoder) error { + p := w.evaluateAccounting() + enc.AddString("scope", "accepted_winning_physical_attempt") + enc.AddString("execution_disposition", w.executionDisposition()) + enc.AddUint64("attempt_ticket", w.ticket) + enc.AddInt("attempt_retry", w.retry) + enc.AddString("background_coverage", "not_established") + enc.AddString("sum_state", p.state) + enc.AddBool("observed_partition_complete", p.complete) + for i, name := range canonicalAttemptLapNames { + enc.AddString(name+"_state", p.states[i]) + if p.states[i] == "measured" { + enc.AddFloat64(name+"_ms", p.values[i]) + } + } + if p.state != "not_observed" && p.state != "invalid_order" { + enc.AddFloat64("observed_subtotal_ms", float64(p.sumNS)/1e6) + enc.AddFloat64("unattributed_ms", float64(w.endNS-w.startNS-p.sumNS)/1e6) + } + return nil +} + +func (w canonicalAttemptWindow) executionDisposition() string { + if w.accounting.nativeMtimeDuplicate { + return "native_mtime_duplicate" + } + return "not_observed" +} diff --git a/internal/indexer/canonical_attempt_accounting_test.go b/internal/indexer/canonical_attempt_accounting_test.go new file mode 100644 index 000000000..3b172bde0 --- /dev/null +++ b/internal/indexer/canonical_attempt_accounting_test.go @@ -0,0 +1,166 @@ +package indexer + +import ( + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" +) + +func controlledAttemptWindow(t *testing.T) (*PublicationStamps, time.Time, canonicalAttemptWindow) { + t.Helper() + s, origin, _ := completeControlledAttempt(t) + s.mu.Lock() + evidence := s.freezeAttemptEvidence(origin) + s.mu.Unlock() + return s, origin, canonicalAttemptWindow{startNS: 1e6, endNS: 31e6, ticket: 1, accounting: evidence} +} + +func TestCanonicalAttemptAccountingProjectsOnlyActualAcceptedP(t *testing.T) { + _, _, window := controlledAttemptWindow(t) + p := window.projectAccounting() + require.EqualValues(t, 1, p.AttemptTicket) + require.Zero(t, p.AttemptRetry) + require.True(t, p.PartitionComplete) + require.Equal(t, "partial", p.SumState, "local complete P is not actual100-edit/5% acceptance") + require.Equal(t, "accepted_winning_physical_attempt", p.Scope) + require.Equal(t, "not_established", p.BackgroundCoverage) + require.Equal(t, 30.0, *p.SubtotalMS) + require.Equal(t, 0.0, *p.UnattributedMS) + require.NotContains(t, p.PhasesMS, "debounce", "quiet/backoff cannot be charged against P") + require.Equal(t, 1.0, *p.PhasesMS["patch_analysis"]) + require.Equal(t, 4.0, *p.PhasesMS["attempt_setup"]) + require.Equal(t, 3.0, *p.PhasesMS["attempt_finalize"]) + for _, name := range canonicalPatchLapNames { + require.Equal(t, 1.0, *p.PhasesMS[name], name) + } + enc := zapcore.NewMapObjectEncoder() + require.NoError(t, window.MarshalLogObject(enc)) + require.Equal(t, "partial", enc.Fields["sum_state"]) + require.Equal(t, 30.0, enc.Fields["observed_subtotal_ms"]) + *p.PhasesMS["patch_analysis"] = 999 + require.Equal(t, 1.0, *window.projectAccounting().PhasesMS["patch_analysis"], "projections own their scalar pointers/maps") +} + +func TestCanonicalAttemptAccountingMissingDeepCannotHideInsideControl(t *testing.T) { + _, _, window := controlledAttemptWindow(t) + window.accounting.laps[patchExecutorControl].totalNS += window.accounting.laps[patchParse].totalNS + window.accounting.laps[patchParse] = canonicalLapEvidence{} + p := window.projectAccounting() + require.False(t, p.PartitionComplete) + require.Equal(t, "missing_laps", p.SumState) + require.Nil(t, p.PhasesMS["patch_parse"]) + require.Equal(t, 0.0, *p.UnattributedMS, "numeric fullness never repairs absent applicable deep evidence") +} + +func TestCanonicalAttemptAccountingRejectsBoundsOverflowDerivedAndOpenChunks(t *testing.T) { + for _, name := range []string{"invalid_order", "outside_P", "outside_executor", "overflow", "derived_mismatch", "open_chunk", "not_observed", "gap"} { + t.Run(name, func(t *testing.T) { + _, _, w := controlledAttemptWindow(t) + want := "invalid_order" + switch name { + case "invalid_order": + w.accounting.invalid = true + case "outside_P": + w.accounting.laps[patchAttemptSetup].firstNS = 0 + case "outside_executor": + w.accounting.laps[patchParse].firstNS = 9e6 + case "overflow": + w.accounting.laps[patchExecutorControl].totalNS = 1<<63 - 1 + case "derived_mismatch": + w.accounting.derivedEnd++ + case "open_chunk": + w.accounting.laps[patchResolve].done = 0 + want = "missing_laps" + case "not_observed": + w.accounting.observed = false + want = "not_observed" + case "gap": + w.accounting.laps[patchExecutorControl].totalNS-- + want = "unclassified_gap" + } + p := w.projectAccounting() + require.False(t, p.PartitionComplete) + require.Equal(t, want, p.SumState) + if want == "invalid_order" { + require.Nil(t, p.SubtotalMS) + require.Nil(t, p.UnattributedMS) + } + }) + } +} + +func TestCanonicalAttemptAccountingLateCopyRequiresAgreementWithEarlyCarriers(t *testing.T) { + for _, name := range []string{"unchanged", "deep_changed", "admission_changed", "lane_changed"} { + t.Run(name, func(t *testing.T) { + s, origin, _ := controlledAttemptWindow(t) + recorder := NewPublicationPhaseRecorder(1, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + record := recorder.Begin("", "accounting", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(1, origin)) + record.Absorb(s) + record.MarkAt(PublicationTicketCompleted, origin.Add(29*time.Millisecond)) + early := record.Snapshot() + earlyTiming := early.EditTimings() + require.Nil(t, earlyTiming.Window.PatchAccounting) + s.mu.Lock() + switch name { + case "deep_changed": + s.patchLaps.laps[patchParse].total++ + case "admission_changed": + s.admissions[canonicalCohortAdmission].finished = origin.Add(4 * time.Millisecond) + case "lane_changed": + s.laneWait.acquired = origin.Add(9 * time.Millisecond) + } + s.mu.Unlock() + _, ok := record.completePhysicalAttempt(s, origin.Add(time.Millisecond), origin.Add(31*time.Millisecond), 0) + require.True(t, ok, "actual wall can remain known even when classification carriers disagree") + late := record.Snapshot().EditTimings() + require.Equal(t, earlyTiming, early.EditTimings(), "no previously exposed snapshot changes") + if name == "unchanged" { + require.True(t, late.Window.PatchAccounting.PartitionComplete) + } else { + require.False(t, late.Window.PatchAccounting.PartitionComplete) + require.Equal(t, "invalid_order", late.Window.PatchAccounting.SumState) + } + }) + } +} + +func TestCanonicalAttemptAccountingRefusesDifferentCapturedCallbackOrigin(t *testing.T) { + s, origin, _ := controlledAttemptWindow(t) + recorder := NewPublicationPhaseRecorder(1, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + record := recorder.Begin("", "origin-fence", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(1, origin)) + record.Absorb(s) + _, ok := record.completePhysicalAttempt(s, origin.Add(500*time.Microsecond), origin.Add(31*time.Millisecond), 0) + require.False(t, ok, "an endpoint request cannot replace the independently captured callback origin") + require.Nil(t, record.Snapshot().attemptWindow) +} + +func TestCanonicalAttemptAccountingFractionalTotalsStayInIntegerBounds(t *testing.T) { + _, _, w := controlledAttemptWindow(t) + w.startNS /= 100 + w.endNS /= 100 + p := &w.accounting + p.startNS /= 100 + p.endNS /= 100 + p.executorStart /= 100 + p.executorEnd /= 100 + p.derivedStart /= 100 + p.derivedEnd /= 100 + for i := range p.laps { + p.laps[i].totalNS /= 100 + p.laps[i].firstNS /= 100 + p.laps[i].lastNS /= 100 + } + got := w.projectAccounting() + require.True(t, got.PartitionComplete) + require.Equal(t, 0.3, *got.SubtotalMS) + require.Equal(t, 0.0, *got.UnattributedMS) + p.laps[patchAttemptFinalize].totalNS++ + require.Equal(t, "invalid_order", w.projectAccounting().SumState, "one excess actual nanosecond is not tolerated or clipped") +} diff --git a/internal/indexer/canonical_attempt_control.go b/internal/indexer/canonical_attempt_control.go new file mode 100644 index 000000000..4a9336e50 --- /dev/null +++ b/internal/indexer/canonical_attempt_control.go @@ -0,0 +1,102 @@ +package indexer + +import "time" + +func isAttemptControl(lap canonicalPatchLap) bool { + return lap >= patchAttemptSetup && lap <= patchAttemptFinalize +} + +// These transitions reuse the SAME bounded ordered accumulator. The current +// control interval is actually open while setup/inspection/control/finalization +// runs, and pauses at actual named-operation endpoints; nothing is a residual. +// All locked helpers require PublicationStamps.mu. +func (p *canonicalPatchLaps) openAttemptControlLocked(at time.Time) { + if !p.controlObserved || p.controlFinished { + return + } + p.beginLapLocked(p.controlKind, at) +} + +func (p *canonicalPatchLaps) closeAttemptControlLocked(at time.Time) { + if p.active != 1 || !isAttemptControl(p.activeKind) { + p.invalid = true + return + } + p.endLapLocked(p.activeKind, p.activeStarted, at) +} + +func (s *PublicationStamps) startAttemptControl(at time.Time) { + s.startAttemptControlKind(at, patchAttemptSetup) +} + +func (s *PublicationStamps) startAttemptControlKind(at time.Time, kind canonicalPatchLap) { + if s == nil || s.patchLaps == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + p := s.patchLaps + if !isAttemptControl(kind) || at.IsZero() || p.controlObserved || p.active != 0 || !p.lastEnd.IsZero() { + p.invalid = true + return + } + p.controlObserved, p.controlKind, p.attemptStarted = true, kind, at + p.openAttemptControlLocked(at) +} + +func (s *PublicationStamps) switchAttemptControlAt(kind canonicalPatchLap, at time.Time) { + if s == nil || s.patchLaps == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + p := s.patchLaps + if !p.controlObserved || p.controlFinished { + return + } + if !isAttemptControl(kind) || at.IsZero() { + p.invalid = true + return + } + p.closeAttemptControlLocked(at) + p.controlKind = kind + p.openAttemptControlLocked(at) +} + +func (s *PublicationStamps) switchAttemptControl(kind canonicalPatchLap) bool { + if s == nil || s.patchLaps == nil { + return false + } + s.mu.Lock() + observed := s.patchLaps.controlObserved && !s.patchLaps.controlFinished && s.patchLaps.controlKind != kind + s.mu.Unlock() + if !observed { + return false + } + s.switchAttemptControlAt(kind, time.Now()) + return true +} + +func (p *canonicalPatchLaps) finishAttemptControlLocked(at time.Time) { + if !p.controlObserved || p.controlFinished { + return + } + p.closeAttemptControlLocked(at) + p.controlFinished, p.attemptFinished = true, at +} + +func (p *canonicalPatchLaps) beginObservedOuterLocked(kind canonicalPatchLap, at time.Time) { + if !p.controlObserved || p.controlFinished { + return + } + p.closeAttemptControlLocked(at) + p.beginLapLocked(kind, at) +} + +func (p *canonicalPatchLaps) endObservedOuterLocked(kind canonicalPatchLap, start, end time.Time) { + if !p.controlObserved || p.controlFinished { + return + } + p.endLapLocked(kind, start, end) + p.openAttemptControlLocked(end) +} diff --git a/internal/indexer/canonical_attempt_control_test.go b/internal/indexer/canonical_attempt_control_test.go new file mode 100644 index 000000000..c724399e6 --- /dev/null +++ b/internal/indexer/canonical_attempt_control_test.go @@ -0,0 +1,132 @@ +package indexer + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Actual observed endpoints drive the same accumulator as production. No +// expected control duration is obtained by subtracting known laps from a wall. +func completeControlledAttempt(t *testing.T) (*PublicationStamps, time.Time, canonicalPatchEvidence) { + t.Helper() + origin := time.Now() + s := &PublicationStamps{patchLaps: &canonicalPatchLaps{origin: origin, ticket: 1, enqueued: origin}} + at := func(n int) time.Time { return origin.Add(time.Duration(n) * time.Millisecond) } + s.markAt(PublicationCycleStarted, at(1)) + s.startAttemptControl(at(1)) + cohort := s.beginCanonicalAdmissionAt(canonicalCohortAdmission, at(2)) + cohort.endAt(at(3)) + batch := s.beginCanonicalAdmissionAt(canonicalBatchAdmission, at(4)) + batch.endAt(at(5)) + s.recordCanonicalLaneWaitAt(false, at(6)) + s.recordCanonicalLaneWaitAt(true, at(7)) + s.switchAttemptControlAt(patchInspection, at(8)) + s.markAt(PublicationPatchExecutionStarted, at(10)) + s.switchAttemptControlAt(patchExecutorControl, at(10)) + for kind := patchGateWait; kind <= patchDerived; kind++ { + start := 12 + 2*int(kind) + lap := s.beginPatchLapAt(kind, at(start)) + lap.endAt(at(start + 1)) + if kind == patchDerived { + s.markAt(PublicationPatchDerivedStarted, at(start)) + s.markAt(PublicationPatchDerivedFinished, at(start+1)) + } + } + analysis := s.beginPatchLapAt(patchBookkeeping, at(26)) + analysis.endAt(at(27)) + s.markAt(PublicationPatchExecutionFinished, at(28)) + s.switchAttemptControlAt(patchAttemptFinalize, at(28)) + s.mu.Lock() + early := s.patchLaps.freeze(origin) + s.patchLaps.finishAttemptControlLocked(at(31)) + s.mu.Unlock() + return s, origin, early +} + +func TestCanonicalAttemptControlUsesOrderedActualScopesAndKeepsEarlyDeepCopy(t *testing.T) { + s, origin, early := completeControlledAttempt(t) + require.False(t, early.invalid, "an open outer finalization cannot invalidate completed executor laps") + s.mu.Lock() + late := s.patchLaps.freeze(origin) + counters := *s.patchLaps + s.mu.Unlock() + require.False(t, counters.invalid) + var total time.Duration + for _, lap := range counters.laps { + total += lap.total + require.Equal(t, lap.begun, lap.done) + } + require.Equal(t, 4*time.Millisecond, counters.laps[patchAttemptSetup].total) + require.Equal(t, 2*time.Millisecond, counters.laps[patchInspection].total) + require.Equal(t, 10*time.Millisecond, counters.laps[patchExecutorControl].total) + require.Equal(t, 3*time.Millisecond, counters.laps[patchAttemptFinalize].total) + require.Equal(t, 30*time.Millisecond, total) + require.Equal(t, early, late, "late control completion never rewrites the early deep evidence") + offsets := map[PublicationPhase]int64{PublicationTicketEnqueued: 0, PublicationPatchExecutionStarted: 10e6, PublicationPatchExecutionFinished: 28e6, PublicationPatchDerivedStarted: 24e6, PublicationPatchDerivedFinished: 25e6} + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + early.project(offsets, false, &out) + for _, name := range canonicalPatchLapNames { + require.Equal(t, "measured", out.States[name], name) + require.Equal(t, 1.0, *out.Milliseconds[name], name) + } +} + +func TestCanonicalAttemptControlRealReturnStartsAfterInitialQuiet(t *testing.T) { + path, idx, w := pointPublicationFixture(t) + w.config.DebounceMs = 20 + var local *PublicationStamps + w.pointReindexRaw = func(file string, s *PublicationStamps) (*IndexResult, error) { + local = s + return idx.incrementalPointWatcherPathObserved(idx.rootPath, file, s) + } + record := newPointPublication(t, "full-control") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + require.Eventually(t, func() bool { return record.Snapshot().attemptWindow != nil }, 2*time.Second, time.Millisecond) + snapshot := record.Snapshot() + require.Equal(t, snapshot.debounce.enteredNS, snapshot.attemptWindow.startNS, "initial quiet is outside P") + local.mu.Lock() + invalid, finished := local.patchLaps.invalid, local.patchLaps.controlFinished + var total int64 + for _, lap := range local.patchLaps.laps { + total += lap.total.Nanoseconds() + } + local.mu.Unlock() + require.False(t, invalid) + require.True(t, finished) + require.Equal(t, snapshot.attemptWindow.endNS-snapshot.attemptWindow.startNS, total, "every actual callback/control/operation interval is disjoint and owned") + require.Equal(t, "measured", snapshot.EditTimings().States["derived"]) +} + +func TestCanonicalAttemptControlRefusesOverlappingAndReversedObservations(t *testing.T) { + for _, overlap := range []bool{false, true} { + t.Run(map[bool]string{false: "reversed", true: "overlap"}[overlap], func(t *testing.T) { + origin := time.Now() + s := &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + s.startAttemptControl(origin) + lap := s.beginPatchLapAt(patchParse, origin.Add(time.Millisecond)) + if overlap { + s.beginPatchLapAt(patchApply, origin.Add(2*time.Millisecond)) + } else { + s.switchAttemptControlAt(patchInspection, origin.Add(-time.Millisecond)) + } + lap.endAt(origin.Add(3 * time.Millisecond)) + s.mu.Lock() + invalid := s.patchLaps.invalid + s.mu.Unlock() + require.True(t, invalid) + }) + } +} + +func TestCanonicalAttemptControlNilAndUnobservedScopeDoesNotAllocate(t *testing.T) { + for _, s := range []*PublicationStamps{nil, {}, {patchLaps: &canonicalPatchLaps{}}} { + var switched bool + allocations := testing.AllocsPerRun(100, func() { switched = s.switchAttemptControl(patchInspection) }) + require.False(t, switched) + require.Zero(t, allocations) + } +} diff --git a/internal/indexer/canonical_attempt_disposition_test.go b/internal/indexer/canonical_attempt_disposition_test.go new file mode 100644 index 000000000..f84b80ea2 --- /dev/null +++ b/internal/indexer/canonical_attempt_disposition_test.go @@ -0,0 +1,115 @@ +package indexer + +import ( + "errors" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func controlledDuplicateAttempt(t *testing.T) canonicalAttemptWindow { + t.Helper() + origin := time.Now() + at := func(n int) time.Time { return origin.Add(time.Duration(n) * time.Millisecond) } + s := &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + s.startAttemptControl(at(1)) + cohort := s.beginCanonicalAdmissionAt(canonicalCohortAdmission, at(2)) + cohort.endAt(at(3)) + s.recordCanonicalLaneWaitAt(false, at(4)) + s.recordCanonicalLaneWaitAt(true, at(5)) + s.switchAttemptControlAt(patchInspection, at(6)) + s.recordNativeMtimeDuplicate() + s.switchAttemptControlAt(patchAttemptFinalize, at(8)) + s.mu.Lock() + s.patchLaps.finishAttemptControlLocked(at(10)) + evidence := s.freezeAttemptEvidence(origin) + s.mu.Unlock() + return canonicalAttemptWindow{startNS: 1e6, endNS: 10e6, ticket: 1, accounting: evidence} +} + +func TestCanonicalAttemptObservedDuplicateKeepsUnexecutedLapsUnavailable(t *testing.T) { + w := controlledDuplicateAttempt(t) + p := w.projectAccounting() + require.True(t, p.PartitionComplete) + require.Equal(t, "native_mtime_duplicate", p.ExecutionDisposition) + require.Equal(t, "not_established", p.BackgroundCoverage) + require.Equal(t, 9.0, *p.SubtotalMS) + require.Zero(t, *p.UnattributedMS) + for _, name := range append(canonicalPatchLapNames[:], "patch_analysis", "executor_control") { + require.Equal(t, "not_applicable", p.PhaseStates[name], name) + require.Nil(t, p.PhasesMS[name], "not executed is never a measured zero") + } + w.accounting.nativeMtimeDuplicate = false + require.Equal(t, "missing_laps", w.projectAccounting().SumState, "unknown execution must retain the existing refusal") + require.True(t, p.PartitionComplete, "a later projection cannot mutate an earlier snapshot") +} + +func TestCanonicalAttemptDuplicateRejectsContradictoryExecution(t *testing.T) { + for _, name := range []string{"executor_start", "executor_end", "derived_start", "derived_end", "deep", "executor_control", "missing_control"} { + t.Run(name, func(t *testing.T) { + w := controlledDuplicateAttempt(t) + want := "invalid_order" + switch name { + case "executor_start": + w.accounting.hasExecutorStart = true + case "executor_end": + w.accounting.hasExecutorEnd = true + case "derived_start": + w.accounting.hasDerivedStart = true + case "derived_end": + w.accounting.hasDerivedEnd = true + case "deep": + w.accounting.laps[patchParse].begun = 1 + case "executor_control": + w.accounting.laps[patchExecutorControl].done = 1 + case "missing_control": + w.accounting.laps[patchInspection] = canonicalLapEvidence{} + want = "missing_laps" + } + p := w.projectAccounting() + require.False(t, p.PartitionComplete) + require.Equal(t, want, p.SumState) + }) + } +} + +func TestWatcherNativeDuplicateDispositionRequiresActualEarlyReturn(t *testing.T) { + for _, name := range []string{"native_duplicate", "receipt_checked", "changed_bytes"} { + t.Run(name, func(t *testing.T) { + root, idx, w := inertTestWatcher(t, "main.go", "package p\nfunc Stable() {}\n") + w.degradedNoFsnotify = true // this direct-call fixture never starts the native loop + t.Cleanup(func() { require.NoError(t, w.Stop()) }) + path := filepath.Join(root, "main.go") + if name == "changed_bytes" { + bumpMtime(t, path, "package p\nfunc Changed() {}\n") + } + called := false + stop := errors.New("observed executor entry") + w.pointReindexRaw = func(string, *PublicationStamps) (*IndexResult, error) { + called = true + return nil, stop + } + ctx := WithPublicationStamps(t.Context()) + s := PublicationStampsFrom(ctx) + s.patchLaps = &canonicalPatchLaps{} + err := w.patchGraphWithReceiptStateRawModern(ctx, idx, path, ChangeModified, 0, name == "receipt_checked", nil, nil) + if name == "native_duplicate" { + require.NoError(t, err) + require.False(t, called) + require.True(t, s.patchLaps.nativeMtimeDuplicate) + } else { + require.ErrorIs(t, err, stop) + require.True(t, called) + require.False(t, s.patchLaps.nativeMtimeDuplicate) + } + }) + } +} + +func TestCanonicalDuplicateDispositionNilAndUnobservedDoesNotAllocate(t *testing.T) { + for _, s := range []*PublicationStamps{nil, {}} { + require.Zero(t, testing.AllocsPerRun(100, func() { s.recordNativeMtimeDuplicate() })) + } +} diff --git a/internal/indexer/canonical_attempt_window.go b/internal/indexer/canonical_attempt_window.go new file mode 100644 index 000000000..4cfb83d65 --- /dev/null +++ b/internal/indexer/canonical_attempt_window.go @@ -0,0 +1,112 @@ +package indexer + +import ( + "time" + + "go.uber.org/zap" +) + +// Fixed callback-local readings, never a timer history. Initial quiet wait is +// separate; a retry observes callback entry without relabelling its backoff. +type canonicalPointTimer struct { + debounce canonicalDebounceWait + entered time.Time + background *canonicalBackgroundOwner + successor bool +} + +// Immutable once published. Earlier snapshots keep their original nil/value; +// physical mutation return may postdate receipt terminal and the caller ACK. +type canonicalAttemptWindow struct { + startNS, endNS int64 + retry int + ticket uint64 + accounting canonicalAttemptEvidence +} + +func (p *pointPublicationAttempt) finishPhysicalAttempt(start, end time.Time, retry int, logger *zap.Logger) { + window, accepted := p.record.completePhysicalAttempt(p.stamps, start, end, retry) + if !accepted || logger == nil { + return + } + logger.Info("graph patch attempt complete", + zap.String("patch_lap_scope", "accepted_physical_attempt_complete"), + zap.String("publication_key", p.record.Key()), + zap.Uint64("attempt_ticket", window.ticket), + zap.Int("attempt_retry", window.retry), + zap.Float64("attempt_wall_ms", float64(window.endNS-window.startNS)/1e6), + zap.Object("attempt_accounting", window)) +} + +func (p *PublicationPhaseRecord) completePhysicalAttempt(stamps *PublicationStamps, start, end time.Time, retry int) (canonicalAttemptWindow, bool) { + unavailable := canonicalAttemptWindow{} + if p == nil || stamps == nil || start.IsZero() || end.IsZero() || retry < 0 { + return unavailable, false + } + window := canonicalAttemptWindow{startNS: start.Sub(p.origin).Nanoseconds(), endNS: end.Sub(p.origin).Nanoseconds(), retry: retry} + if window.startNS < 0 || window.endNS < window.startNS { + return unavailable, false + } + stamps.mu.Lock() + hasCycle := false + valid := stamps.patchLaps != nil + var ticket uint64 + var lateDeep canonicalPatchEvidence + var lateAdmissions canonicalAdmissionEvidence + var lateLane canonicalLaneWaitEvidence + if valid { + stamps.patchLaps.finishAttemptControlLocked(end) + ticket = stamps.patchLaps.ticket + window.ticket = ticket + lateDeep = stamps.patchLaps.freeze(p.origin) + lateAdmissions = stamps.freezeCanonicalAdmissions(p.origin) + lateLane = stamps.laneWait.freeze(p.origin) + window.accounting = stamps.freezeAttemptEvidence(p.origin) + if window.accounting.observed && (window.accounting.startNS != window.startNS || window.accounting.endNS != window.endNS) { + valid = false + } + } + for _, mark := range stamps.marks { + if mark.phase == PublicationCycleStarted { + hasCycle = true + if mark.at.Before(start) { + valid = false + } + } + if mark.at.After(end) { + valid = false + } + } + stamps.mu.Unlock() + if !valid || !hasCycle { + return unavailable, false + } + // A weak membership owner does not prolong the entire recorder's lifetime. + // An absent owner is unavailable evidence, never a fabricated zero. + owner := p.owner.Value() + if owner == nil { + return unavailable, false + } + owner.mu.Lock() + defer owner.mu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() + if owner.byKey[p.key] != p || !p.canonicalBound || p.ticket != ticket || p.acceptedAttempt != stamps || p.attemptWindow != nil { + return unavailable, false + } + for _, mark := range p.marks { + if mark.phase == PublicationTicketFailed { + return unavailable, false + } + } + if window.accounting.observed { + if p.patchLaps == nil || *p.patchLaps != lateDeep || p.admissions == nil || *p.admissions != lateAdmissions || + (p.laneWait == nil && (lateLane.started || lateLane.invalid)) || (p.laneWait != nil && *p.laneWait != lateLane) || + !window.accounting.agreesWithAdmissions(lateAdmissions, lateLane) { + window.accounting.invalid = true + } + } + p.attemptWindow = &window // a fresh immutable observation, including after terminal + p.acceptedAttempt = nil + return window, true +} diff --git a/internal/indexer/canonical_attempt_window_test.go b/internal/indexer/canonical_attempt_window_test.go new file mode 100644 index 000000000..388fe3785 --- /dev/null +++ b/internal/indexer/canonical_attempt_window_test.go @@ -0,0 +1,119 @@ +package indexer + +import ( + "runtime" + "sync" + "testing" + "time" + "weak" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +func TestCanonicalAttemptWindowFinishesAfterDoneAndCohortRelease(t *testing.T) { + path, _, w := pointPublicationFixture(t) + core, logs := observer.New(zap.InfoLevel) + w.logger = zap.New(core) + slots := w.mutationSlots() + drained := make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { slots <- struct{}{} }) } + t.Cleanup(unblock) + w.mutationBeforeComplete = func(string, uint64) { <-slots; close(drained) } + record := newPointPublication(t, "late-physical") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + awaitPublicationSignal(t, drained) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err, "Done delivery precedes cohort release exactly as before") + record.Mark(PublicationTicketCompleted) + early := record.Snapshot() + earlyTimings := early.EditTimings() + require.Equal(t, "measured", earlyTimings.States["derived"], "open finalization must not invalidate the frozen completed executor") + require.Nil(t, earlyTimings.Window.PatchWallMS, "actual return is still blocked on cohort release") + require.Empty(t, logs.FilterMessage("graph patch attempt complete").All()) + completedNS := phaseOffsets(early)[PublicationTicketCompleted] + require.Eventually(t, func() bool { return time.Since(record.origin).Nanoseconds() > completedNS }, + 5*time.Second, time.Millisecond, "the held cohort cannot return until the observed clock advances past Done") + releaseAt := time.Now() + unblock() + require.Eventually(t, func() bool { return record.Snapshot().attemptWindow != nil }, 2*time.Second, time.Millisecond) + late := record.Snapshot() + require.GreaterOrEqual(t, late.attemptWindow.endNS, releaseAt.Sub(record.origin).Nanoseconds()) + require.Greater(t, late.attemptWindow.endNS, completedNS, "P can honestly outlive receipt terminal") + require.NotNil(t, late.EditTimings().Window.PatchWallMS) + require.Equal(t, "partial", late.EditTimings().Window.Coverage) + require.Equal(t, earlyTimings, early.EditTimings(), "previously exposed terminal snapshots remain immutable") + require.Equal(t, early.Phases, late.Phases, "late physical observation never edits the frozen terminal timeline") + require.Eventually(t, func() bool { return logs.FilterMessage("graph patch attempt complete").Len() == 1 }, 2*time.Second, time.Millisecond) + fields := logs.FilterMessage("graph patch attempt complete").All()[0].ContextMap() + require.Equal(t, ticket.Generation, fields["attempt_ticket"]) + accounting := fields["attempt_accounting"].(map[string]any) + require.Equal(t, fields["attempt_ticket"], accounting["attempt_ticket"], "both log views use the same frozen identity") + require.EqualValues(t, fields["attempt_retry"], accounting["attempt_retry"]) + require.Equal(t, int64(0), fields["attempt_retry"]) + require.Equal(t, *late.EditTimings().Window.PatchWallMS, fields["attempt_wall_ms"]) + require.Equal(t, "accepted_physical_attempt_complete", fields["patch_lap_scope"]) + require.Equal(t, "physical_attempt_before_receipt_validation", logs.FilterMessage("graph patch complete").All()[0].ContextMap()["patch_lap_scope"]) +} + +func attemptWindowFixture(t *testing.T) (*PublicationPhaseRecorder, *PublicationPhaseRecord, *PublicationStamps, time.Time) { + t.Helper() + origin := time.Now() + recorder := NewPublicationPhaseRecorder(1, 1) + record := recorder.Begin("", "original", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(7, origin.Add(time.Millisecond))) + stamps := record.newCanonicalPatchStamps() + stamps.markAt(PublicationCycleStarted, origin.Add(3*time.Millisecond)) + stamps.markAt(PublicationPatchExecutionStarted, origin.Add(4*time.Millisecond)) + stamps.markAt(PublicationPatchExecutionFinished, origin.Add(5*time.Millisecond)) + record.Absorb(stamps) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + return recorder, record, stamps, origin +} + +func TestCanonicalAttemptWindowRequiresMembershipExactWinnerAndOneShot(t *testing.T) { + for _, name := range []string{"winner_after_terminal", "rekey_same_record", "evicted_rebound_key", "wrong_attempt", "failed_terminal", "absent_owner", "invalid_order", "missing_callback", "before_origin", "one_shot"} { + t.Run(name, func(t *testing.T) { + recorder, record, stamps, origin := attemptWindowFixture(t) + start, end := origin.Add(2*time.Millisecond), origin.Add(8*time.Millisecond) + want := false + switch name { + case "winner_after_terminal": + record.MarkAt(PublicationTicketCompleted, origin.Add(6*time.Millisecond)) + want = true + case "rekey_same_record": + require.True(t, recorder.Rekey("original", "receipt")) + want = true + case "evicted_rebound_key": + recorder.Begin("", "other", PublicationSourceMCPEdit, origin) + replacement := recorder.Begin("", "original", PublicationSourceMCPEdit, origin) + require.NotSame(t, record, replacement) + case "wrong_attempt": + stamps = record.newCanonicalPatchStamps() + stamps.markAt(PublicationCycleStarted, origin.Add(3*time.Millisecond)) + case "failed_terminal": + record.MarkAt(PublicationTicketFailed, origin.Add(6*time.Millisecond)) + case "absent_owner": + record.owner = weak.Pointer[PublicationPhaseRecorder]{} + case "invalid_order": + end = origin.Add(time.Millisecond) + case "missing_callback": + start = time.Time{} + case "before_origin": + start = origin.Add(-time.Millisecond) + case "one_shot": + _, ok := record.completePhysicalAttempt(stamps, start, end, 0) + require.True(t, ok) + } + _, ok := record.completePhysicalAttempt(stamps, start, end, 0) + require.Equal(t, want, ok) + if name == "evicted_rebound_key" { + replacement, _ := recorder.Lookup("original") + require.Nil(t, replacement.Snapshot().attemptWindow) + } + require.False(t, record.bindCanonicalTicket(8, end), "a record cannot be rebound to a newer generation") + }) + } +} diff --git a/internal/indexer/canonical_background_handoff_test.go b/internal/indexer/canonical_background_handoff_test.go new file mode 100644 index 000000000..7f6d1cdb1 --- /dev/null +++ b/internal/indexer/canonical_background_handoff_test.go @@ -0,0 +1,175 @@ +package indexer + +import ( + "encoding/json" + "math" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func controlledBackgroundHandoff(t *testing.T) (*canonicalBackgroundOwner, *Watcher, *PublicationPhaseRecord, time.Time) { + t.Helper() + recorder := NewPublicationPhaseRecorder(2, 1) + w := &Watcher{} + t.Cleanup(func() { runtime.KeepAlive(recorder); runtime.KeepAlive(w) }) + origin := time.Now() + record := recorder.Begin("", "controlled-background", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(10, origin)) + b := newCanonicalBackground(record, w, "sample.go", 10, origin) + b.armStarted(w, "sample.go", 10, false, origin.Add(time.Nanosecond)) + b.armFinished(w, "sample.go", 10, false, origin.Add(2*time.Nanosecond)) + return b, w, record, origin +} + +func TestRequestedBackgroundAccumulatesActualOrderedHandoffsNotGenerationDelta(t *testing.T) { + b, w, _, origin := controlledBackgroundHandoff(t) + require.Same(t, b, b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true)) + // Other paths may consume intervening global generations. Only these actual + // Stop/arm observations count; a generation delta is never measured work. + b.armStarted(w, "sample.go", 40, true, origin.Add(5*time.Nanosecond)) + b.armFinished(w, "sample.go", 40, true, origin.Add(6*time.Nanosecond)) + require.Same(t, b, b.stopped(w, "sample.go", 40, true, origin.Add(7*time.Nanosecond), origin.Add(8*time.Nanosecond), true, true)) + b.armStarted(w, "sample.go", 90, true, origin.Add(9*time.Nanosecond)) + b.armFinished(w, "sample.go", 90, true, origin.Add(10*time.Nanosecond)) + b.callback(w, "sample.go", 90, true, origin.Add(11*time.Nanosecond)) + require.False(t, b.invalid) + require.Equal(t, uint64(2), b.handoffs) + require.Equal(t, uint64(90), b.current) + require.Equal(t, [8]int64{1, 1, 1, 2, 2, 2, 2, 0}, b.phases) + require.Equal(t, backgroundExecuting, b.stage) +} + +func TestRequestedBackgroundHandoffRefusesForeignAndUnobservedTransitions(t *testing.T) { + for _, name := range []string{"owner", "path", "generation", "stop_failed", "callback_started", "observed_successor", "arm_without_stop", "reordered", "retry", "count_overflow", "phase_overflow"} { + t.Run(name, func(t *testing.T) { + b, w, record, origin := controlledBackgroundHandoff(t) + want := "foreign_timer_identity" + switch name { + case "owner": + b.stopped(&Watcher{}, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true) + case "path": + b.stopped(w, "foreign.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true) + case "generation": + b.stopped(w, "sample.go", 11, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true) + case "stop_failed": + b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), false, true) + want = "timer_stop_not_successful" + case "observed_successor": + b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, false) + want = "observed_successor" + case "arm_without_stop": + b.armStarted(w, "sample.go", 40, true, origin.Add(3*time.Nanosecond)) + want = "unobserved_timer_handoff" + case "reordered": + b.stopped(w, "sample.go", 10, false, origin.Add(time.Nanosecond), origin.Add(4*time.Nanosecond), true, true) + want = "invalid_timing_order" + case "retry": + b.invalidate("retry_or_backoff") + want = "retry_or_backoff" + case "count_overflow": + require.Same(t, b, b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true)) + b.handoffs = math.MaxUint64 + b.armStarted(w, "sample.go", 40, true, origin.Add(5*time.Nanosecond)) + want = "handoff_count_overflow" + case "phase_overflow": + require.Same(t, b, b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true)) + b.armStarted(w, "sample.go", 40, true, origin.Add(5*time.Nanosecond)) + b.armFinished(w, "sample.go", 40, true, origin.Add(6*time.Nanosecond)) + b.phases[6] = math.MaxInt64 + b.stopped(w, "sample.go", 40, true, origin.Add(7*time.Nanosecond), origin.Add(8*time.Nanosecond), true, true) + want = "timing_overflow" + case "callback_started": + require.Same(t, b, b.stopped(w, "sample.go", 10, false, origin.Add(3*time.Nanosecond), origin.Add(4*time.Nanosecond), true, true)) + b.armStarted(w, "sample.go", 40, true, origin.Add(5*time.Nanosecond)) + b.armFinished(w, "sample.go", 40, true, origin.Add(6*time.Nanosecond)) + b.callback(w, "sample.go", 40, true, origin.Add(7*time.Nanosecond)) + b.stopped(w, "sample.go", 40, true, origin.Add(8*time.Nanosecond), origin.Add(9*time.Nanosecond), true, true) + want = "timer_callback_already_started" + } + b.publishUnavailable() + winner, unavailable := record.CoalescedPublicationEvidence() + require.Nil(t, winner) + require.NotNil(t, unavailable) + require.Equal(t, want, unavailable.Reason) + require.Equal(t, uint64(10), unavailable.RequestedGeneration) + wire, err := json.Marshal(unavailable) + require.NoError(t, err) + var payload map[string]any + require.NoError(t, json.Unmarshal(wire, &payload)) + require.Equal(t, record.Snapshot().Key, payload["publication_key"]) + require.Equal(t, want, payload["reason"]) + require.Equal(t, "requested_background_observation", payload["scope"]) + require.Nil(t, record.RequestedBackground()) + unavailable.Reason = "caller-mutation" + _, again := record.CoalescedPublicationEvidence() + require.Equal(t, want, again.Reason) + }) + } +} + +func TestRequestedBackgroundRefusalIsImmutableAndCannotResurrectEvictedOwner(t *testing.T) { + recorder := NewPublicationPhaseRecorder(1, 1) + w := &Watcher{} + origin := time.Now() + record := recorder.Begin("", "receipt", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(10, origin)) + b := newCanonicalBackground(record, w, "sample.go", 10, origin) + _, early := record.CoalescedPublicationEvidence() + require.Nil(t, early) + b.invalidate("retry_or_backoff") + b.publishUnavailable() + _, accepted := record.CoalescedPublicationEvidence() + require.NotNil(t, accepted) + b.invalidate("invalid_timing_order") + b.publishUnavailable() + _, same := record.CoalescedPublicationEvidence() + require.Equal(t, "retry_or_backoff", same.Reason) + recorder.Begin("", "evict", PublicationSourceMCPEdit, origin) + winner, absent := record.CoalescedPublicationEvidence() + require.Nil(t, winner) + require.Nil(t, absent) + replacement := recorder.Begin("", "receipt", PublicationSourceMCPEdit, origin) + require.True(t, replacement.bindCanonicalTicket(11, origin)) + b.publishUnavailable() + _, replacementFailure := replacement.CoalescedPublicationEvidence() + require.Nil(t, replacementFailure) + require.Equal(t, "retry_or_backoff", accepted.Reason) + require.Nil(t, early) + runtime.KeepAlive(w) + runtime.KeepAlive(recorder) +} + +func TestRequestedBackgroundRefusalFollowsSameRetainedFinalKeyWithoutMutatingSnapshots(t *testing.T) { + recorder := NewPublicationPhaseRecorder(1, 1) + watcher := &Watcher{} + origin := time.Now() + record := recorder.Begin("", "batch-file-original", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(7, origin)) + background := newCanonicalBackground(record, watcher, "fixture.go", 7, origin) + background.invalidate("preexisting_timer_stop_not_successful") + background.publishUnavailable() + _, early := record.CoalescedPublicationEvidence() + require.NotNil(t, early) + require.Equal(t, "batch-file-original", early.PublicationKey) + require.True(t, recorder.Rekey("batch-file-original", "mutation-original")) + _, later := record.CoalescedPublicationEvidence() + require.NotNil(t, later) + require.Equal(t, "mutation-original", later.PublicationKey) + require.Equal(t, "batch-file-original", early.PublicationKey, "earlier snapshot is immutable") + later.PublicationKey = "foreign" + _, again := record.CoalescedPublicationEvidence() + require.Equal(t, "mutation-original", again.PublicationKey) + recorder.Begin("", "evict", PublicationSourceMCPEdit, origin) + _, expired := record.CoalescedPublicationEvidence() + require.Nil(t, expired) + replacement := recorder.Begin("", "mutation-original", PublicationSourceMCPEdit, origin) + require.True(t, replacement.bindCanonicalTicket(8, origin)) + background.publishUnavailable() + _, foreign := replacement.CoalescedPublicationEvidence() + require.Nil(t, foreign, "old owner cannot donate its refusal to replacement") + runtime.KeepAlive(watcher) + runtime.KeepAlive(recorder) +} diff --git a/internal/indexer/canonical_coalesced_attempt.go b/internal/indexer/canonical_coalesced_attempt.go new file mode 100644 index 000000000..ecc4c77ba --- /dev/null +++ b/internal/indexer/canonical_coalesced_attempt.go @@ -0,0 +1,137 @@ +package indexer + +import ( + "fmt" + "time" + "weak" +) + +// A successor is its own physical observation. It never rebinds the edit's +// ticket or copies accepted-P evidence into a receipt it merely serves. +func (p *PublicationPhaseRecord) beginCoalescedAttempt(generation uint64, retry int, entered time.Time) *PublicationPhaseRecord { + if p == nil || generation == 0 || retry < 0 || entered.IsZero() { + return nil + } + owner := p.owner.Value() + if owner == nil { + return nil + } + owner.mu.Lock() + defer owner.mu.Unlock() + p.mu.Lock() + if owner.byKey[p.key] != p || !p.canonicalBound || p.ticket >= generation || entered.Before(p.origin) { + p.mu.Unlock() + return nil + } + checkout, key := p.checkoutID, fmt.Sprintf("%s/physical-%d-retry-%d", p.key, generation, retry) + p.mu.Unlock() + if owner.byKey[key] != nil { + return nil // a callback identity cannot be rebound to another origin + } + record := owner.beginLocked(checkout, key, "canonical_coalesced_attempt", entered) + // This origin is the observed native arm (or callback entry when absent), + // not request admission. Bind identity without inventing ticket_enqueued. + record.mu.Lock() + record.canonicalBound, record.ticket = true, generation + record.mu.Unlock() + return record +} + +func (p *PublicationPhaseRecord) linkCoalescedWinner(winner *PublicationPhaseRecord, requested, applied uint64) { + if p == nil || winner == nil || p == winner || requested == 0 || requested >= applied { + return + } + owner := p.owner.Value() + if owner == nil || winner.owner.Value() != owner { + return + } + owner.mu.Lock() + defer owner.mu.Unlock() + // Keys may have changed while the callback ran. Membership is checked under + // the existing recorder->record order, not by a stale captured key. + p.mu.Lock() + defer p.mu.Unlock() + winner.mu.Lock() + defer winner.mu.Unlock() + if owner.byKey[p.key] != p || owner.byKey[winner.key] != winner || !p.canonicalBound || + p.ticket != requested || winner.ticket != applied || winner.acceptedAttempt == nil || p.attemptWindow != nil || p.coalescedLinked { + return + } + for _, mark := range p.marks { + if mark.phase == PublicationTicketFailed { + return + } + } + p.coalescedWinner = weak.Make(winner) + p.coalescedLinked = true +} + +// CoalescedPublication is a separate actual winner, not the requested edit's P +// window or a complete per-edit background window. An evicted/rebound owner or +// winner remains unavailable. The weak link cannot retain bounded history. +type CoalescedPublication struct { + RequestedGeneration uint64 `json:"requested_generation"` + AppliedGeneration uint64 `json:"applied_generation"` + Scope string `json:"scope"` + BackgroundCoverage string `json:"background_coverage"` + Publication PublicationPhaseSnapshot `json:"publication_phases"` + Timings EditPhaseTimings `json:"timings"` + Background *RequestedBackgroundTiming `json:"-"` // same fenced immutable observation, projected once +} + +func (p *PublicationPhaseRecord) CoalescedPublication() *CoalescedPublication { + winner, _ := p.CoalescedPublicationEvidence() + return winner +} + +// CoalescedPublicationEvidence captures the related winner/background or an +// actual immutable refusal under one recorder-membership fence. A refusal is +// available even when no independent winner could bind; it is never coverage. +func (p *PublicationPhaseRecord) CoalescedPublicationEvidence() (*CoalescedPublication, *RequestedBackgroundUnavailable) { + if p == nil { + return nil, nil + } + owner := p.owner.Value() + if owner == nil { + return nil, nil + } + owner.mu.Lock() + defer owner.mu.Unlock() + p.mu.Lock() + winner, requested, key, background := p.coalescedWinner.Value(), p.ticket, p.key, p.background + var unavailable *RequestedBackgroundUnavailable + if p.backgroundUnavailable != nil { + copy := *p.backgroundUnavailable + unavailable = © + } + p.mu.Unlock() + if owner.byKey[key] != p { + return nil, nil + } + if unavailable != nil && unavailable.RequestedGeneration != requested { + unavailable = nil + } + if unavailable != nil { + // The recorder membership above proves this is the same retained owner + // after a legitimate final-key binding. Project only the copied diagnostic; + // neither its frozen timing evidence nor earlier caller snapshots change. + unavailable.PublicationKey = key + } + if winner == nil { + return nil, unavailable + } + snapshot := winner.Snapshot() + if owner.byKey[snapshot.Key] != winner || snapshot.Ticket <= requested { + return nil, unavailable + } + out := &CoalescedPublication{RequestedGeneration: requested, AppliedGeneration: snapshot.Ticket, + Scope: "independent_physical_winner_serving_coalesced_receipt", BackgroundCoverage: "not_established", + Publication: snapshot, Timings: snapshot.EditTimings()} + if background != nil && background.requested == requested && background.applied == snapshot.Ticket && snapshot.attemptWindow != nil && *snapshot.attemptWindow == background.physical { + out.Background = background.project(out.Timings.Window.PatchAccounting) + } + if out.Background != nil { + unavailable = nil + } + return out, unavailable +} diff --git a/internal/indexer/canonical_debounce.go b/internal/indexer/canonical_debounce.go new file mode 100644 index 000000000..ab4c08c64 --- /dev/null +++ b/internal/indexer/canonical_debounce.go @@ -0,0 +1,62 @@ +package indexer + +import "time" + +// Only the initial exact-generation timer owns this quiet-wait pair. Retry +// backoff and the callback's claim lock/cohort admission are different work. +type canonicalDebounceWait struct { + started, entered time.Time +} + +type canonicalDebounceEvidence struct { + startedNS, enteredNS int64 + entered bool +} + +func (p canonicalDebounceWait) freeze(origin time.Time) canonicalDebounceEvidence { + return canonicalDebounceEvidence{startedNS: p.started.Sub(origin).Nanoseconds(), enteredNS: p.entered.Sub(origin).Nanoseconds(), entered: !p.entered.IsZero()} +} + +func (p canonicalDebounceEvidence) project(offsets map[PublicationPhase]int64, terminal bool, out *EditPhaseTimings) { + out.Milliseconds["debounce"], out.States["debounce"] = nil, "not_observed" + if p.startedNS < 0 || p.entered && p.enteredNS < p.startedNS { + out.States["debounce"] = "invalid_order" + return + } + if !p.entered { + if !terminal { + out.States["debounce"] = "pending" + } + return + } + enqueue, hasEnqueue := offsets[PublicationTicketEnqueued] + if !hasEnqueue { + // A native serving record has no request-admission timestamp. Its + // independently observed arm start is the applicable lower boundary. + enqueue, hasEnqueue = offsets[PublicationTimerArmStarted] + } + cycle, hasCycle := offsets[PublicationCycleStarted] + if !hasEnqueue || !hasCycle { + return + } + if enqueue < 0 || p.startedNS < enqueue || p.enteredNS > cycle { + out.States["debounce"] = "invalid_order" + return + } + if terminal { + last := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + last = PublicationTicketFailed + } + end, known := offsets[last] + if !known { + return + } + if p.enteredNS > end { + out.States["debounce"] = "invalid_order" + return + } + } + ms := float64(p.enteredNS-p.startedNS) / 1e6 + out.Milliseconds["debounce"], out.States["debounce"] = &ms, "measured" +} diff --git a/internal/indexer/canonical_debounce_test.go b/internal/indexer/canonical_debounce_test.go new file mode 100644 index 000000000..a2f3e8d04 --- /dev/null +++ b/internal/indexer/canonical_debounce_test.go @@ -0,0 +1,108 @@ +package indexer + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCanonicalDebounceObservesInitialTimerBeforeClaimedHook(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = 20 + entered := make(chan time.Time, 1) + release := make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(unblock) + w.pointMutationClaimed = func(string) { + entered <- time.Now() + <-release + } + record := newPointPublication(t, "quiet-wait") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + var claimed time.Time + select { + case claimed = <-entered: + case <-time.After(5 * time.Second): + t.Fatal("initial timer did not claim its generation") + } + require.Nil(t, record.Snapshot().debounce, "unvalidated attempt must not publish its pair") + unblock() + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + snapshot := record.Snapshot() + require.NotNil(t, snapshot.debounce) + require.LessOrEqual(t, snapshot.debounce.enteredNS, claimed.Sub(record.origin).Nanoseconds(), "hook/admission work is outside quiet wait") + timing := snapshot.EditTimings() + require.Equal(t, "measured", timing.States["debounce"]) + require.NotNil(t, timing.Milliseconds["debounce"]) + require.Equal(t, "partial", timing.Window.Coverage, "quiet wait and the separate physical wall do not close accounting") +} + +func TestCanonicalDebounceRequiresActualContainedPair(t *testing.T) { + for _, tc := range []struct { + name string + pair canonicalDebounceEvidence + terminal bool + remove PublicationPhase + terminalNS int64 + want string + }{ + {name: "actual_zero", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 2, entered: true}, want: "measured"}, + {name: "actual_wait", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 4, entered: true}, terminal: true, terminalNS: 8, want: "measured"}, + {name: "negative", pair: canonicalDebounceEvidence{startedNS: -1, enteredNS: 4, entered: true}, want: "invalid_order"}, + {name: "reversed", pair: canonicalDebounceEvidence{startedNS: 4, enteredNS: 2, entered: true}, want: "invalid_order"}, + {name: "before_enqueue", pair: canonicalDebounceEvidence{startedNS: 0, enteredNS: 4, entered: true}, want: "invalid_order"}, + {name: "after_cycle", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 7, entered: true}, want: "invalid_order"}, + {name: "missing_enqueue", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 4, entered: true}, remove: PublicationTicketEnqueued, want: "not_observed"}, + {name: "missing_cycle", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 4, entered: true}, remove: PublicationCycleStarted, want: "not_observed"}, + {name: "unfinished", pair: canonicalDebounceEvidence{startedNS: 2}, want: "pending"}, + {name: "terminal_unfinished", pair: canonicalDebounceEvidence{startedNS: 2}, terminal: true, terminalNS: 8, want: "not_observed"}, + {name: "missing_terminal", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 4, entered: true}, terminal: true, want: "not_observed"}, + {name: "outside_terminal", pair: canonicalDebounceEvidence{startedNS: 2, enteredNS: 4, entered: true}, terminal: true, terminalNS: 3, want: "invalid_order"}, + } { + t.Run(tc.name, func(t *testing.T) { + offsets := map[PublicationPhase]int64{PublicationTicketEnqueued: 1, PublicationCycleStarted: 6} + if tc.terminalNS != 0 { + offsets[PublicationTicketCompleted] = tc.terminalNS + } + delete(offsets, tc.remove) + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + tc.pair.project(offsets, tc.terminal, &out) + require.Equal(t, tc.want, out.States["debounce"]) + if tc.want == "measured" { + require.NotNil(t, out.Milliseconds["debounce"]) + } else { + require.Nil(t, out.Milliseconds["debounce"]) + } + }) + } +} + +func TestCanonicalNativeDebounceUsesOnlyActualArmProof(t *testing.T) { + for _, name := range []string{"actual_arm", "missing_arm", "arm_after_start", "terminal_before_callback"} { + t.Run(name, func(t *testing.T) { + pair := canonicalDebounceEvidence{startedNS: 0, enteredNS: 5, entered: true} + offsets := map[PublicationPhase]int64{PublicationTimerArmStarted: 0, PublicationCycleStarted: 6, PublicationTicketCompleted: 10} + switch name { + case "missing_arm": + delete(offsets, PublicationTimerArmStarted) + case "arm_after_start": + offsets[PublicationTimerArmStarted] = 1 + case "terminal_before_callback": + offsets[PublicationTicketCompleted] = 4 + } + out := EditPhaseTimings{Milliseconds: make(map[string]*float64), States: make(map[string]string)} + pair.project(offsets, true, &out) + if name == "actual_arm" { + require.Equal(t, "measured", out.States["debounce"]) + require.Equal(t, 0.000005, *out.Milliseconds["debounce"]) + } else { + require.Nil(t, out.Milliseconds["debounce"]) + } + require.NotContains(t, offsets, PublicationTicketEnqueued, "native arm must not invent request admission") + }) + } +} diff --git a/internal/indexer/canonical_lane_wait.go b/internal/indexer/canonical_lane_wait.go new file mode 100644 index 000000000..2f3dae7f8 --- /dev/null +++ b/internal/indexer/canonical_lane_wait.go @@ -0,0 +1,118 @@ +package indexer + +import ( + "context" + "time" +) + +// A single canonical point attempt owns this pair. It precedes the executor, +// so it is deliberately separate from the executor-contained patch laps. +type canonicalLaneWait struct { + started, acquired time.Time + invalid bool +} + +type canonicalLaneWaitEvidence struct { + startNS, acquiredNS int64 + started, acquired, invalid bool +} + +func beginCanonicalLaneWait(ctx context.Context) *PublicationStamps { + s := PublicationStampsFrom(ctx) + if s == nil || s.patchLaps == nil { + return nil // no clock/allocation on other paths + } + s.recordCanonicalLaneWaitAt(false, time.Now()) + return s +} + +func (s *PublicationStamps) canonicalLaneAcquired() { + if s == nil { + return + } + s.recordCanonicalLaneWaitAt(true, time.Now()) +} + +func (s *PublicationStamps) recordCanonicalLaneWaitAt(acquired bool, at time.Time) { + if s == nil || s.patchLaps == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + pair := &s.laneWait + if at.IsZero() { + pair.invalid = true + return + } + if !acquired { + if !pair.started.IsZero() { + pair.invalid = true + return + } + pair.started = at + s.patchLaps.beginObservedOuterLocked(patchRepositoryLane, at) + return + } + if pair.started.IsZero() || !pair.acquired.IsZero() || at.Before(pair.started) { + pair.invalid = true + return + } + pair.acquired = at + s.patchLaps.endObservedOuterLocked(patchRepositoryLane, pair.started, at) +} + +func (p canonicalLaneWait) freeze(origin time.Time) canonicalLaneWaitEvidence { + out := canonicalLaneWaitEvidence{started: !p.started.IsZero(), acquired: !p.acquired.IsZero(), invalid: p.invalid} + if out.started { + out.startNS = p.started.Sub(origin).Nanoseconds() + } + if out.acquired { + out.acquiredNS = p.acquired.Sub(origin).Nanoseconds() + } + return out +} + +func (p canonicalLaneWaitEvidence) project(offsets map[PublicationPhase]int64, terminal bool, out *EditPhaseTimings) { + out.Milliseconds["lane_wait"], out.States["lane_wait"] = nil, "not_observed" + if p.invalid || p.started && p.startNS < 0 || p.acquired && (!p.started || p.acquiredNS < p.startNS) { + out.States["lane_wait"] = "invalid_order" + return + } + if !p.started { + return + } + if !p.acquired { + if !terminal { + out.States["lane_wait"] = "pending" + } + return + } + enqueue, hasEnqueue := offsets[PublicationTicketEnqueued] + cycle, hasCycle := offsets[PublicationCycleStarted] + executor, hasExecutor := offsets[PublicationPatchExecutionStarted] + if !hasEnqueue || !hasCycle || !hasExecutor { + return + } + if enqueue < 0 || cycle < enqueue || p.startNS < cycle || p.acquiredNS > executor { + out.States["lane_wait"] = "invalid_order" + return + } + if terminal { + last := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + last = PublicationTicketFailed + } + finished, known := offsets[last] + if !known { + return + } + if p.acquiredNS > finished { + out.States["lane_wait"] = "invalid_order" + return + } + } + // Only a genuine wholly contained pair is additive. This is not the cohort + // semaphore, batch RLock, cycle-to-admitted envelope or a full patch wall. + ms := float64(p.acquiredNS-p.startNS) / 1e6 + out.Milliseconds["lane_wait"], out.States["lane_wait"] = &ms, "measured" +} diff --git a/internal/indexer/canonical_lane_wait_test.go b/internal/indexer/canonical_lane_wait_test.go new file mode 100644 index 000000000..e79f1974a --- /dev/null +++ b/internal/indexer/canonical_lane_wait_test.go @@ -0,0 +1,176 @@ +package indexer + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCanonicalLaneWaitObservesActualBlockedGrantAfterBatchAdmission(t *testing.T) { + origin := time.Now() + record := NewPublicationPhaseRecorder(1, 1).Begin("", "lane", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(1, time.Now())) + stamps := record.newCanonicalPatchStamps() + stamps.markAt(PublicationCycleStarted, time.Now()) + ctx, cancel := context.WithCancel(context.WithValue(t.Context(), publicationStampsKey{}, stamps)) + coordinator := newRepositoryMutationCoordinator(nil) + <-coordinator.lane // real lane occupied before this attempt enters + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { coordinator.lane <- struct{}{} }) } + hookSawStart := false + coordinator.batchAdmissionHook = func() { + stamps.mu.Lock() + hookSawStart = !stamps.laneWait.started.IsZero() + stamps.mu.Unlock() + } + done := make(chan error, 1) + finished := make(chan struct{}) + go func() { + defer close(finished) + done <- coordinator.runExclusive(ctx, func() error { + stamps.markAt(PublicationPatchExecutionStarted, time.Now()) + stamps.markAt(PublicationPatchExecutionFinished, time.Now()) + return nil + }) + }() + t.Cleanup(func() { + cancel() + release() + select { + case <-finished: + case <-time.After(2 * time.Second): + t.Error("lane observation did not join") + } + }) + require.Eventually(t, func() bool { stamps.mu.Lock(); defer stamps.mu.Unlock(); return !stamps.laneWait.started.IsZero() }, 2*time.Second, time.Millisecond) + grantedAfter := time.Now() + release() + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(2 * time.Second): + t.Fatal("occupied lane did not grant") + } + require.False(t, hookSawStart, "batch admission/hook work is outside pure lane wait") + record.Absorb(stamps) + record.Mark(PublicationTicketCompleted) + snapshot := record.Snapshot() + require.NotNil(t, snapshot.laneWait) + require.True(t, snapshot.laneWait.acquired) + require.GreaterOrEqual(t, snapshot.laneWait.acquiredNS, grantedAfter.Sub(origin).Nanoseconds()) + timing := snapshot.EditTimings() + require.Equal(t, "measured", timing.States["lane_wait"]) + require.NotNil(t, timing.Milliseconds["lane_wait"]) + require.Nil(t, timing.Window.PatchWallMS, "the lane pair is not a whole patch wall") +} + +func TestCanonicalLaneWaitCancellationHasNoGrant(t *testing.T) { + origin := time.Now() + stamps := &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + ctx, cancel := context.WithCancel(context.WithValue(t.Context(), publicationStampsKey{}, stamps)) + coordinator := newRepositoryMutationCoordinator(nil) + <-coordinator.lane + done := make(chan error, 1) + finished := make(chan struct{}) + go func() { + defer close(finished) + done <- coordinator.runExclusive(ctx, func() error { t.Error("canceled wait entered executor"); return nil }) + }() + t.Cleanup(func() { + cancel() + select { + case <-finished: + case <-time.After(2 * time.Second): + t.Error("canceled lane wait did not join") + } + coordinator.lane <- struct{}{} + }) + require.Eventually(t, func() bool { stamps.mu.Lock(); defer stamps.mu.Unlock(); return !stamps.laneWait.started.IsZero() }, 2*time.Second, time.Millisecond) + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(2 * time.Second): + t.Fatal("lane wait ignored cancellation") + } + stamps.mu.Lock() + evidence := stamps.laneWait.freeze(origin) + stamps.mu.Unlock() + require.True(t, evidence.started) + require.False(t, evidence.acquired) + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + evidence.project(nil, true, &out) + require.Nil(t, out.Milliseconds["lane_wait"], "a canceled attempt cannot report a zero-duration grant") + require.Equal(t, "not_observed", out.States["lane_wait"]) +} + +func TestCanonicalLaneWaitDoesNotObserveForeignOrUninstrumentedContexts(t *testing.T) { + empty := context.WithValue(context.Background(), publicationStampsKey{}, &PublicationStamps{}) + for _, ctx := range []context.Context{context.Background(), empty} { + allocs := testing.AllocsPerRun(100, func() { + token := beginCanonicalLaneWait(ctx) + if token != nil { + t.Fatal("noncanonical context observed a lane") + } + token.canonicalLaneAcquired() + }) + require.Zero(t, allocs) + } +} + +func TestCanonicalLaneWaitProjectionRequiresOwnedContainingBoundaries(t *testing.T) { + valid := canonicalLaneWaitEvidence{started: true, acquired: true, startNS: int64(4 * time.Millisecond), acquiredNS: int64(5 * time.Millisecond)} + for _, tc := range []struct { + name string + change func(*canonicalLaneWaitEvidence, map[PublicationPhase]int64) + state string + }{ + {name: "valid", state: "measured"}, + {name: "actual zero", state: "measured", change: func(v *canonicalLaneWaitEvidence, _ map[PublicationPhase]int64) { v.acquiredNS = v.startNS }}, + {name: "before cycle", state: "invalid_order", change: func(v *canonicalLaneWaitEvidence, _ map[PublicationPhase]int64) { v.startNS = 0 }}, + {name: "after executor", state: "invalid_order", change: func(v *canonicalLaneWaitEvidence, _ map[PublicationPhase]int64) { + v.acquiredNS = int64(9 * time.Millisecond) + }}, + {name: "reversed", state: "invalid_order", change: func(v *canonicalLaneWaitEvidence, _ map[PublicationPhase]int64) { v.acquiredNS = 1 }}, + {name: "missing grant", state: "not_observed", change: func(v *canonicalLaneWaitEvidence, _ map[PublicationPhase]int64) { v.acquired = false }}, + {name: "missing enqueue", state: "not_observed", change: func(_ *canonicalLaneWaitEvidence, m map[PublicationPhase]int64) { delete(m, PublicationTicketEnqueued) }}, + {name: "missing cycle", state: "not_observed", change: func(_ *canonicalLaneWaitEvidence, m map[PublicationPhase]int64) { delete(m, PublicationCycleStarted) }}, + {name: "missing executor", state: "not_observed", change: func(_ *canonicalLaneWaitEvidence, m map[PublicationPhase]int64) { + delete(m, PublicationPatchExecutionStarted) + }}, + {name: "terminal before grant", state: "invalid_order", change: func(_ *canonicalLaneWaitEvidence, m map[PublicationPhase]int64) { + m[PublicationTicketCompleted] = int64(2 * time.Millisecond) + }}, + {name: "missing terminal", state: "not_observed", change: func(_ *canonicalLaneWaitEvidence, m map[PublicationPhase]int64) { + delete(m, PublicationTicketCompleted) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + pair := valid + offsets := map[PublicationPhase]int64{PublicationTicketEnqueued: int64(time.Millisecond), PublicationCycleStarted: int64(2 * time.Millisecond), PublicationPatchExecutionStarted: int64(8 * time.Millisecond), PublicationTicketCompleted: int64(10 * time.Millisecond)} + if tc.change != nil { + tc.change(&pair, offsets) + } + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + pair.project(offsets, true, &out) + require.Equal(t, tc.state, out.States["lane_wait"]) + if tc.state == "measured" { + require.NotNil(t, out.Milliseconds["lane_wait"]) + } else { + require.Nil(t, out.Milliseconds["lane_wait"]) + } + }) + } +} + +func TestCanonicalLaneWaitPhysicalAttemptLogDoesNotRequireTerminal(t *testing.T) { + pair := canonicalLaneWaitEvidence{started: true, acquired: true, startNS: int64(4 * time.Millisecond), acquiredNS: int64(5 * time.Millisecond)} + offsets := map[PublicationPhase]int64{PublicationTicketEnqueued: int64(time.Millisecond), PublicationCycleStarted: int64(2 * time.Millisecond), PublicationPatchExecutionStarted: int64(8 * time.Millisecond)} + out := EditPhaseTimings{Milliseconds: map[string]*float64{}, States: map[string]string{}} + pair.project(offsets, false, &out) + require.Equal(t, "measured", out.States["lane_wait"]) + require.Equal(t, 1.0, *out.Milliseconds["lane_wait"]) +} diff --git a/internal/indexer/canonical_patch_laps.go b/internal/indexer/canonical_patch_laps.go new file mode 100644 index 000000000..650e616e9 --- /dev/null +++ b/internal/indexer/canonical_patch_laps.go @@ -0,0 +1,272 @@ +package indexer + +import "time" + +// These are the fixed canonical patch laps, not arbitrary trace categories. +// One attempt owns bounded scalar state, regardless of files/chunks/retries. +type canonicalPatchLap uint8 + +const ( + patchGateWait canonicalPatchLap = iota + patchParse + patchApply + patchSearch + patchResolve + patchMtimes + patchDerived + // Supplementary actual work, kept private until its additive projection is + // reviewed. It is not a replacement for any requested patch lap. + patchBookkeeping + canonicalDeepLapCount +) + +const ( + patchAttemptSetup canonicalPatchLap = canonicalDeepLapCount + iota + patchInspection + patchExecutorControl + patchAttemptFinalize + patchCohortAdmission + patchBatchAdmission + patchRepositoryLane + canonicalPatchLapCount +) + +var canonicalPatchLapNames = [...]string{ + "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", +} + +type canonicalLapTotal struct { + total time.Duration + first, last time.Time + begun, done uint64 + incomplete bool +} + +type canonicalPatchLaps struct { + laps [canonicalPatchLapCount]canonicalLapTotal + lastEnd time.Time + active int + activeKind canonicalPatchLap + activeStarted time.Time + controlObserved, controlFinished bool + controlKind canonicalPatchLap + attemptStarted, attemptFinished time.Time + invalid bool + nativeMtimeDuplicate bool // observed early return before the executor + origin, enqueued time.Time // immutable original admission identity, for attempt logs + ticket uint64 +} + +// This disposition belongs only to the native mtime gate, not to a failed or +// otherwise unobserved executor. It adds no clock read or allocation. +func (s *PublicationStamps) recordNativeMtimeDuplicate() { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if p := s.patchLaps; p != nil { + if p.nativeMtimeDuplicate { + p.invalid = true + } + p.nativeMtimeDuplicate = true + } +} + +func (p *PublicationPhaseRecord) newCanonicalPatchStamps() *PublicationStamps { + p.mu.Lock() + defer p.mu.Unlock() + laps := &canonicalPatchLaps{origin: p.origin, ticket: p.ticket} + for _, mark := range p.marks { + if mark.phase == PublicationTicketEnqueued { + laps.enqueued = mark.at + break + } + } + return &PublicationStamps{patchLaps: laps} +} + +type canonicalLapSpan struct { + stamps *PublicationStamps + lap canonicalPatchLap + started time.Time +} + +func (s *PublicationStamps) beginPatchLap(lap canonicalPatchLap) canonicalLapSpan { + if s == nil || s.patchLaps == nil { + return canonicalLapSpan{} // no clock/allocation on unobserved paths + } + return s.beginPatchLapAt(lap, time.Now()) +} + +func (s *PublicationStamps) beginPatchLapAt(lap canonicalPatchLap, at time.Time) canonicalLapSpan { + s.mu.Lock() + defer s.mu.Unlock() + p := s.patchLaps + if p.controlObserved && !p.controlFinished && p.active == 1 && isAttemptControl(p.activeKind) { + p.closeAttemptControlLocked(at) + } + p.beginLapLocked(lap, at) + return canonicalLapSpan{stamps: s, lap: lap, started: at} +} + +func (p *canonicalPatchLaps) beginLapLocked(lap canonicalPatchLap, at time.Time) { + v := &p.laps[lap] + if at.IsZero() || p.active != 0 || !p.lastEnd.IsZero() && at.Before(p.lastEnd) { + p.invalid = true + } + if v.begun == 0 { + v.first = at + } + v.begun++ + if p.active == 0 { + p.activeKind, p.activeStarted = lap, at + } + p.active++ +} + +func (p *canonicalPatchLaps) endLapLocked(lap canonicalPatchLap, start, at time.Time) { + v := &p.laps[lap] + if at.IsZero() || at.Before(start) || p.active != 1 || p.activeKind != lap || !p.activeStarted.Equal(start) { + p.invalid = true + } else if elapsed := at.Sub(start); elapsed < 0 || v.total > time.Duration(1<<63-1)-elapsed { + p.invalid = true + } else { + v.total += elapsed + } + v.last = at + v.done++ + p.active-- + p.lastEnd = at +} + +func (span *canonicalLapSpan) end() time.Time { + if span.stamps == nil { + return time.Time{} + } + return span.endAt(time.Now()) +} + +func (span *canonicalLapSpan) endAt(at time.Time) time.Time { + s := span.stamps + if s == nil { + return time.Time{} + } + span.stamps = nil // completion is idempotent; no closure or per-row atomics + s.mu.Lock() + defer s.mu.Unlock() + p := s.patchLaps + p.endLapLocked(span.lap, span.started, at) + if p.active == 0 { + p.openAttemptControlLocked(at) + } + return at +} + +func (s *PublicationStamps) incompletePatchLaps() { + if s == nil || s.patchLaps == nil { + return + } + s.mu.Lock() + // The legacy fallback has inseparable extraction/application/search work. + // Previously measured chunks cannot make these complete for this attempt. + for _, lap := range [...]canonicalPatchLap{patchParse, patchApply, patchSearch} { + s.patchLaps.laps[lap].incomplete = true + } + s.mu.Unlock() +} + +type canonicalLapEvidence struct { + totalNS, firstNS, lastNS int64 + begun, done uint64 + incomplete bool +} + +type canonicalPatchEvidence struct { + laps [canonicalDeepLapCount]canonicalLapEvidence + invalid bool +} + +// freeze converts only same-process monotonic readings, never wall strings. +// Called under the collector lock; the returned value has no mutable aliases. +func (p *canonicalPatchLaps) freeze(origin time.Time) canonicalPatchEvidence { + openOuterControl := p.controlObserved && p.active == 1 && isAttemptControl(p.activeKind) + out := canonicalPatchEvidence{invalid: p.invalid || p.active != 0 && !openOuterControl} + for i, v := range p.laps[:canonicalDeepLapCount] { + out.laps[i] = canonicalLapEvidence{totalNS: int64(v.total), begun: v.begun, done: v.done, incomplete: v.incomplete} + if v.begun != 0 { + out.laps[i].firstNS, out.laps[i].lastNS = v.first.Sub(origin).Nanoseconds(), v.last.Sub(origin).Nanoseconds() + } + } + return out +} + +// project validates global disjointness and enclosing ownership before ANY +// measured scalar becomes visible. Incomplete/missing laps remain unavailable. +func (p *canonicalPatchEvidence) project(offsets map[PublicationPhase]int64, terminal bool, out *EditPhaseTimings) { + start, hasStart := offsets[PublicationPatchExecutionStarted] + end, hasEnd := offsets[PublicationPatchExecutionFinished] + enqueue, hasEnqueue := offsets[PublicationTicketEnqueued] + invalid := p.invalid || hasEnqueue && enqueue < 0 || hasStart && start < enqueue || hasEnd && end < start + if terminal { + last := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + last = PublicationTicketFailed + } + if finished, ok := offsets[last]; ok && hasEnd && end > finished { + invalid = true + } + } + var sum int64 + for _, v := range p.laps { + if v.begun == 0 { + continue + } + if v.totalNS < 0 || v.firstNS < 0 || v.lastNS < v.firstNS || hasStart && v.firstNS < start || hasEnd && v.lastNS > end || v.totalNS > v.lastNS-v.firstNS { + invalid = true + } + sum += v.totalNS + } + if hasStart && hasEnd && sum > end-start { + invalid = true + } + // The old derived pair and the new scalar must describe the SAME call, + // not two intervals that could overlap or be counted twice. + derived := p.laps[patchDerived] + if a, ok := offsets[PublicationPatchDerivedStarted]; ok { + b, hasB := offsets[PublicationPatchDerivedFinished] + if !hasB || derived.begun != 1 || derived.done != 1 || derived.firstNS != a || derived.lastNS != b || derived.totalNS != b-a { + invalid = true + } + } else if derived.begun != 0 { + invalid = true + } + for i, name := range canonicalPatchLapNames { + v := p.laps[i] + out.Milliseconds[name], out.States[name] = nil, "not_observed" + if invalid { + out.States[name] = "invalid_order" + continue + } + if !hasStart || !hasEnd || !hasEnqueue || v.begun == 0 || v.begun != v.done || v.incomplete { + continue + } + ms := float64(v.totalNS) / 1e6 + out.Milliseconds[name], out.States[name] = &ms, "measured" + } +} + +func (s *PublicationStamps) beginDerivedPatchLap() canonicalLapSpan { + span := s.beginPatchLap(patchDerived) + if span.stamps != nil { + s.markAt(PublicationPatchDerivedStarted, span.started) + } + return span +} + +func (span *canonicalLapSpan) endDerived() { + s := span.stamps + if at := span.end(); !at.IsZero() { + s.markAt(PublicationPatchDerivedFinished, at) + } +} diff --git a/internal/indexer/canonical_patch_laps_test.go b/internal/indexer/canonical_patch_laps_test.go new file mode 100644 index 000000000..2d9d4e2c5 --- /dev/null +++ b/internal/indexer/canonical_patch_laps_test.go @@ -0,0 +1,132 @@ +package indexer + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func scalarPatchFixture(t *testing.T) (*PublicationPhaseRecord, *PublicationStamps, time.Time) { + t.Helper() + origin := time.Now() + r := NewPublicationPhaseRecorder(1, 1).Begin("", "scalar", PublicationSourceMCPEdit, origin) + require.True(t, r.bindCanonicalTicket(1, origin.Add(time.Millisecond))) + s := &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + s.markAt(PublicationPatchExecutionStarted, origin.Add(2*time.Millisecond)) + return r, s, origin +} + +func scalarSpan(s *PublicationStamps, lap canonicalPatchLap, origin time.Time, from, through time.Duration) { + span := s.beginPatchLapAt(lap, origin.Add(from)) + span.endAt(origin.Add(through)) +} + +func TestCanonicalPatchScalarsAggregateRepeatedDisjointSpans(t *testing.T) { + r, s, origin := scalarPatchFixture(t) + for _, span := range []struct { + lap canonicalPatchLap + a, b time.Duration + }{ + {patchGateWait, 2, 3}, {patchParse, 3, 4}, {patchApply, 4, 5}, {patchSearch, 5, 6}, + {patchApply, 6, 7}, {patchParse, 8, 9}, {patchResolve, 9, 10}, {patchMtimes, 10, 11}, {patchDerived, 12, 13}, + } { + scalarSpan(s, span.lap, origin, span.a*time.Millisecond, span.b*time.Millisecond) + } + s.markAt(PublicationPatchDerivedStarted, origin.Add(12*time.Millisecond)) + s.markAt(PublicationPatchDerivedFinished, origin.Add(13*time.Millisecond)) + s.markAt(PublicationPatchExecutionFinished, origin.Add(14*time.Millisecond)) + r.Absorb(s) + r.MarkAt(PublicationTicketCompleted, origin.Add(15*time.Millisecond)) + snapshot := r.Snapshot() + timings := snapshot.EditTimings() + require.Equal(t, 2.0, *timings.Milliseconds["apply"]) + require.Equal(t, 2.0, *timings.Milliseconds["patch_parse"]) + require.Equal(t, 1.0, *timings.Milliseconds["search_lap"]) + require.Equal(t, 6.0, *timings.Window.PublicationUnattributedWallMS) + for _, name := range canonicalPatchLapNames { + require.Equal(t, "measured", timings.States[name], name) + } + encoded, err := json.Marshal(snapshot) + require.NoError(t, err) + snapshot.patchLaps = nil + legacy, err := json.Marshal(snapshot) + require.NoError(t, err) + require.Equal(t, string(legacy), string(encoded), "scalar evidence must not change the legacy timeline wire") + // The record owns a frozen copy, not the still-accessible local collector. + scalarSpan(s, patchApply, origin, 16*time.Millisecond, 20*time.Millisecond) + require.Equal(t, 2.0, *r.Snapshot().EditTimings().Milliseconds["apply"]) +} + +func TestCanonicalPatchScalarsRejectOverlapAndUnfinishedWork(t *testing.T) { + for _, unfinished := range []bool{false, true} { + t.Run(map[bool]string{false: "overlap", true: "unfinished"}[unfinished], func(t *testing.T) { + r, s, origin := scalarPatchFixture(t) + parse := s.beginPatchLapAt(patchParse, origin.Add(3*time.Millisecond)) + if !unfinished { + apply := s.beginPatchLapAt(patchApply, origin.Add(4*time.Millisecond)) + parse.endAt(origin.Add(5 * time.Millisecond)) + apply.endAt(origin.Add(6 * time.Millisecond)) + } + s.markAt(PublicationPatchExecutionFinished, origin.Add(7*time.Millisecond)) + r.Absorb(s) + r.MarkAt(PublicationTicketCompleted, origin.Add(8*time.Millisecond)) + out := r.Snapshot().EditTimings() + require.Nil(t, out.Milliseconds["patch_parse"]) + require.Equal(t, "invalid_order", out.States["patch_parse"]) + require.Nil(t, out.Window.PublicationUnattributedWallMS) + }) + } +} + +func TestCanonicalPatchScalarsKeepCombinedFallbackUnavailable(t *testing.T) { + r, s, origin := scalarPatchFixture(t) + scalarSpan(s, patchGateWait, origin, 2*time.Millisecond, 3*time.Millisecond) + scalarSpan(s, patchParse, origin, 3*time.Millisecond, 4*time.Millisecond) + scalarSpan(s, patchApply, origin, 4*time.Millisecond, 5*time.Millisecond) + scalarSpan(s, patchSearch, origin, 5*time.Millisecond, 6*time.Millisecond) + s.incompletePatchLaps() + s.markAt(PublicationPatchExecutionFinished, origin.Add(8*time.Millisecond)) + r.Absorb(s) + out := r.Snapshot().EditTimings() + for _, name := range []string{"patch_parse", "apply", "search_lap"} { + require.Nil(t, out.Milliseconds[name], "prior measured chunks do not complete a combined fallback") + require.Equal(t, "not_observed", out.States[name]) + } + require.Equal(t, "measured", out.States["gate_wait"]) + require.Equal(t, 1.0, *out.Milliseconds["gate_wait"]) +} + +func TestCanonicalPatchScalarsValidateDerivedPairAndOwnership(t *testing.T) { + for _, tc := range []struct { + name string + rawStart, rawEnd time.Duration + }{ + {"different_pair", 5, 7}, {"outside_executor", 9, 10}, + } { + t.Run(tc.name, func(t *testing.T) { + r, s, origin := scalarPatchFixture(t) + scalarSpan(s, patchParse, origin, 3*time.Millisecond, 4*time.Millisecond) + scalarSpan(s, patchDerived, origin, 5*time.Millisecond, 6*time.Millisecond) + s.markAt(PublicationPatchDerivedStarted, origin.Add(tc.rawStart*time.Millisecond)) + s.markAt(PublicationPatchDerivedFinished, origin.Add(tc.rawEnd*time.Millisecond)) + s.markAt(PublicationPatchExecutionFinished, origin.Add(8*time.Millisecond)) + r.Absorb(s) + r.MarkAt(PublicationTicketCompleted, origin.Add(9*time.Millisecond)) + out := r.Snapshot().EditTimings() + require.Nil(t, out.Milliseconds["derived"]) + require.Nil(t, out.Milliseconds["patch_parse"], "a mismatched derived interval must not bless another overlapping subtotal") + require.Nil(t, out.Window.PublicationUnattributedWallMS) + }) + } +} + +func TestCanonicalPatchNilObserverIsInert(t *testing.T) { + var s *PublicationStamps + span := s.beginPatchLap(patchParse) + require.Nil(t, span.stamps) + require.True(t, span.started.IsZero()) + require.True(t, span.end().IsZero()) + s.incompletePatchLaps() +} diff --git a/internal/indexer/canonical_patch_log.go b/internal/indexer/canonical_patch_log.go new file mode 100644 index 000000000..679e745a7 --- /dev/null +++ b/internal/indexer/canonical_patch_log.go @@ -0,0 +1,88 @@ +package indexer + +import "go.uber.org/zap/zapcore" + +// canonicalPatchLog is immutable physical-attempt evidence. Logging happens +// before final watcher winner validation, so it must never promise that these +// observations will be attached to a receipt. Encoding only formats the copy. +type canonicalPatchLog struct { + ticket uint64 + values [canonicalPatchLapCount]float64 + states [canonicalPatchLapCount]string + counts [canonicalPatchLapCount]uint64 + laneWaitMS *float64 + laneWaitState string + debounceMS *float64 + debounceState string + admissionMS [canonicalAdmissionCount]*float64 + admissionStates [canonicalAdmissionCount]string +} + +func (s *PublicationStamps) snapshotPatchLog() (canonicalPatchLog, bool) { + if s == nil || s.patchLaps == nil { + return canonicalPatchLog{}, false // no new native-path clocks/allocations/encoder + } + s.mu.Lock() + origin := s.patchLaps.origin + evidence := s.patchLaps.freeze(origin) + lane := s.laneWait.freeze(origin) + debounce := s.debounce + admissions := s.freezeCanonicalAdmissions(origin) + ticket := s.patchLaps.ticket + offsets := make(map[PublicationPhase]int64, len(s.marks)+1) + if !s.patchLaps.enqueued.IsZero() { + offsets[PublicationTicketEnqueued] = s.patchLaps.enqueued.Sub(origin).Nanoseconds() + } + for _, mark := range s.marks { + if _, exists := offsets[mark.phase]; !exists { + offsets[mark.phase] = mark.at.Sub(origin).Nanoseconds() + } + } + s.mu.Unlock() + projection := EditPhaseTimings{Milliseconds: make(map[string]*float64, canonicalPatchLapCount), States: make(map[string]string, canonicalPatchLapCount)} + evidence.project(offsets, false, &projection) + lane.project(offsets, false, &projection) + admissions.project(offsets, false, &lane, &projection) + projection.States["debounce"] = "not_observed" + if !debounce.started.IsZero() { + debounce.freeze(origin).project(offsets, false, &projection) + } + out := canonicalPatchLog{ticket: ticket, laneWaitMS: projection.Milliseconds["lane_wait"], laneWaitState: projection.States["lane_wait"], debounceMS: projection.Milliseconds["debounce"], debounceState: projection.States["debounce"]} + for i, name := range canonicalAdmissionNames { + out.admissionMS[i], out.admissionStates[i] = projection.Milliseconds[name], projection.States[name] + } + for i, name := range canonicalPatchLapNames { + out.states[i], out.counts[i] = projection.States[name], evidence.laps[i].done + if ms := projection.Milliseconds[name]; ms != nil { + out.values[i] = *ms + } + } + return out, true +} + +func (p canonicalPatchLog) MarshalLogObject(enc zapcore.ObjectEncoder) error { + enc.AddString("patch_lap_scope", "physical_attempt_before_receipt_validation") + enc.AddUint64("attempt_ticket", p.ticket) + enc.AddString("attempt_debounce_state", p.debounceState) + if p.debounceMS != nil { + enc.AddFloat64("attempt_debounce_ms", *p.debounceMS) + } + enc.AddString("attempt_lane_wait_state", p.laneWaitState) + if p.laneWaitMS != nil { + enc.AddFloat64("attempt_lane_wait_ms", *p.laneWaitMS) + } + for i, name := range canonicalAdmissionNames { + enc.AddString("attempt_"+name+"_state", p.admissionStates[i]) + if p.admissionMS[i] != nil { + enc.AddFloat64("attempt_"+name+"_ms", *p.admissionMS[i]) + } + } + for i, name := range canonicalPatchLapNames { + enc.AddString("attempt_"+name+"_state", p.states[i]) + enc.AddUint64("attempt_"+name+"_count", p.counts[i]) + if p.states[i] == "measured" { + enc.AddFloat64("attempt_"+name+"_ms", p.values[i]) + } + } + return nil +} diff --git a/internal/indexer/canonical_patch_log_test.go b/internal/indexer/canonical_patch_log_test.go new file mode 100644 index 000000000..145cfb61d --- /dev/null +++ b/internal/indexer/canonical_patch_log_test.go @@ -0,0 +1,82 @@ +package indexer + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +func TestCanonicalPatchLogOwnsItsSnapshotBeforeWinnerPublication(t *testing.T) { + path, idx, w := pointPublicationFixture(t) + core, logs := observer.New(zap.InfoLevel) + w.logger = zap.New(core) + var local *PublicationStamps + w.pointReindexRaw = func(path string, s *PublicationStamps) (*IndexResult, error) { + local = s + return idx.incrementalPointWatcherPathObserved(idx.rootPath, path, s) + } + record := newPointPublication(t, "log") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + entries := logs.FilterMessage("graph patch complete").All() + require.Len(t, entries, 1, "instrumentation must extend the existing line, not emit another patch line") + fields := entries[0].ContextMap() + require.Equal(t, "physical_attempt_before_receipt_validation", fields["patch_lap_scope"]) + require.Equal(t, ticket.Generation, fields["attempt_ticket"]) + require.Eventually(t, func() bool { return record.Snapshot().attemptWindow != nil }, 2*time.Second, time.Millisecond) + status := record.Snapshot().EditTimings() + require.Equal(t, "measured", status.States["debounce"]) + require.Equal(t, status.States["debounce"], fields["attempt_debounce_state"]) + require.Equal(t, *status.Milliseconds["debounce"], fields["attempt_debounce_ms"]) + require.Equal(t, status.States["lane_wait"], fields["attempt_lane_wait_state"]) + require.NotNil(t, status.Milliseconds["lane_wait"]) + require.Equal(t, *status.Milliseconds["lane_wait"], fields["attempt_lane_wait_ms"]) + for _, name := range canonicalAdmissionNames { + require.Equal(t, status.States[name], fields["attempt_"+name+"_state"], name) + if ms := status.Milliseconds[name]; ms != nil { + require.Equal(t, *ms, fields["attempt_"+name+"_ms"], name) + } + } + for _, name := range canonicalPatchLapNames { + require.Equal(t, status.States[name], fields["attempt_"+name+"_state"], name) + if ms := status.Milliseconds[name]; ms != nil { + require.Equal(t, *ms, fields["attempt_"+name+"_ms"], name) + } + } + // Actual unrelated graph work and a changed local collector must affect + // neither the already-emitted log nor the receipt's accepted scalar copy. + other := filepath.Join(filepath.Dir(path), "other.go") + writeTestFile(t, other, "package p\nfunc Unrelated() {}\n") + require.NoError(t, idx.IndexFile(other)) + late := local.beginPatchLap(patchParse) + late.end() + require.Equal(t, fields, entries[0].ContextMap(), "encoders may be evaluated later or more than once") + require.Equal(t, status, record.Snapshot().EditTimings()) +} + +func TestCanonicalPatchNativeLogKeepsOriginalFieldsOnly(t *testing.T) { + path, _, w := pointPublicationFixture(t) + core, logs := observer.New(zap.InfoLevel) + w.logger = zap.New(core) + require.NoError(t, w.patchGraphAfterReceiptCheck(path, ChangeModified)) + entries := logs.FilterMessage("graph patch complete").All() + require.Len(t, entries, 1) + fields := entries[0].ContextMap() + require.Contains(t, fields, "incident_edges") + require.Contains(t, fields, "ms") + for key := range fields { + require.False(t, strings.HasPrefix(key, "attempt_"), key) + } + require.NotContains(t, fields, "patch_lap_scope") + var s *PublicationStamps + var observed bool + allocs := testing.AllocsPerRun(100, func() { _, observed = s.snapshotPatchLog() }) + require.False(t, observed) + require.Zero(t, allocs, "the nil-observer log branch must not allocate") +} diff --git a/internal/indexer/canonical_patch_pipeline_test.go b/internal/indexer/canonical_patch_pipeline_test.go new file mode 100644 index 000000000..b24a15c7b --- /dev/null +++ b/internal/indexer/canonical_patch_pipeline_test.go @@ -0,0 +1,167 @@ +package indexer + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer/source" +) + +func beginRealPatchObservation(t *testing.T) (*PublicationPhaseRecord, *PublicationStamps) { + t.Helper() + r := newPointPublication(t, "real-core") + require.True(t, r.bindCanonicalTicket(1, time.Now())) + s := &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + s.markAt(PublicationPatchExecutionStarted, time.Now()) + return r, s +} + +func finishRealPatchObservation(r *PublicationPhaseRecord, s *PublicationStamps) PublicationPhaseSnapshot { + s.markAt(PublicationPatchExecutionFinished, time.Now()) + r.Absorb(s) + r.Mark(PublicationTicketCompleted) + return r.Snapshot() +} + +func TestCanonicalPatchRealChunksAggregateWithoutChargingInterleavedWork(t *testing.T) { + root := t.TempDir() + idx := newTestIndexer(graph.New()) + idx.SetRootPath(root) + var paths []string + for i := range incrementalBatchFiles + 1 { + path := filepath.Join(root, fmt.Sprintf("f%02d.go", i)) + writeTestFile(t, path, fmt.Sprintf("package p\nfunc Before%02d() {}\n", i)) + paths = append(paths, path) + } + _, err := idx.Index(root) + require.NoError(t, err) + for i, path := range paths { + bumpMtime(t, path, fmt.Sprintf("package p\nfunc After%02d() { After%02d() }\n", i, (i+1)%len(paths))) + } + r, s := beginRealPatchObservation(t) + observedInterleavedGap := false + idx.affectedByDeltaHook = func(string, []string) { + if observedInterleavedGap { + return + } + s.mu.Lock() + parseEnd := s.patchLaps.laps[patchParse].last + s.mu.Unlock() + require.False(t, parseEnd.IsZero(), "the real apply hook follows a completed parse chunk") + require.Eventually(t, func() bool { return time.Now().After(parseEnd) }, 5*time.Second, time.Millisecond, + "observe an actual interleaved apply interval before the next parse chunk") + observedInterleavedGap = true + } + var bookkeepingHooks []string + idx.incrementalCatchupHook = func(kind string, _ []string) { + if kind != "dataflow" && kind != "ref_facts" { + return + } + s.mu.Lock() + active := s.patchLaps.active + bookkeeping := s.patchLaps.laps[patchBookkeeping] + s.mu.Unlock() + require.Equal(t, 1, active, "actual follow-up work must own one disjoint interval") + require.Equal(t, bookkeeping.done+1, bookkeeping.begun, kind) + bookkeepingHooks = append(bookkeepingHooks, kind) + } + result, err := idx.incrementalWatcherPaths(root, paths, incrementalPathMode{ + detectDeletions: true, forceExplicitFiles: true, publication: s, + }) + require.NoError(t, err) + require.Empty(t, result.FailedFiles) + for i, path := range paths { + caller := nodeByName(t, idx, filepath.Base(path), fmt.Sprintf("After%02d", i)) + next := (i + 1) % len(paths) + callee := nodeByName(t, idx, filepath.Base(paths[next]), fmt.Sprintf("After%02d", next)) + require.Equal(t, callee.ID, callEdge(t, idx, caller.ID).To, "the actual deferred resolver must bind the new cross-file call") + } + snapshot := finishRealPatchObservation(r, s) + evidence := snapshot.patchLaps + require.NotNil(t, evidence) + require.False(t, evidence.invalid) + require.Equal(t, []string{"dataflow", "ref_facts"}, bookkeepingHooks, "the actual follow-up passes must still run once in order") + require.Equal(t, uint64(1), evidence.laps[patchBookkeeping].done) + require.Equal(t, evidence.laps[patchBookkeeping].begun, evidence.laps[patchBookkeeping].done) + require.GreaterOrEqual(t, evidence.laps[patchParse].done, uint64(2), "must execute actual repeated chunks") + require.GreaterOrEqual(t, evidence.laps[patchSearch].done, uint64(2)) + require.GreaterOrEqual(t, evidence.laps[patchApply].done, uint64(4), "apply is split at each real search boundary") + parse := evidence.laps[patchParse] + require.True(t, observedInterleavedGap) + require.Less(t, parse.totalNS, parse.lastNS-parse.firstNS, "the first→last parse envelope includes intervening apply/search and must not be used as parse time") + var sum int64 + timings := snapshot.EditTimings() + require.NotContains(t, timings.Milliseconds, "bookkeeping", "private evidence does not silently change the requested lap contract") + for i, name := range canonicalPatchLapNames { + v := evidence.laps[i] + require.Equal(t, v.begun, v.done, name) + require.Equal(t, "measured", timings.States[name], name) + require.Equal(t, float64(v.totalNS)/1e6, *timings.Milliseconds[name], name) + sum += v.totalNS + } + offsets := phaseOffsets(snapshot) + require.LessOrEqual(t, sum, offsets[PublicationPatchExecutionFinished]-offsets[PublicationPatchExecutionStarted]) + require.NotNil(t, timings.Window.PublicationUnattributedWallMS) +} + +// This controlled source changes only the prepared refresh read. It forces the +// real legacy combined fallback through an existing source seam, without a +// production knob or timing-dependent filesystem race. All later reads serve +// the actual owned file bytes again. +type refreshMismatchSource struct { + source.ContentSource + reads atomic.Int32 +} + +func (s *refreshMismatchSource) Open(path string) (io.ReadCloser, source.FileMeta, error) { + rc, meta, err := s.ContentSource.Open(path) + if err != nil || path != "edit.go" || s.reads.Add(1) != 2 { + return rc, meta, err + } + data, err := io.ReadAll(rc) + _ = rc.Close() + if err != nil { + return nil, meta, err + } + data = append(data, []byte("\n// prepared refresh moved on\n")...) + meta.Size = int64(len(data)) + return io.NopCloser(bytes.NewReader(data)), meta, nil +} + +func TestCanonicalPatchRealCombinedFallbackCannotPublishPartialLapTotals(t *testing.T) { + path, idx, _ := pointPublicationFixture(t) + fs, err := source.NewFilesystemSource(filepath.Dir(path)) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, fs.Close()) }) + changing := &refreshMismatchSource{ContentSource: fs} + idx.SetContentSource(changing) + t.Cleanup(func() { idx.SetContentSource(nil) }) + r, s := beginRealPatchObservation(t) + _, _, reparsed, failed, _ := idx.reindexIncrementalChunk([]string{path}, &reparsePendingEnrichmentBatch{ + deferResolverCatchup: true, publication: s, + }, false) + require.Empty(t, failed) + require.Contains(t, reparsed, path) + require.GreaterOrEqual(t, changing.reads.Load(), int32(3), "speculative read, changed refresh and actual fallback must all run") + require.NotNil(t, nodeByName(t, idx, "edit.go", "After"), "the fallback must actually parse and apply") + snapshot := finishRealPatchObservation(r, s) + for _, lap := range []canonicalPatchLap{patchParse, patchApply, patchSearch} { + require.True(t, snapshot.patchLaps.laps[lap].incomplete) + name := canonicalPatchLapNames[lap] + require.Nil(t, snapshot.EditTimings().Milliseconds[name]) + require.Equal(t, "not_observed", snapshot.EditTimings().States[name]) + } + require.FileExists(t, path) + data, err := os.ReadFile(path) + require.NoError(t, err) + require.NotContains(t, string(data), "prepared refresh moved on", "the diagnostic source must not change actual disk content") +} diff --git a/internal/indexer/canonical_requested_background.go b/internal/indexer/canonical_requested_background.go new file mode 100644 index 000000000..ed204c68f --- /dev/null +++ b/internal/indexer/canonical_requested_background.go @@ -0,0 +1,445 @@ +package indexer + +import ( + "context" + "math" + "sync" + "time" + "weak" +) + +type canonicalSchedulingKey struct{} + +// Preserve the outer scheduler's actual entry across MultiWatcher routing. +// Native/unobserved admission performs no clock read or context allocation. +func canonicalSchedulingEntry(ctx context.Context) (context.Context, time.Time) { + if entered, ok := ctx.Value(canonicalSchedulingKey{}).(time.Time); ok { + return ctx, entered + } + if PublicationRecordFromContext(ctx) == nil { + return ctx, time.Time{} + } + entered := time.Now() + return context.WithValue(ctx, canonicalSchedulingKey{}, entered), entered +} + +type canonicalBackgroundStage uint8 + +const ( + backgroundAdmission canonicalBackgroundStage = iota + backgroundInitialArm + backgroundInitialQuiet + backgroundHandoff + backgroundNativeArm + backgroundNativeQuiet + backgroundExecuting + backgroundReturned +) + +// One ordered scalar per named lap, not an interval/timer history. Native +// handoffs are counted only after actual successful Stop and actual next arm. +type canonicalBackgroundOwner struct { + mu sync.Mutex + record weak.Pointer[PublicationPhaseRecord] + watcher weak.Pointer[Watcher] + path, key string + requested, current, handoffs uint64 + admissionPredecessor uint64 + origin, last, armStart, entered time.Time + phases [8]int64 + stage canonicalBackgroundStage + invalid bool + reason string +} + +func newCanonicalBackground(record *PublicationPhaseRecord, w *Watcher, path string, requested uint64, entered time.Time) *canonicalBackgroundOwner { + if record == nil || w == nil || path == "" || requested == 0 || entered.IsZero() { + return nil + } + return &canonicalBackgroundOwner{record: weak.Make(record), watcher: weak.Make(w), path: path, key: record.key, requested: requested, current: requested, origin: entered, last: entered} +} +func (b *canonicalBackgroundOwner) failLocked(reason string) { + b.invalid = true + if b.reason == "" { + b.reason = reason + } +} +func (b *canonicalBackgroundOwner) invalidate(reason string) { + b.mu.Lock() + b.failLocked(reason) + b.mu.Unlock() +} +func (b *canonicalBackgroundOwner) matchesLocked(w *Watcher, path string, generation uint64) bool { + if b.watcher.Value() != w || b.path != path || b.current != generation { + b.failLocked("foreign_timer_identity") + return false + } + return !b.invalid +} +func (b *canonicalBackgroundOwner) addLocked(lap int, start, end time.Time) bool { + if start.IsZero() || end.IsZero() || end.Before(start) { + b.failLocked("invalid_timing_order") + return false + } + delta := end.Sub(start).Nanoseconds() + if delta < 0 || b.phases[lap] > math.MaxInt64-delta { + b.failLocked("timing_overflow") + return false + } + b.phases[lap] += delta + return true +} +func (b *canonicalBackgroundOwner) armStarted(w *Watcher, path string, generation uint64, successor bool, at time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + if b.invalid { + return + } + if successor { + if b.watcher.Value() != w || b.path != path || generation <= b.current { + b.failLocked("foreign_timer_identity") + return + } + if b.stage != backgroundHandoff { + b.failLocked("unobserved_timer_handoff") + return + } + if b.handoffs == math.MaxUint64 { + b.failLocked("handoff_count_overflow") + return + } + if !b.addLocked(4, b.last, at) { + return + } + b.current = generation + b.handoffs++ + b.armStart = at + b.stage = backgroundNativeArm + } else { + if !b.matchesLocked(w, path, generation) { + return + } + if b.stage != backgroundAdmission { + b.failLocked("repeated_timer_arm") + return + } + if !b.addLocked(0, b.last, at) { + return + } + b.armStart = at + b.stage = backgroundInitialArm + } + b.last = at +} +func (b *canonicalBackgroundOwner) armFinished(w *Watcher, path string, generation uint64, successor bool, at time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + if !b.matchesLocked(w, path, generation) { + return + } + lap, expected, next := 1, backgroundInitialArm, backgroundInitialQuiet + if successor { + lap, expected, next = 5, backgroundNativeArm, backgroundNativeQuiet + } + if b.stage != expected { + b.failLocked("missing_timer_arm") + return + } + if b.addLocked(lap, b.last, at) { + b.last = at + b.stage = next + } +} +func (b *canonicalBackgroundOwner) stopped(w *Watcher, path string, generation uint64, successor bool, start, end time.Time, stopped, native bool) *canonicalBackgroundOwner { + b.mu.Lock() + defer b.mu.Unlock() + if !b.matchesLocked(w, path, generation) { + return nil + } + if !native { + b.failLocked("observed_successor") + return nil + } + if !stopped { + b.failLocked("timer_stop_not_successful") + return nil + } + if b.stage == backgroundExecuting || b.stage == backgroundReturned { + b.failLocked("timer_callback_already_started") + return nil + } + lap, expected := 2, backgroundInitialQuiet + if successor { + lap, expected = 6, backgroundNativeQuiet + } + if b.stage != expected { + b.failLocked("unobserved_timer_handoff") + return nil + } + if !b.addLocked(lap, b.last, start) || !b.addLocked(3, start, end) { + return nil + } + b.last = end + b.stage = backgroundHandoff + return b +} +func (b *canonicalBackgroundOwner) callback(w *Watcher, path string, generation uint64, successor bool, at time.Time) { + b.mu.Lock() + if b.matchesLocked(w, path, generation) { + if !successor { + if b.stage != backgroundInitialQuiet { + b.failLocked("timer_callback_already_started") + } else if b.addLocked(7, b.last, at) { + b.last = at + b.entered = at + b.stage = backgroundExecuting + } + } else if b.stage != backgroundNativeQuiet { + b.failLocked("timer_callback_already_started") + } else if b.addLocked(6, b.last, at) { + b.last = at + b.entered = at + b.stage = backgroundExecuting + } + } + b.mu.Unlock() + b.publishUnavailable() // Callback has not acquired watcher/coordinator locks. +} + +// Bounded diagnostic only: it provides neither a measured window nor coverage. +// The first actual invalidation stays immutable for this retained incarnation. +type RequestedBackgroundUnavailable struct { + PublicationKey string `json:"publication_key"` + Scope string `json:"scope"` + Reason string `json:"reason"` + RequestedGeneration uint64 `json:"requested_generation"` + LastObservedGeneration uint64 `json:"last_observed_generation"` + ObservedNativeHandoffs uint64 `json:"observed_native_handoffs"` +} + +func (b *canonicalBackgroundOwner) unavailableLocked(reason string) RequestedBackgroundUnavailable { + return RequestedBackgroundUnavailable{Scope: "requested_background_observation", Reason: reason, PublicationKey: b.key, RequestedGeneration: b.requested, LastObservedGeneration: b.current, ObservedNativeHandoffs: b.handoffs} +} + +// Caller must not hold watcher/coordinator locks. Recorder membership fencing +// follows the existing recorder→record order; no expired owner is retained. +func (b *canonicalBackgroundOwner) publishUnavailable() { + b.mu.Lock() + if b.reason == "" { + b.mu.Unlock() + return + } + evidence := b.unavailableLocked(b.reason) + b.mu.Unlock() + record := b.record.Value() + if record == nil { + return + } + owner := record.owner.Value() + if owner == nil { + return + } + owner.mu.Lock() + defer owner.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if owner.byKey[record.key] == record && record.ticket == b.requested && record.background == nil && record.backgroundUnavailable == nil { + record.backgroundUnavailable = &evidence + } +} + +type canonicalBackgroundEvidence struct { + origin time.Time + phases [8]int64 + startNS, endNS, lastArmStartNS int64 + requested, applied, handoffs uint64 + admissionPredecessor uint64 + physical canonicalAttemptWindow + debounce canonicalDebounceEvidence +} + +func (b *canonicalBackgroundOwner) finish(attempt *pointPublicationAttempt, entered, end time.Time, retry int) { + if retry != 0 { + b.invalidate("retry_or_backoff") + b.publishUnavailable() + return + } + if attempt == nil { + return + } + b.mu.Lock() + if b.invalid { + b.mu.Unlock() + b.publishUnavailable() + return + } + if b.stage != backgroundExecuting || b.entered != entered || end.Before(entered) { + b.failLocked("serving_attempt_binding_mismatch") + b.mu.Unlock() + b.publishUnavailable() + return + } + b.stage = backgroundReturned + evidence := canonicalBackgroundEvidence{origin: b.origin, phases: b.phases, requested: b.requested, applied: b.current, handoffs: b.handoffs, admissionPredecessor: b.admissionPredecessor, startNS: entered.Sub(b.origin).Nanoseconds(), endNS: end.Sub(b.origin).Nanoseconds(), lastArmStartNS: b.armStart.Sub(b.origin).Nanoseconds()} + diagnostic := b.unavailableLocked("serving_attempt_binding_mismatch") + b.mu.Unlock() + record := b.record.Value() + if record == nil { + return + } + owner := record.owner.Value() + if owner == nil { + return + } + if !attempt.independent { + b.finishDirect(record, owner, attempt, evidence, entered, end, diagnostic) + return + } + if attempt.record == nil || attempt.record == record || attempt.record.owner.Value() != owner { + b.invalidate("serving_attempt_binding_mismatch") + b.publishUnavailable() + return + } + owner.mu.Lock() + defer owner.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + winner := attempt.record + winner.mu.Lock() + defer winner.mu.Unlock() + if owner.byKey[record.key] != record || owner.byKey[winner.key] != winner || record.ticket != b.requested || record.background != nil || record.backgroundUnavailable != nil { + return + } + if record.coalescedWinner.Value() != winner || winner.attemptWindow == nil || winner.debounce == nil || winner.ticket != evidence.applied || winner.ticket <= b.requested { + record.backgroundUnavailable = &diagnostic + return + } + for _, mark := range record.marks { + if mark.phase == PublicationTicketFailed { + return + } + } + evidence.applied = winner.ticket + evidence.physical = *winner.attemptWindow + evidence.debounce = *winner.debounce + if !winner.origin.Add(time.Duration(evidence.physical.startNS)).Equal(entered) || !winner.origin.Add(time.Duration(evidence.physical.endNS)).Equal(end) || !evidence.valid() { + diagnostic.Reason = "physical_evidence_incomplete" + record.backgroundUnavailable = &diagnostic + return + } + record.background = &evidence // Fresh late scalar; no earlier snapshot mutation. +} +func (b canonicalBackgroundEvidence) valid() bool { + if b.origin.IsZero() || b.requested == 0 || b.applied < b.requested || (b.applied == b.requested && b.handoffs != 0) || (b.applied > b.requested && b.handoffs == 0) || (b.admissionPredecessor != 0 && b.admissionPredecessor >= b.requested) || b.lastArmStartNS < 0 || b.startNS < b.lastArmStartNS || b.endNS < b.startNS { + return false + } + p := b.physical.evaluateAccounting() + if !p.complete || b.physical.retry != 0 || b.physical.ticket != b.applied || b.physical.endNS-b.physical.startNS != b.endNS-b.startNS { + return false + } + if !b.debounce.entered || b.debounce.startedNS < 0 || b.debounce.enteredNS != b.physical.startNS || b.debounce.enteredNS-b.debounce.startedNS != b.startNS-b.lastArmStartNS { + return false + } + sum := int64(0) + for _, delta := range b.phases { + if delta < 0 || sum > math.MaxInt64-delta { + return false + } + sum += delta + } + return sum == b.startNS && p.sumNS >= 0 && sum <= math.MaxInt64-p.sumNS && sum+p.sumNS == b.endNS +} + +var requestedBackgroundLapNames = [...]string{"scheduling_admission_setup", "initial_timer_arm", "canceled_quiet_wait", "timer_stop", "coalescing_control", "successor_timer_arm", "successor_quiet_wait", "requested_quiet_wait"} + +// RequestedBackgroundTiming is one measured scheduler-entry→serving-return +// window. Repeated native arms/quiet/stops are actual disjoint scalar totals; +// every canceled timer must have an observed successful pre-callback Stop. +type RequestedBackgroundTiming struct { + Scope string `json:"scope"` + Coverage string `json:"coverage"` + SumState string `json:"sum_state"` + OriginWall string `json:"origin_wall"` + RequestedGeneration uint64 `json:"requested_generation"` + AppliedGeneration uint64 `json:"applied_generation"` + NativeHandoffs uint64 `json:"observed_native_handoffs"` + AdmissionPredecessor uint64 `json:"admission_predecessor_generation,omitempty"` + AdmissionPredecessorStopped bool `json:"admission_predecessor_stopped,omitempty"` + WallMS float64 `json:"background_wall_ms"` + PhasesMS map[string]float64 `json:"phases_ms"` + PartitionComplete bool `json:"observed_partition_complete"` + UnattributedMS float64 `json:"unattributed_ms"` +} + +func (b canonicalBackgroundEvidence) project(physical *EditPatchAccounting) *RequestedBackgroundTiming { + if physical == nil || !physical.PartitionComplete { + return nil + } + coverage := "one_canceled_timer_one_native_successor" + if b.applied == b.requested { + coverage = "direct_requested_attempt" + } else if b.handoffs > 1 { + coverage = "successful_pre_callback_native_handoffs" + } + out := &RequestedBackgroundTiming{Scope: "accepted_scheduling_call_entry_to_serving_attempt_return", Coverage: coverage, SumState: "partial", OriginWall: b.origin.UTC().Format(time.RFC3339Nano), RequestedGeneration: b.requested, AppliedGeneration: b.applied, NativeHandoffs: b.handoffs, AdmissionPredecessor: b.admissionPredecessor, AdmissionPredecessorStopped: b.admissionPredecessor != 0, WallMS: float64(b.endNS) / 1e6, PhasesMS: make(map[string]float64, len(requestedBackgroundLapNames)+int(canonicalPatchLapCount)), PartitionComplete: true} + for i, name := range requestedBackgroundLapNames { + out.PhasesMS[name] = float64(b.phases[i]) / 1e6 + } + for name, value := range physical.PhasesMS { + if value != nil { + out.PhasesMS[name] = *value + } + } + return out +} +func (p *PublicationPhaseRecord) RequestedBackground() *RequestedBackgroundTiming { + if p == nil { + return nil + } + if owner := p.owner.Value(); owner != nil { + owner.mu.Lock() + p.mu.Lock() + var direct *canonicalBackgroundEvidence + if owner.byKey[p.key] == p && p.background != nil && p.background.requested == p.ticket && p.background.applied == p.ticket && p.attemptWindow != nil && *p.attemptWindow == p.background.physical { + copy := *p.background + direct = © + } + p.mu.Unlock() + owner.mu.Unlock() + if direct != nil { + return direct.project(direct.physical.projectAccounting()) + } + } + if related := p.CoalescedPublication(); related != nil { + return related.Background + } + return nil +} + +// A direct callback owns the original record's accepted P. It must never be +// relabeled as a coalesced/independent winner or borrow an earlier native P. +func (b *canonicalBackgroundOwner) finishDirect(record *PublicationPhaseRecord, owner *PublicationPhaseRecorder, attempt *pointPublicationAttempt, evidence canonicalBackgroundEvidence, entered, end time.Time, diagnostic RequestedBackgroundUnavailable) { + owner.mu.Lock() + defer owner.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if owner.byKey[record.key] != record || record.ticket != b.requested || record.background != nil || record.backgroundUnavailable != nil { + return + } + if attempt.record != record || evidence.requested != evidence.applied || evidence.handoffs != 0 || record.coalescedWinner.Value() != nil || record.attemptWindow == nil || record.debounce == nil { + record.backgroundUnavailable = &diagnostic + return + } + evidence.physical, evidence.debounce = *record.attemptWindow, *record.debounce + if !record.origin.Add(time.Duration(evidence.physical.startNS)).Equal(entered) || !record.origin.Add(time.Duration(evidence.physical.endNS)).Equal(end) || !evidence.valid() { + diagnostic.Reason = "physical_evidence_incomplete" + record.backgroundUnavailable = &diagnostic + return + } + for _, mark := range record.marks { + if mark.phase == PublicationTicketFailed { + return + } + } + record.background = &evidence +} diff --git a/internal/indexer/canonical_requested_background_test.go b/internal/indexer/canonical_requested_background_test.go new file mode 100644 index 000000000..529486581 --- /dev/null +++ b/internal/indexer/canonical_requested_background_test.go @@ -0,0 +1,302 @@ +package indexer + +import ( + "context" + "math" + "runtime" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRequestedBackgroundMeasuresCanceledInitialTimerAndActualP(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + record := newPointPublication(t, "background") + reached, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + w.mutationBeforeAdmission = func() { close(reached); <-release } + type admitted struct { + ticket *MutationTicket + err error + } + admission := make(chan admitted, 1) + go func() { + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + admission <- admitted{ticket, err} + }() + awaitPublicationSignal(t, reached) + gateReleased := time.Now() + unblock() + got := <-admission + require.NoError(t, got.err) + first := got.ticket + require.NotNil(t, first) + w.mu.Lock() + owner := w.mutationWaiters[path][first.Generation].timerObservation.background + w.mu.Unlock() + require.NotNil(t, owner) + owner.mu.Lock() + entered, armStarted := owner.origin, owner.armStart + owner.mu.Unlock() + require.False(t, entered.After(gateReleased), "actual scheduler entry precedes blocked admission") + require.False(t, armStarted.Before(gateReleased), "admission delay remains inside the measured prefix") + before := record.RequestedBackground() + require.Nil(t, before) + native := w.scheduleFileMutation(path, ChangeModified) + require.NotNil(t, native) + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + result := awaitMutationRetryResult(t, first) + require.NoError(t, result.Err) + require.Equal(t, native.Generation, result.AppliedGeneration) + require.NoError(t, awaitMutationRetryResult(t, native).Err) + require.Eventually(t, func() bool { return record.RequestedBackground() != nil }, 5*time.Second, time.Millisecond) + background := record.RequestedBackground() + require.True(t, background.PartitionComplete) + require.Equal(t, first.Generation, background.RequestedGeneration) + require.Equal(t, native.Generation, background.AppliedGeneration) + require.Equal(t, entered.UTC().Format(time.RFC3339Nano), background.OriginWall) + require.Equal(t, "accepted_scheduling_call_entry_to_serving_attempt_return", background.Scope) + require.GreaterOrEqual(t, background.PhasesMS["scheduling_admission_setup"], 0.0) + require.Greater(t, background.PhasesMS["successor_quiet_wait"], 0.0) + require.Zero(t, background.UnattributedMS) + winner := record.CoalescedPublication() + require.NotNil(t, winner.Background, "B and P come from one related immutable observation") + require.NotNil(t, winner.Timings.Window.PatchWallMS) + require.NotNil(t, winner.Timings.Milliseconds["debounce"]) + require.NotContains(t, background.PhasesMS, "debounce", "the whole pair is not added beside B's arm/quiet split") + require.InDelta(t, *winner.Timings.Milliseconds["debounce"], background.PhasesMS["successor_timer_arm"]+background.PhasesMS["successor_quiet_wait"], 0.000001) + requirePointExecution(t, record, false) + sum := 0.0 + for _, value := range background.PhasesMS { + sum += value + } + require.InDelta(t, background.WallMS, sum, 0.000001, "wire conversion only; production validates integer totals") + background.PhasesMS["timer_stop"] = 999 + require.NotEqual(t, 999.0, record.RequestedBackground().PhasesMS["timer_stop"], "projections own their maps") + require.Nil(t, before, "an earlier observation cannot acquire late evidence") + // A retained original cannot resurrect the independent winner after eviction. + recorder := record.owner.Value() + require.NotNil(t, recorder) + recorder.Begin("", "eviction", PublicationSourceMCPEdit, time.Now()) + recorder.Begin("", "eviction2", PublicationSourceMCPEdit, time.Now()) + require.Nil(t, record.RequestedBackground()) + runtime.KeepAlive(recorder) +} + +func TestRequestedBackgroundPreservesAcceptedWinnerDuringLaterAdmission(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + slots := w.mutationSlots() + drained := make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { slots <- struct{}{} }) } + t.Cleanup(unblock) + w.mutationBeforeComplete = func(string, uint64) { <-slots; close(drained) } + record := newPointPublication(t, "background-late-admission") + first := w.scheduleFileMutationWithRecord(path, ChangeModified, record) + native := w.scheduleFileMutation(path, ChangeModified) + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + awaitPublicationSignal(t, drained) + require.NoError(t, awaitMutationRetryResult(t, first).Err) + require.NoError(t, awaitMutationRetryResult(t, native).Err) + require.Nil(t, record.RequestedBackground()) + early := record.CoalescedPublication() + require.NotNil(t, early) + require.Nil(t, early.Background) + require.Nil(t, early.Timings.Window.PatchWallMS) + // Completion already accepted the serving identity and retired its waiter. + // Later work cannot erase this historical observation during final release. + w.scheduleFileMutation("later.go", ChangeModified) + unblock() + require.Eventually(t, func() bool { + winner := record.CoalescedPublication() + return winner != nil && winner.Timings.Window.PatchWallMS != nil + }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { return record.RequestedBackground() != nil }, 5*time.Second, time.Millisecond) + require.Equal(t, native.Generation, record.RequestedBackground().AppliedGeneration) + late := record.CoalescedPublication() + require.NotNil(t, late.Background) + require.NotNil(t, late.Timings.Window.PatchWallMS, "one response cannot pair complete B with older missing P") + require.Nil(t, early.Background) + require.Nil(t, early.Timings.Window.PatchWallMS, "earlier related snapshots remain immutable") +} + +func TestRequestedBackgroundMeasuresNativeChainButRefusesObservedPredecessor(t *testing.T) { + for _, secondObserved := range []bool{false, true} { + t.Run(map[bool]string{false: "second_native_reset", true: "second_observed_owner"}[secondObserved], func(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + firstRecord := newPointPublication(t, "first-background") + first := w.scheduleFileMutationWithRecord(path, ChangeModified, firstRecord) + secondRecord := newPointPublication(t, "second-background") + var second *MutationTicket + if secondObserved { + second = w.scheduleFileMutationWithRecord(path, ChangeModified, secondRecord) + } else { + second = w.scheduleFileMutation(path, ChangeModified) + } + third := w.scheduleFileMutation(path, ChangeModified) + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + for _, ticket := range []*MutationTicket{first, second, third} { + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + } + require.Eventually(t, func() bool { + r := firstRecord + if secondObserved { + r = secondRecord + } + winner := r.CoalescedPublication() + return winner != nil && winner.Timings.Window.PatchWallMS != nil + }, 5*time.Second, time.Millisecond) + if secondObserved { + require.Nil(t, firstRecord.RequestedBackground()) + _, firstFailure := firstRecord.CoalescedPublicationEvidence() + _, secondFailure := secondRecord.CoalescedPublicationEvidence() + require.NotNil(t, firstFailure) + require.NotNil(t, secondFailure) + require.Equal(t, "observed_successor", firstFailure.Reason) + require.Equal(t, "preexisting_path_work", secondFailure.Reason) + } else { + require.Eventually(t, func() bool { return firstRecord.RequestedBackground() != nil }, 5*time.Second, time.Millisecond) + background := firstRecord.RequestedBackground() + require.Equal(t, uint64(2), background.NativeHandoffs) + require.Equal(t, third.Generation, background.AppliedGeneration) + require.Equal(t, "successful_pre_callback_native_handoffs", background.Coverage) + } + require.Nil(t, secondRecord.RequestedBackground()) + }) + } +} + +func TestRequestedBackgroundRejectsIncompleteOrInvalidIntegerBounds(t *testing.T) { + _, origin, window := controlledAttemptWindow(t) + window.ticket = 2 + valid := canonicalBackgroundEvidence{origin: origin, requested: 1, applied: 2, handoffs: 1, physical: window, debounce: canonicalDebounceEvidence{startedNS: 0, enteredNS: 1e6, entered: true}, phases: [8]int64{1, 1, 1, 1, 1, 1, 999999}, lastArmStartNS: 5, startNS: 1e6 + 5, endNS: 31e6 + 5} + require.True(t, valid.valid()) + for _, name := range []string{"missing", "reversed", "overflow", "incomplete_P", "wrong_winner", "retry", "gap", "debounce_disagrees", "debounce_missing"} { + t.Run(name, func(t *testing.T) { + b := valid + switch name { + case "missing": + b.origin = time.Time{} + case "reversed": + b.phases[3] = -1 + case "overflow": + b.phases[6] = math.MaxInt64 + case "incomplete_P": + b.physical.accounting.laps[patchParse] = canonicalLapEvidence{} + case "wrong_winner": + b.applied = 3 + case "retry": + b.physical.retry = 1 + case "gap": + b.endNS++ + case "debounce_disagrees": + b.debounce.enteredNS++ + case "debounce_missing": + b.debounce.entered = false + } + require.False(t, b.valid()) + }) + } +} + +func TestRequestedBackgroundCallbackAndStopFailuresInvalidateOwnership(t *testing.T) { + for _, name := range []string{"callback_ran", "stop_false", "observed_successor", "retry", "second_stop"} { + t.Run(name, func(t *testing.T) { + record := newPointPublication(t, "background-invalid") + w := &Watcher{} + b := newCanonicalBackground(record, w, "sample.go", 1, time.Now()) + require.NotNil(t, b) + switch name { + case "callback_ran": + b.callback(w, "sample.go", 1, false, time.Now()) + case "stop_false": + b.stopped(w, "sample.go", 1, false, time.Now(), time.Now(), false, true) + case "observed_successor": + b.stopped(w, "sample.go", 1, false, time.Now(), time.Now(), true, false) + case "retry": + b.invalidate("retry_or_backoff") + case "second_stop": + b.stopped(w, "sample.go", 1, false, time.Now(), time.Now(), true, true) + b.stopped(w, "sample.go", 1, true, time.Now(), time.Now(), true, true) + } + b.mu.Lock() + invalid := b.invalid + b.mu.Unlock() + require.True(t, invalid) + require.Nil(t, record.RequestedBackground()) + }) + } +} + +func TestCanonicalSchedulingEntryKeepsOuterIdentityAndNativeAbstention(t *testing.T) { + plain := context.Background() + ctx, at := canonicalSchedulingEntry(plain) + require.Equal(t, plain, ctx) + require.True(t, at.IsZero()) + require.Zero(t, testing.AllocsPerRun(100, func() { canonicalSchedulingEntry(plain) })) + original := WithPublicationRecord(plain, newPointPublication(t, "entry")) + outer, entered := canonicalSchedulingEntry(original) + require.False(t, entered.IsZero()) + inner, reused := canonicalSchedulingEntry(outer) + require.Same(t, outer, inner) + require.Equal(t, entered, reused) +} + +func TestRequestedBackgroundRejectsSupersessionBeforeAcceptance(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + // Retain the original plus both independent physical attempts; otherwise + // this exercises bounded eviction before it can inspect supersession. + recorder := NewPublicationPhaseRecorder(4, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + record := recorder.Begin("", "background-before-acceptance", PublicationSourceMCPEdit, time.Now()) + first := w.scheduleFileMutationWithRecord(path, ChangeModified, record) + native := w.scheduleFileMutation(path, ChangeModified) + reached, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(unblock) + w.mutationBeforeComplete = func(_ string, generation uint64) { + if generation == native.Generation { + close(reached) + <-release + } + } + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + awaitPublicationSignal(t, reached) + require.Nil(t, record.RequestedBackground()) + newer := w.scheduleFileMutation(path, ChangeModified) + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + unblock() + for _, ticket := range []*MutationTicket{first, native, newer} { + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.Equal(t, newer.Generation, result.AppliedGeneration) + } + require.Eventually(t, func() bool { + winner := record.CoalescedPublication() + return winner != nil && winner.Timings.Window.PatchWallMS != nil + }, 5*time.Second, time.Millisecond) + require.Equal(t, newer.Generation, record.CoalescedPublication().AppliedGeneration) + require.Nil(t, record.RequestedBackground(), "the unobserved superseding quiet/reset chain is unavailable") + _, unavailable := record.CoalescedPublicationEvidence() + require.NotNil(t, unavailable) + require.Equal(t, "serving_attempt_binding_mismatch", unavailable.Reason) +} diff --git a/internal/indexer/canonical_requested_direct_background_test.go b/internal/indexer/canonical_requested_direct_background_test.go new file mode 100644 index 000000000..437167ff1 --- /dev/null +++ b/internal/indexer/canonical_requested_direct_background_test.go @@ -0,0 +1,108 @@ +package indexer + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRequestedBackgroundOwnCallbackStartsAtObservedAdmission(t *testing.T) { + for _, preexisting := range []bool{false, true} { + name := "clean_direct" + if preexisting { + name = "stopped_unobserved_native_predecessor" + } + t.Run(name, func(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + var predecessor *MutationTicket + if preexisting { + predecessor = w.scheduleFileMutation(path, ChangeModified) + } + record := newPointPublication(t, "direct-background") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NotNil(t, ticket) + w.mu.Lock() + observation := w.mutationWaiters[path][ticket.Generation].timerObservation + require.NotNil(t, observation) + owner := observation.background + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + require.NotNil(t, owner) + owner.mu.Lock() + origin, arm := owner.origin, owner.armStart + owner.mu.Unlock() + require.False(t, origin.IsZero()) + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.Equal(t, ticket.Generation, result.AppliedGeneration) + if predecessor != nil { + require.NoError(t, awaitMutationRetryResult(t, predecessor).Err) + } + require.Eventually(t, func() bool { return record.RequestedBackground() != nil }, time.Second, time.Millisecond, "actual own requested callback/return must have its own B") + background := record.RequestedBackground() + require.False(t, arm.Before(origin)) + require.Equal(t, "direct_requested_attempt", background.Coverage) + require.Equal(t, ticket.Generation, background.RequestedGeneration) + require.Equal(t, ticket.Generation, background.AppliedGeneration) + require.Zero(t, background.NativeHandoffs, "no count inferred from predecessor generation") + if predecessor != nil { + require.Equal(t, predecessor.Generation, background.AdmissionPredecessor) + require.True(t, background.AdmissionPredecessorStopped) + } else { + require.Zero(t, background.AdmissionPredecessor) + require.False(t, background.AdmissionPredecessorStopped) + } + require.Equal(t, origin.UTC().Format(time.RFC3339Nano), background.OriginWall) + require.True(t, background.PartitionComplete) + requirePointExecution(t, record, true) + sum := 0.0 + for _, value := range background.PhasesMS { + sum += value + } + require.InDelta(t, background.WallMS, sum, 0.000001) + require.Greater(t, background.PhasesMS["requested_quiet_wait"], 0.0) + require.Nil(t, record.CoalescedPublication(), "own P must not be relabeled as a foreign winner") + snapshot := record.RequestedBackground() + snapshot.PhasesMS["requested_quiet_wait"] = 999 + require.NotEqual(t, 999.0, record.RequestedBackground().PhasesMS["requested_quiet_wait"]) + }) + } +} + +func TestRequestedBackgroundRefusesActuallyStartedPredecessorTimer(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + native := w.scheduleFileMutation(path, ChangeModified) + entered, resume := make(chan struct{}), make(chan struct{}) + var once sync.Once + release := func() { once.Do(func() { close(resume) }) } + t.Cleanup(release) + // Replace only this test-owned, successfully stopped quiet timer with a real + // fired callback. It remains in the same actual watcher slot until admission; + // Stop must return false without an extra production hook or fake bool. + w.mu.Lock() + require.True(t, w.pending[path].Stop()) + w.asyncWork.Done() + w.asyncWork.Add(1) + w.pending[path] = time.AfterFunc(0, func() { defer w.asyncWork.Done(); close(entered); <-resume }) + w.mu.Unlock() + awaitPublicationSignal(t, entered) + record := newPointPublication(t, "failed-predecessor-stop") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NotNil(t, ticket) + release() + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + require.NoError(t, awaitMutationRetryResult(t, native).Err) + require.Nil(t, record.RequestedBackground(), "already-started predecessor cannot supply observed replacement coverage") + _, unavailable := record.CoalescedPublicationEvidence() + require.NotNil(t, unavailable) + require.Equal(t, "preexisting_timer_stop_not_successful", unavailable.Reason) +} diff --git a/internal/indexer/cold_manifest_census_test.go b/internal/indexer/cold_manifest_census_test.go index 4ba7f20c7..deb84a138 100644 --- a/internal/indexer/cold_manifest_census_test.go +++ b/internal/indexer/cold_manifest_census_test.go @@ -777,7 +777,7 @@ func TestColdScopedDeferredTailPreservesUnselectedAttempt(t *testing.T) { // This is the actual cold orchestration entry whose sorted lane set only // contains "selected". The previously registered "overlap" repository's // lane is not part of this batch, despite its pending deferred contracts. - results, err := mi.indexMultiRepo([]config.RepoEntry{{Path: selectedRoot, Name: "selected"}}) + results, err := mi.indexFixtureRepos([]config.RepoEntry{{Path: selectedRoot, Name: "selected"}}) require.NoError(t, err) require.Contains(t, results, "selected") assert.Same(t, registry, unselected.pendingContractReg, diff --git a/internal/indexer/contract_convergence_test.go b/internal/indexer/contract_convergence_test.go index 2878294d7..ce2382ee9 100644 --- a/internal/indexer/contract_convergence_test.go +++ b/internal/indexer/contract_convergence_test.go @@ -3375,7 +3375,7 @@ func TestContractWatcherPatchOvertakenAfterItsWritesEndsItsWindow(t *testing.T) require.NoError(t, err) path := filepath.Join(idx.rootPath, "consts.go") const generation = 7 - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, p) // A newer save of the file is scheduled while the patch ran. w.mu.Lock() @@ -3419,7 +3419,7 @@ func TestContractCaptureAdmitsAnUnacceptedRowOnTheRuleAJobProvesItsReadsBy(t *te require.NoError(t, err) path := filepath.Join(idx.rootPath, "consts.go") const generation = 7 - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, p) w.mu.Lock() w.pendingGeneration[p] = generation + 1 @@ -4736,7 +4736,7 @@ func TestContractWatcherPatchThatFailsKeepsItsWindowOpen(t *testing.T) { require.NoError(t, err) path := filepath.Join(idx.rootPath, "consts.go") const generation = 7 - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { if _, err := idx.incrementalPointWatcherPath(idx.rootPath, p); err != nil { return nil, err } diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index e3406b473..22bff4e4e 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -105,7 +105,6 @@ var editDeltaAbsentCapabilities = map[string]string{ "graph.ContractOwnerScalarInvalidator": "optional companion of ContractOwnerReplacer; the fallback re-derives the scalars through ordinary node writes", "graph.ContractOwnerScalarLiveness": "optional companion of ContractOwnerReplacer; absent, the fallback does not assume liveness and checks it through reads", "graph.ContractStateStore": "per-repository ledger read (index state, file metadata, mtimes, failures, receipts): absent, the engine treats nothing as recorded and takes its conservative path; a delta forces its explicit paths", - "graph.CoordinatedBulkLoader": "whole-repository or administrative operation (full index, repository reset, untrack or cleanup, warm start, vector corpus, bulk load windows, WAL maintenance); not on a file delta's per-save path", "graph.CoverageEnrichmentWriter": "node-keyed enrichment side tables (blame, churn, coverage, releases): their rows for an evicted node are the layer below's, hidden with the node; a delta writes none", "graph.CrossRepoFlagMarker": "cross-repository edge flagging; single-repository delta", "graph.CurrentGenerationRepoEvicter": "whole-repository or administrative operation (full index, repository reset, untrack or cleanup, warm start, vector corpus, bulk load windows, WAL maintenance); not on a file delta's per-save path", diff --git a/internal/indexer/edit_derived_timings_test.go b/internal/indexer/edit_derived_timings_test.go new file mode 100644 index 000000000..a5f6b0639 --- /dev/null +++ b/internal/indexer/edit_derived_timings_test.go @@ -0,0 +1,91 @@ +package indexer + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func derivedTimingSnapshot() PublicationPhaseSnapshot { + return PublicationPhaseSnapshot{Clock: "monotonic", Source: PublicationSourceMCPEdit, Terminal: true, + Phases: []PublicationPhaseOffset{ + {Phase: PublicationReceived}, + {Phase: PublicationDiskWriteStarted, OffsetNS: 1e6}, + {Phase: PublicationReceiptCommitted, OffsetNS: 2e6}, + {Phase: PublicationTicketEnqueued, OffsetNS: 3e6}, + {Phase: PublicationPatchExecutionStarted, OffsetNS: 4e6}, + {Phase: PublicationPatchDerivedStarted, OffsetNS: 5e6}, + {Phase: PublicationPatchDerivedFinished, OffsetNS: 7e6}, + {Phase: PublicationPatchExecutionFinished, OffsetNS: 8e6}, + {Phase: PublicationTicketCompleted, OffsetNS: 10e6}, + }} +} + +func TestEditTimingsDerivedSubtractsOnlyContainedDisjointTail(t *testing.T) { + snapshot := derivedTimingSnapshot() + timings := snapshot.EditTimings() + require.Equal(t, "measured", timings.States["derived"]) + require.Equal(t, 2.0, *timings.Milliseconds["derived"]) + require.Equal(t, 6.0, *timings.Window.PublicationUnattributedWallMS, "10ms wall minus disjoint disk/publish/derived; do not add outer execution") + require.Nil(t, timings.Window.PatchWallMS) + require.Nil(t, timings.Window.AcknowledgementWallMS) + for _, phase := range []string{"debounce", "lane_wait", "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes"} { + require.Nil(t, timings.Milliseconds[phase], phase) + require.Equal(t, "not_observed", timings.States[phase], phase) + } + *timings.Milliseconds["derived"] = 99 + require.Equal(t, 2.0, *snapshot.EditTimings().Milliseconds["derived"], "projections own their values") +} + +func TestEditTimingsDerivedRequiresItsActualOwnershipBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + phase PublicationPhase + offset int64 + remove bool + omit PublicationPhase + live bool + state string + measured *float64 + }{ + {name: "observed_zero", phase: PublicationPatchDerivedFinished, offset: 5e6, state: "measured", measured: new(float64)}, + {name: "reversed_pair", phase: PublicationPatchDerivedFinished, offset: 4e6, state: "invalid_order"}, + {name: "negative_start", phase: PublicationPatchDerivedStarted, offset: -1, state: "invalid_order"}, + {name: "before_executor", phase: PublicationPatchDerivedStarted, offset: 3e6, state: "invalid_order"}, + {name: "after_executor", phase: PublicationPatchDerivedFinished, offset: 9e6, state: "invalid_order"}, + {name: "executor_before_admission", phase: PublicationTicketEnqueued, offset: 6e6, state: "invalid_order"}, + {name: "executor_after_terminal", phase: PublicationPatchExecutionFinished, offset: 11e6, state: "invalid_order"}, + {name: "missing_executor", phase: PublicationPatchExecutionStarted, remove: true, state: "not_observed"}, + {name: "missing_admission", phase: PublicationTicketEnqueued, remove: true, state: "not_observed"}, + {name: "missing_admission_with_known_reversal", phase: PublicationPatchDerivedStarted, offset: 3e6, omit: PublicationTicketEnqueued, state: "invalid_order"}, + {name: "pending_pair", phase: PublicationPatchDerivedFinished, remove: true, live: true, state: "pending"}, + {name: "terminal_missing_end", phase: PublicationPatchDerivedFinished, remove: true, state: "not_observed"}, + } { + t.Run(tc.name, func(t *testing.T) { + snapshot := derivedTimingSnapshot() + phases := snapshot.Phases[:0] + for _, phase := range snapshot.Phases { + if phase.Phase == tc.omit { + continue + } + if tc.live && phase.Phase == PublicationTicketCompleted { + continue + } + if phase.Phase == tc.phase { + if tc.remove { + continue + } + phase.OffsetNS = tc.offset + } + phases = append(phases, phase) + } + snapshot.Phases, snapshot.Terminal = phases, !tc.live + timings := snapshot.EditTimings() + require.Equal(t, tc.state, timings.States["derived"]) + require.Equal(t, tc.measured, timings.Milliseconds["derived"], "missing or contradictory evidence must not become measured zero") + if tc.state == "invalid_order" || tc.live { + require.Nil(t, timings.Window.PublicationUnattributedWallMS) + } + }) + } +} diff --git a/internal/indexer/edit_phase_timings.go b/internal/indexer/edit_phase_timings.go new file mode 100644 index 000000000..df3d15387 --- /dev/null +++ b/internal/indexer/edit_phase_timings.go @@ -0,0 +1,182 @@ +package indexer + +// EditPhaseTimings projects duration evidence from a single immutable +// publication snapshot. Missing measurements are null, never measured zero. +// This is deliberately partial: publication timestamps do not yet delimit +// dispatcher admission, the response/ack, or the complete patch-lap split. +type EditPhaseTimings struct { + Milliseconds map[string]*float64 `json:"phases_ms"` + States map[string]string `json:"phase_states"` + Window EditTimingWindow `json:"phase_timing"` +} + +// EditTimingWindow distinguishes the receipt's publication interval from the +// acknowledgement and physical patch intervals. Those lifetimes can overlap; +// adding their walls would double-count work. The publication residual includes +// uninstrumented transitions and waits inside that interval; it does not +// identify the acknowledgement's default reindex wait. It is not a phase +// measurement or a claim of complete phase accounting. +type EditTimingWindow struct { + Clock string `json:"clock"` + Coverage string `json:"coverage"` + AcknowledgementWallMS *float64 `json:"ack_wall_ms"` + PatchWallMS *float64 `json:"patch_wall_ms"` + PatchAccounting *EditPatchAccounting `json:"patch_accounting,omitempty"` + PublicationWallMS *float64 `json:"publication_wall_ms"` + PublicationUnattributedWallMS *float64 `json:"publication_unattributed_ms"` +} + +var editAckPhaseNames = [...]string{ + "dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait", "edit_control", "caller_result", +} + +var editPatchPhaseNames = [...]string{ + "debounce", "cohort_admission", "batch_admission", "lane_wait", "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived", +} + +// EditTimings uses nanosecond offsets from the same-process monotonic clock, +// rather than the rounded milliseconds or wall-clock strings in the snapshot. +// It does not interpret cycle_started→admitted as pure lane wait, nor split +// structural apply into overlapping nested spans. Those require explicit +// instrumentation at their actual boundaries. +func (s PublicationPhaseSnapshot) EditTimings() EditPhaseTimings { + out := EditPhaseTimings{ + Milliseconds: make(map[string]*float64, len(editAckPhaseNames)+len(editPatchPhaseNames)), + States: make(map[string]string, len(editAckPhaseNames)+len(editPatchPhaseNames)), + Window: EditTimingWindow{Clock: "unavailable", Coverage: "partial"}, + } + for _, name := range editAckPhaseNames { + out.Milliseconds[name] = nil + out.States[name] = "not_observed" + if s.Source == PublicationSourceFreshRequest || s.Source == PublicationSourceBackgroundCompaction { + out.States[name] = "not_applicable" + } + } + for _, name := range editPatchPhaseNames { + out.Milliseconds[name] = nil + out.States[name] = "not_observed" + } + if s.Clock != "monotonic" { + return out + } + out.Window.Clock = s.Clock + if s.attemptWindow != nil { + // P is independent of receipt/ACK lifetime and may finish later. + ms := float64(s.attemptWindow.endNS-s.attemptWindow.startNS) / 1e6 + out.Window.PatchWallMS = &ms + out.Window.PatchAccounting = s.attemptWindow.projectAccounting() + } + offsets := make(map[PublicationPhase]int64, len(s.Phases)) + for _, phase := range s.Phases { + offsets[phase.Phase] = phase.OffsetNS + } + if s.Source == PublicationSourceMCPEdit { + out.Milliseconds["disk_write"], out.States["disk_write"] = editPhaseInterval( + offsets, PublicationDiskWriteStarted, PublicationReceiptCommitted, s.Terminal) + out.Milliseconds["publish"], out.States["publish"] = editPhaseInterval( + offsets, PublicationReceiptCommitted, PublicationTicketEnqueued, s.Terminal) + } + out.Milliseconds["derived"], out.States["derived"] = editDerivedInterval(offsets, s.Terminal) + if s.patchLaps != nil { + s.patchLaps.project(offsets, s.Terminal, &out) + } + if s.admissions != nil { + s.admissions.project(offsets, s.Terminal, s.laneWait, &out) + } + if s.debounce != nil { + s.debounce.project(offsets, s.Terminal, &out) + } + if s.laneWait != nil { + s.laneWait.project(offsets, s.Terminal, &out) + } + terminal := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + terminal = PublicationTicketFailed + } + if !s.Terminal { + return out + } + wall, state := editPhaseInterval(offsets, PublicationReceived, terminal, true) + if state != "measured" { + return out + } + out.Window.PublicationWallMS = wall + // Only subtract disjoint, wholly contained measured intervals. An invalid + // order leaves the residual unknown; never clamp it to a plausible zero. + var measured float64 + for _, name := range []string{"disk_write", "publish", "debounce", "cohort_admission", "batch_admission", "lane_wait", "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived"} { + if out.States[name] == "invalid_order" || out.States[name] == "pending" { + return out + } + if ms := out.Milliseconds[name]; ms != nil { + measured += *ms + } + } + if enqueued, ok := offsets[PublicationTicketEnqueued]; ok && enqueued > offsets[terminal] { + return out + } + if committed, ok := offsets[PublicationReceiptCommitted]; ok && committed > offsets[terminal] { + return out + } + if measured <= *wall { + residual := *wall - measured + out.Window.PublicationUnattributedWallMS = &residual + } + return out +} + +// Only an observed derived call inside the accepted canonical executor can +// populate this lap. A pair without its ownership boundaries remains unknown; +// contradictory order is refused, not clamped or charged to an earlier receipt. +func editDerivedInterval(offsets map[PublicationPhase]int64, terminal bool) (*float64, string) { + ms, state := editPhaseInterval(offsets, PublicationPatchDerivedStarted, PublicationPatchDerivedFinished, terminal) + if state != "measured" { + return ms, state + } + var previous int64 + missing := false + for _, phase := range []PublicationPhase{PublicationTicketEnqueued, PublicationPatchExecutionStarted, + PublicationPatchDerivedStarted, PublicationPatchDerivedFinished, PublicationPatchExecutionFinished} { + offset, ok := offsets[phase] + if !ok { + missing = true + continue + } + if offset < previous { + return nil, "invalid_order" + } + previous = offset + } + if terminal { + end := PublicationTicketCompleted + if _, failed := offsets[PublicationTicketFailed]; failed { + end = PublicationTicketFailed + } + if offset, ok := offsets[end]; ok && previous > offset { + return nil, "invalid_order" + } + } + if missing { + return nil, "not_observed" + } + return ms, state +} + +func editPhaseInterval(offsets map[PublicationPhase]int64, start, end PublicationPhase, terminal bool) (*float64, string) { + begin, hasStart := offsets[start] + finish, hasEnd := offsets[end] + if hasStart && begin < 0 || hasEnd && finish < 0 || hasStart && hasEnd && finish < begin { + return nil, "invalid_order" + } + if !hasStart { + return nil, "not_observed" + } + if !hasEnd { + if !terminal { + return nil, "pending" + } + return nil, "not_observed" + } + ms := float64(finish-begin) / 1e6 + return &ms, "measured" +} diff --git a/internal/indexer/edit_phase_timings_test.go b/internal/indexer/edit_phase_timings_test.go new file mode 100644 index 000000000..2ddd59eb7 --- /dev/null +++ b/internal/indexer/edit_phase_timings_test.go @@ -0,0 +1,119 @@ +package indexer + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func requireEditTimingKeys(t *testing.T, timings EditPhaseTimings) { + t.Helper() + // Keep this census independent of the production arrays: adding or dropping + // a required lap must be an explicit wire-contract decision. + want := []string{"dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait", "edit_control", "caller_result", + "debounce", "cohort_admission", "batch_admission", "lane_wait", "gate_wait", "patch_parse", "apply", "search_lap", "resolve", "mtimes", "derived"} + require.Len(t, timings.Milliseconds, len(want)) + require.Len(t, timings.States, len(want)) + for _, key := range want { + require.Contains(t, timings.Milliseconds, key) + require.NotEmpty(t, timings.States[key], key) + } +} + +func TestEditTimingsOnlyProjectProvenDisjointIntervals(t *testing.T) { + origin := time.Now() + record := NewPublicationPhaseRecorder(1, 1).Begin("checkout", "edit", PublicationSourceMCPEdit, origin) + record.MarkAt(PublicationDiskWriteStarted, origin.Add(time.Millisecond)) + record.MarkAt(PublicationReceiptCommitted, origin.Add(1250500*time.Nanosecond)) + record.MarkAt(PublicationTicketEnqueued, origin.Add(3*time.Millisecond)) + record.MarkAt(PublicationCycleStarted, origin.Add(4*time.Millisecond)) + record.MarkAt(PublicationAdmitted, origin.Add(6*time.Millisecond)) + record.MarkAt(PublicationTicketCompleted, origin.Add(10*time.Millisecond)) + snapshot := record.Snapshot() + before, err := json.Marshal(snapshot) + require.NoError(t, err) + timings := snapshot.EditTimings() + requireEditTimingKeys(t, timings) + require.Equal(t, "monotonic", timings.Window.Clock) + require.Equal(t, "partial", timings.Window.Coverage) + require.Equal(t, "measured", timings.States["disk_write"]) + require.Equal(t, 0.2505, *timings.Milliseconds["disk_write"], "derive from nanoseconds, not rounded offset_ms") + require.Equal(t, 1.7495, *timings.Milliseconds["publish"]) + require.Nil(t, timings.Milliseconds["lane_wait"], "cycle start→admitted includes more than pure lane wait") + require.Equal(t, "not_observed", timings.States["lane_wait"]) + require.Nil(t, timings.Window.AcknowledgementWallMS, "publication completion is not the response time") + require.Nil(t, timings.Window.PatchWallMS, "the receipt does not delimit a physical patch") + require.Equal(t, 10.0, *timings.Window.PublicationWallMS) + require.Equal(t, 8.0, *timings.Window.PublicationUnattributedWallMS) + after, err := json.Marshal(snapshot) + require.NoError(t, err) + require.Equal(t, string(before), string(after), "duration projection must not alter the legacy timeline") + *timings.Milliseconds["disk_write"] = 99 + require.Equal(t, 0.2505, *snapshot.EditTimings().Milliseconds["disk_write"], "callers own their projection") +} + +func TestEditTimingsUnknownPendingZeroAndTerminalMissingAreDistinct(t *testing.T) { + origin := time.Now() + record := NewPublicationPhaseRecorder(1, 1).Begin("checkout", "edit", PublicationSourceMCPEdit, origin) + empty := record.Snapshot().EditTimings() + requireEditTimingKeys(t, empty) + require.Nil(t, empty.Milliseconds["disk_write"]) + require.Equal(t, "not_observed", empty.States["disk_write"]) + record.MarkAt(PublicationDiskWriteStarted, origin.Add(time.Millisecond)) + pending := record.Snapshot().EditTimings() + require.Nil(t, pending.Milliseconds["disk_write"]) + require.Equal(t, "pending", pending.States["disk_write"]) + require.Nil(t, pending.Window.PublicationWallMS) + record.MarkAt(PublicationReceiptCommitted, origin.Add(time.Millisecond)) + committed := record.Snapshot().EditTimings() + require.NotNil(t, committed.Milliseconds["disk_write"]) + require.Zero(t, *committed.Milliseconds["disk_write"], "two observed simultaneous boundaries are measured zero") + require.Equal(t, "measured", committed.States["disk_write"]) + require.Equal(t, "pending", committed.States["publish"]) + record.MarkAt(PublicationTicketFailed, origin.Add(2*time.Millisecond)) + failed := record.Snapshot().EditTimings() + require.Equal(t, "not_observed", failed.States["publish"], "terminal failure cannot imply successful admission") + require.Nil(t, failed.Milliseconds["publish"]) + encoded, err := json.Marshal(failed) + require.NoError(t, err) + require.Contains(t, string(encoded), `"disk_write":0`) + require.Contains(t, string(encoded), `"publish":null`) +} + +func TestEditTimingsRejectInvalidOrderingWithoutClamping(t *testing.T) { + for _, terminal := range []bool{false, true} { + snapshot := PublicationPhaseSnapshot{Clock: "monotonic", Source: PublicationSourceMCPEdit, Terminal: terminal, + Phases: []PublicationPhaseOffset{ + {Phase: PublicationReceived}, + {Phase: PublicationDiskWriteStarted, OffsetNS: 3e6}, + {Phase: PublicationReceiptCommitted, OffsetNS: 2e6}, + {Phase: PublicationTicketEnqueued, OffsetNS: 1e6}, + {Phase: PublicationTicketFailed, OffsetNS: 4e6}, + }} + timings := snapshot.EditTimings() + for _, name := range []string{"disk_write", "publish"} { + require.Nil(t, timings.Milliseconds[name]) + require.Equal(t, "invalid_order", timings.States[name]) + } + require.Nil(t, timings.Window.PublicationUnattributedWallMS) + } +} + +func TestEditTimingsNoTimelineAndNonEditSources(t *testing.T) { + missing := (PublicationPhaseSnapshot{}).EditTimings() + requireEditTimingKeys(t, missing) + require.Equal(t, "unavailable", missing.Window.Clock) + for name, ms := range missing.Milliseconds { + require.Nil(t, ms, name) + require.Equal(t, "not_observed", missing.States[name]) + } + for _, source := range []string{PublicationSourceFreshRequest, PublicationSourceBackgroundCompaction} { + timings := (PublicationPhaseSnapshot{Source: source, Clock: "monotonic"}).EditTimings() + for _, name := range []string{"dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait"} { + require.Nil(t, timings.Milliseconds[name]) + require.Equal(t, "not_applicable", timings.States[name]) + } + } +} diff --git a/internal/indexer/file_delta.go b/internal/indexer/file_delta.go index 59d38e447..a4cac5ec3 100644 --- a/internal/indexer/file_delta.go +++ b/internal/indexer/file_delta.go @@ -50,9 +50,10 @@ type preparedExtraction struct { // rawSHA256 is gitstate.BlobSHA256 of the bytes read, before the source // transforms; set only for a build that proves its reads // (Indexer.contentProof). - rawSHA256 string - parseLease *parseAdmissionLease - releaseOnce sync.Once + rawSHA256 string + readRepairBlobSHA string + parseLease *parseAdmissionLease + releaseOnce sync.Once } // fileDeltaProbe exposes phase timings and the three delta boundaries used by @@ -69,7 +70,8 @@ type fileDeltaProbe struct { fingerprintTime time.Duration readVersion fileReadVersion // rawSHA256 is preparedExtraction.rawSHA256 of the probe's read. - rawSHA256 string + rawSHA256 string + readRepairBlobSHA string } // prepareFileDelta parses the current file once and caches that exact @@ -89,6 +91,10 @@ func (idx *Indexer) prepareFileDelta(filePath string) (fileDeltaProbe, bool) { // chunk still waits, guaranteeing progress even when it is larger than the // entire budget (its weight is clamped to capacity). func (idx *Indexer) prepareFileDeltaWithAdmission(filePath string, tryOnly bool) (fileDeltaProbe, bool, bool) { + return idx.prepareFileDeltaWithAdmissionObserved(filePath, tryOnly, nil) +} + +func (idx *Indexer) prepareFileDeltaWithAdmissionObserved(filePath string, tryOnly bool, repair *ReadRepairObservation) (fileDeltaProbe, bool, bool) { var probe fileDeltaProbe absPath, err := filepath.Abs(filePath) if err != nil { @@ -150,6 +156,9 @@ func (idx *Indexer) prepareFileDeltaWithAdmission(filePath string, tryOnly bool) if _, skip := idx.newContentAdmissionGate().skip(lang, int64(len(src))); skip { return probe, false, false } + if repair != nil && relPath == repair.path { + probe.readRepairBlobSHA = readRepairBlobSHA(src) + } src = idx.transforms.run(relPath, src) var pool *crashpool.Pool @@ -200,14 +209,15 @@ func (idx *Indexer) prepareFileDeltaWithAdmission(filePath string, tryOnly bool) // reach once the map slot is overwritten. displaced := idx.prepared[absPath] idx.prepared[absPath] = &preparedExtraction{ - absPath: absPath, - relPath: relPath, - lang: lang, - src: append([]byte(nil), src...), - result: result, - readVersion: readVersion, - rawSHA256: probe.rawSHA256, - parseLease: parseLease, + absPath: absPath, + relPath: relPath, + lang: lang, + src: append([]byte(nil), src...), + result: result, + readVersion: readVersion, + rawSHA256: probe.rawSHA256, + readRepairBlobSHA: probe.readRepairBlobSHA, + parseLease: parseLease, } idx.preparedMu.Unlock() // Released outside the lock: release closes the tree through cgo and diff --git a/internal/indexer/file_meta.go b/internal/indexer/file_meta.go index dd646109a..d534bca61 100644 --- a/internal/indexer/file_meta.go +++ b/internal/indexer/file_meta.go @@ -195,6 +195,8 @@ func persistFileMetaRows(target graph.Store, repoPrefix string, rows []graph.Fil const reparsePendingEnrichmentBatchSize = 256 type reparsePendingEnrichmentBatch struct { + publication *PublicationStamps // optional local canonical attempt; never a live record + readRepair *ReadRepairObservation // npmConsumerFiles are unchanged importers parked by this mutation. npmConsumerFiles []string byFile map[string]bool @@ -273,3 +275,10 @@ func (idx *Indexer) setReparsePendingEnrichments(pendingByFile map[string]bool) idx.graph.AddBatch(updates, nil) } } + +func (b *reparsePendingEnrichmentBatch) publicationStamps() *PublicationStamps { + if b == nil { + return nil + } + return b.publication +} diff --git a/internal/indexer/incremental_batch.go b/internal/indexer/incremental_batch.go index 2779b774d..788f93c4a 100644 --- a/internal/indexer/incremental_batch.go +++ b/internal/indexer/incremental_batch.go @@ -32,6 +32,7 @@ type incrementalBatchStage struct { mtimeKey string readVersion fileReadVersion rawSHA256 string + readRepairBlobSHA string relPath string graphPath string legacyIDs []string @@ -68,7 +69,11 @@ type fileReadReceipt struct { readVersion fileReadVersion // rawSHA256 is the content identity of the bytes read, set only for a // build that proves its reads (Indexer.contentProof). - rawSHA256 string + rawSHA256 string + readRepairBlobSHA string + // Only a committed stage can supply publisher proof. Inert validation + // receipts deliberately leave this false even for identical fresh bytes. + readRepairApplied bool } type incrementalFallback struct { @@ -158,7 +163,9 @@ func (idx *Indexer) reindexIncrementalFilesBatched( deleteTiming := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "graph_delete", zap.Int("deleted_files", len(deletedFiles))) defer deleteTiming.abort() + deletion := markerBatch.publicationStamps().beginPatchLap(patchApply) idx.evictDeletedFilesBatched(deletedFiles, &invalidation) + deletion.end() deleteTiming.complete(nil) passPlan, reparsed, failed, versionChanged := idx.reindexIncrementalStalePass( @@ -265,6 +272,7 @@ func (idx *Indexer) reindexIncrementalChunk( parseTiming := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "chunk_parse", zap.Int("candidate_files", len(files)), zap.Bool("includes_admission_wait", true)) defer parseTiming.abort() + parse := markerBatch.publicationStamps().beginPatchLap(patchParse) stages := make([]*incrementalBatchStage, 0, len(files)) // Each staged result carries the tree-sitter tree its extraction @@ -316,7 +324,11 @@ func (idx *Indexer) reindexIncrementalChunk( // the partial chunk and retries this file in the next chunk; this // prevents repositories from each holding a partial batch while all // wait for capacity owned by the others. - probe, probeOK, admissionBusy := idx.prepareFileDeltaWithAdmission(filePath, len(stages) > 0) + var repair *ReadRepairObservation + if markerBatch != nil { + repair = markerBatch.readRepair + } + probe, probeOK, admissionBusy := idx.prepareFileDeltaWithAdmissionObserved(filePath, len(stages) > 0, repair) if admissionBusy { break } @@ -359,7 +371,8 @@ func (idx *Indexer) reindexIncrementalChunk( idx.discardPreparedExtraction(filePath) receipts = append(receipts, fileReadReceipt{ absPath: filePath, mtimeKey: idx.relKey(filePath), readVersion: probe.readVersion, - rawSHA256: probe.rawSHA256, + rawSHA256: probe.rawSHA256, + readRepairBlobSHA: probe.readRepairBlobSHA, }) plan.InertFiles++ continue @@ -398,9 +411,10 @@ func (idx *Indexer) reindexIncrementalChunk( idx.applyRepoPrefix(prepared.result.Nodes, prepared.result.Edges) stage := &incrementalBatchStage{ absPath: filePath, mtimeKey: idx.relKey(filePath), - readVersion: prepared.readVersion, - rawSHA256: prepared.rawSHA256, - relPath: prepared.relPath, graphPath: graphPath, + readVersion: prepared.readVersion, + rawSHA256: prepared.rawSHA256, + readRepairBlobSHA: prepared.readRepairBlobSHA, + relPath: prepared.relPath, graphPath: graphPath, legacyIDs: legacyIDs, legacyPath: legacyPath, src: prepared.src, result: prepared.result, prepared: prepared, priorNodes: priorNodes, storedGraph: storedGraph, storedDerived: storedDerived, probe: probe, @@ -424,6 +438,7 @@ func (idx *Indexer) reindexIncrementalChunk( } } + parse.end() parseTiming.complete(nil, zap.Int("consumed_files", consumed), zap.Int("staged_files", len(stages)), zap.Int("inert_files", plan.InertFiles), zap.Int("read_failed_files", len(readFailed)), zap.Int("fallback_files", len(fallbacks)), zap.Int("nodes", nodeCount), zap.Int("edges", edgeCount), @@ -440,7 +455,9 @@ func (idx *Indexer) reindexIncrementalChunk( for _, stage := range stages { receipts = append(receipts, fileReadReceipt{ absPath: stage.absPath, mtimeKey: stage.mtimeKey, readVersion: stage.readVersion, - rawSHA256: stage.rawSHA256, + rawSHA256: stage.rawSHA256, + readRepairBlobSHA: stage.readRepairBlobSHA, + readRepairApplied: true, }) } // Commit no longer reads the staged source, result, or tree. Return @@ -453,12 +470,21 @@ func (idx *Indexer) reindexIncrementalChunk( receiptTiming := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "chunk_receipts", zap.Int("receipts", len(receipts))) defer receiptTiming.abort() + mtimes := markerBatch.publicationStamps().beginPatchLap(patchMtimes) freshPaths, stalePaths := idx.recordFileReadVersionsBatched(receipts) + mtimes.end() receiptTiming.complete(nil, zap.Int("fresh_files", len(freshPaths)), zap.Int("stale_files", len(stalePaths))) freshSet := make(map[string]struct{}, len(freshPaths)) for _, filePath := range freshPaths { freshSet[filePath] = struct{}{} } + if markerBatch != nil && markerBatch.readRepair != nil { + for _, receipt := range receipts { + if _, fresh := freshSet[receipt.absPath]; fresh && receipt.readRepairBlobSHA != "" { + markerBatch.readRepair.acceptFreshBlob(idx.relKey(receipt.absPath), receipt.readRepairBlobSHA, receipt.readRepairApplied) + } + } + } var reparsed []string failed := append(readFailed, stalePaths...) @@ -470,6 +496,7 @@ func (idx *Indexer) reindexIncrementalChunk( } var fallbackTiming *reconcilePhaseTiming if len(fallbacks) > 0 { + markerBatch.publicationStamps().incompletePatchLaps() // Legacy fallback combines extraction and graph mutation; do not label // this interval as parse-only or silently charge it to graph apply. fallbackTiming = idx.startReconcilePhase(idx.logger, idx.repoPrefix, "chunk_fallback_parse_apply", @@ -634,6 +661,7 @@ func (idx *Indexer) commitIncrementalStages( timing := idx.startReconcilePhase(idx.logger, idx.repoPrefix, "chunk_graph_apply", zap.Int("staged_files", len(stages))) defer timing.abort() + apply := markerBatch.publicationStamps().beginPatchLap(patchApply) var plan DerivedInvalidationPlan if journal := idx.contractCoreInputs; journal != nil { paths := make([]string, 0, len(stages)) @@ -835,8 +863,12 @@ func (idx *Indexer) commitIncrementalStages( idx.applyLap("metadata") idx.persistIncrementalSidecars(stages) idx.applyLap("sidecars") + apply.end() + search := markerBatch.publicationStamps().beginPatchLap(patchSearch) idx.updateIncrementalSearch(stages) idx.upsertIncrementalFTS(stages) + search.end() + apply = markerBatch.publicationStamps().beginPatchLap(patchApply) idx.applyLap("search") for _, stage := range stages { @@ -856,6 +888,7 @@ func (idx *Indexer) commitIncrementalStages( stage.graphPath, stage.priorNodes, stage.result.Nodes, )) } + apply.end() timing.complete(nil, zap.Int("structural_files", len(structural)), zap.Int("metadata_files", len(metadata))) return plan } diff --git a/internal/indexer/incremental_watcher_batch.go b/internal/indexer/incremental_watcher_batch.go index 86f241bbf..1d7e2f425 100644 --- a/internal/indexer/incremental_watcher_batch.go +++ b/internal/indexer/incremental_watcher_batch.go @@ -37,7 +37,7 @@ func (idx *Indexer) incrementalReindexPathsWithReceiptMode( paths []string, mode incrementalPathMode, ) (result *IndexResult, receipt *graph.MutationReceipt, batch *reparsePendingEnrichmentBatch, err error) { - batch = &reparsePendingEnrichmentBatch{deferResolverCatchup: true} + batch = &reparsePendingEnrichmentBatch{deferResolverCatchup: true, publication: mode.publication, readRepair: mode.readRepair} receiptStore, _ := idx.graph.(graph.MutationReceiptStore) if receiptStore == nil { result, err = idx.incrementalReindexPathsMode(root, paths, mode, batch) @@ -200,8 +200,13 @@ func (idx *Indexer) runIncrementalResolutionCatchup( return nil } idx.observeIncrementalCatchup("resolve", files) + resolve := batch.publicationStamps().beginPatchLap(patchResolve) idx.resolveWithDeferredEvidence(batch, func() { resolveFiles(files) }) + resolve.end() + // Measure this concrete follow-up block, not a gap inferred between two + // resolver laps. The attempt's existing ordered collector rejects overlap. + bookkeeping := batch.publicationStamps().beginPatchLap(patchBookkeeping) idx.observeIncrementalCatchup("dataflow", files) idx.materializeDataflowParamsForFiles(files) @@ -214,6 +219,7 @@ func (idx *Indexer) runIncrementalResolutionCatchup( factFiles := appendUniqueSorted(files, affected.files...) idx.observeIncrementalCatchup("ref_facts", factFiles) idx.persistRefFactsForFiles(factFiles) + bookkeeping.end() return factFiles } @@ -354,9 +360,14 @@ func (idx *Indexer) incrementalEvictWatcherPath(path string) (nodesRemoved, edge // complete resolver/semantic/derived tail without re-entering a public // coordinated API. func (idx *Indexer) incrementalPointWatcherPath(root, path string) (*IndexResult, error) { + return idx.incrementalPointWatcherPathObserved(root, path, nil) +} + +func (idx *Indexer) incrementalPointWatcherPathObserved(root, path string, stamps *PublicationStamps) (*IndexResult, error) { return idx.incrementalWatcherPaths(root, []string{path}, incrementalPathMode{ detectDeletions: true, forceExplicitFiles: true, + publication: stamps, }) } @@ -364,7 +375,9 @@ func (idx *Indexer) incrementalWatcherPaths(root string, paths []string, mode in // The coordinator owns the repository lane before invoking this executor. // Take the reachability topology gate second and retain it through resolver // and derived catch-up so readers observe the complete old or new graph. + gate := mode.publication.beginPatchLap(patchGateWait) finishTopologyMutation := reach.BeginTopologyMutation(idx.graph) + gate.end() topologyChanged := true defer func() { finishTopologyMutation(topologyChanged) }() @@ -384,10 +397,14 @@ func (idx *Indexer) incrementalWatcherPaths(root string, paths []string, mode in } idx.resolver.ResolveFilesAndIncoming(frontier) }) + names := mode.publication.beginPatchLap(patchResolve) idx.resolveReceiptNamePendings(receipt) + names.end() } else if needed && !exact { idx.observeIncrementalCatchup("resolve", nil) + resolve := mode.publication.beginPatchLap(patchResolve) idx.ResolveAll() + resolve.end() } if mode.forceExplicitFiles || !idx.deferGlobalPasses.Load() { if mode.exactPointSemantic { @@ -396,7 +413,9 @@ func (idx *Indexer) incrementalWatcherPaths(root string, paths []string, mode in idx.runIncrementalWatcherSemantic(result.DerivedInvalidation.Files) } idx.observeIncrementalCatchup("derived", result.DerivedInvalidation.Files) + derived := mode.publication.beginDerivedPatchLap() idx.runStandaloneIncrementalDerivedPassesWithPrior(result.DerivedInvalidation, result.capabilityPrior) + derived.endDerived() } topologyChanged = incrementalTopologyChanged(result) return result, nil @@ -554,19 +573,27 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( }) crossRepoFiles = appendUniqueSorted(crossRepoFiles, resolvedFiles...) if receipt != nil && receipt.Complete { + names := batch.publicationStamps().beginPatchLap(patchResolve) mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) + names.end() } } else if needed && len(files) > 0 { + resolve := batch.publicationStamps().beginPatchLap(patchResolve) mi.runMasterResolveFiles(files, false) + resolve.end() if receipt != nil && receipt.Complete { + names := batch.publicationStamps().beginPatchLap(patchResolve) mi.runMasterResolveNames(vanishedReceiptNames(mi.graph, receipt)) + names.end() } } else if needed { scope := map[string]struct{}{repoPrefix: {}} if repoPrefix == "" { scope = nil } + resolve := batch.publicationStamps().beginPatchLap(patchResolve) mi.runMasterResolve(scope, false) + resolve.end() // No scoped frontier survived. Refresh any known files without adding a // second graph-wide dataflow or durable-fact scan. if idx != nil && result != nil { @@ -592,10 +619,12 @@ func (mi *MultiIndexer) resolveIncrementalRepoMutationMode( if idx != nil { idx.observeIncrementalCatchup("cross_repo", crossRepoFiles) } + cross := batch.publicationStamps().beginPatchLap(patchResolve) if len(crossRepoFiles) > 0 { mi.runCrossRepoResolveFiles(crossRepoFiles) } else { mi.runCrossRepoResolve(false) } + cross.end() } } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 0776789a3..035ab698c 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -7012,6 +7012,11 @@ type incrementalPathMode struct { forceExplicitFiles bool surfaceFirstVersionChange bool exactPointSemantic bool + // publication carries only a canonical point attempt's local collector. + // Existing batch/reconcile/direct callers leave it nil. No live receipt or + // mutable Indexer tracing state is installed by this per-call observation. + publication *PublicationStamps + readRepair *ReadRepairObservation } // indexedFilesAbsentFromDisk returns the repo-relative keys of files the graph @@ -7449,7 +7454,9 @@ func (idx *Indexer) incrementalReindexPathsMode( invalidation.Files = appendUniqueSorted(invalidation.Files, idx.graphFilePaths(reparsedFiles)...) invalidation.structuralFiles = appendUniqueSorted(invalidation.structuralFiles, idx.graphFilePaths(reparsedFiles)...) invalidation.Files = appendUniqueSorted(invalidation.Files, deletedDependencyFiles...) + mtimes := mode.publication.beginPatchLap(patchMtimes) idx.pruneDeletedFileMtimes(deletedFiles) + mtimes.end() idx.flushReparsePendingEnrichment(markerBatch) if len(evictedGlobalsFiles) > 0 { evicted := make(map[string]struct{}, len(evictedGlobalsFiles)) diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index f2507de14..41f38ab64 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -13,6 +13,7 @@ import ( "sync" "sync/atomic" "time" + "weak" "go.uber.org/zap" "golang.org/x/mod/modfile" @@ -267,12 +268,12 @@ type MultiIndexer struct { // every per-repo Indexer. When nil (the default), per-repo // deferred passes skip semantic enrichment — this is the root // cause of "enrich:0" in daemon mode. Set by the daemon via - // SetSemanticManager before IndexAll / TrackRepo. + // SetSemanticManager before TrackRepo. semanticMgr *semantic.Manager } // SetEmbedder installs the embedding provider every per-repo indexer -// should use. Must be called before IndexAll / TrackRepo for vectors +// should use. Must be called before TrackRepo for vectors // to land in the graph — without this the fresh Indexer created per // repo has embedder=nil and vector-plan preparation skips the embedding pass. // Safe to call zero or one times; subsequent calls silently replace. @@ -427,7 +428,7 @@ func (mi *MultiIndexer) newPerRepoIndexerGuardedWithMode( // SetEmbeddingChunkOptions installs the AST sub-chunking configuration // every per-repo Indexer this MultiIndexer constructs should use, and // re-applies it to every per-repo Indexer already built. Call before -// IndexAll / TrackRepo so the warmup indexers pick it up. +// TrackRepo so the warmup indexers pick it up. func (mi *MultiIndexer) SetEmbeddingChunkOptions(opts embedding.ChunkOptions) { mi.mu.Lock() mi.embedChunkOpts = opts @@ -625,16 +626,6 @@ func (mi *MultiIndexer) RunDeferredPassesAllResult(ctx context.Context) Deferred return mi.BeginDeferredPasses(ctx, nil).FinishTailResult() } -// finishColdDeferredPasses completes the cold-index deferred tail without -// repeating the whole-graph cross-repository resolver. The receipt-guided tail -// already performs either exact cross-repository catch-up or a fail-closed full -// pass; only contract-bridge reconciliation remains afterward. -func (mi *MultiIndexer) finishColdDeferredPasses(ctx context.Context, completedIndexers []*Indexer) DeferredPassesResult { - result := mi.beginDeferredPasses(ctx, nil, completedIndexers, true).FinishTailResult() - mi.ReconcileContractEdges() - return result -} - // current requires this Indexer's repository mutation lane to be held. func (a *deferredPassAttempt) current() bool { return a.indexer.deferredAttempt == a && a.indexer.pendingContractReg == a.registry @@ -2073,79 +2064,6 @@ func NewMultiIndexer( } } -// IndexAll indexes all active repos concurrently. Each repo gets its own -// Indexer instance with repo-specific config. Returns per-repo IndexResults. -func (mi *MultiIndexer) IndexAll() (map[string]*IndexResult, error) { - return mi.IndexScoped("", "") -} - -// IndexScoped is IndexAll restricted to repos whose workspace and -// project slugs match. Empty filters disable that axis (so empty/empty -// is equivalent to IndexAll). Resolution honours the same precedence -// as resolveWorkspaceID/resolveProjectID — RepoEntry override → -// `.gortex.yaml::workspace` → repo prefix — so a `gortex server -// --workspace foo` invocation matches both repos that declare -// `workspace: foo` in their own `.gortex.yaml` and repos pinned to -// `foo` from the user's global config. -// -// Returns an error when the filters exclude every active repo, so a -// `--workspace typo` surfaces as a startup failure rather than a -// silently empty graph. -func (mi *MultiIndexer) IndexScoped(workspaceSlug, projectSlug string) (map[string]*IndexResult, error) { - repos := mi.configMgr.ActiveRepos() - if len(repos) == 0 { - return nil, nil - } - if workspaceSlug != "" || projectSlug != "" { - filtered := mi.filterReposByScope(repos, workspaceSlug, projectSlug) - if len(filtered) == 0 { - return nil, fmt.Errorf("scope filter matched zero of %d active repos (workspace=%q project=%q)", len(repos), workspaceSlug, projectSlug) - } - repos = filtered - } - - // indexMultiRepo owns the complete coordinated pipeline, including the - // one contract reconciliation required before graph-wide derivation. Do - // not repeat it here: reconciliation evicts and rebuilds a generation and - // a second identical pass is pure global work. - // - // A lone repo takes the same path as any other — it is simply the first - // tracked repo, and its nodes carry its prefix like everyone else's. - return mi.indexMultiRepo(repos) -} - -// filterReposByScope returns the subset of repos whose resolved -// workspace and project slugs match the supplied filters. Empty -// filters disable that axis. Loads each repo's `.gortex.yaml` first so -// resolution sees the workspace/project declared there — matching only -// against `RepoEntry.Workspace` would miss repos that declare their -// slug in their own config file (the typical case for first-party -// repos). -func (mi *MultiIndexer) filterReposByScope(repos []config.RepoEntry, workspaceSlug, projectSlug string) []config.RepoEntry { - if workspaceSlug == "" && projectSlug == "" { - return repos - } - out := make([]config.RepoEntry, 0, len(repos)) - for _, e := range repos { - absPath, err := filepath.Abs(e.Path) - if err != nil { - continue - } - prefix := config.ResolvePrefix(e) - mi.configMgr.LoadWorkspaceConfig(prefix, absPath) - cfg := mi.configMgr.GetRepoConfig(prefix) - entryCopy := e - if workspaceSlug != "" && resolveWorkspaceID(&entryCopy, cfg, prefix) != workspaceSlug { - continue - } - if projectSlug != "" && resolveProjectID(&entryCopy, cfg, prefix) != projectSlug { - continue - } - out = append(out, e) - } - return out -} - func (idx *Indexer) installGoResolverContext() { mi := idx.repositoryMutationOwner if mi == nil { @@ -2291,370 +2209,6 @@ func readGoModModule(repoPath string) string { return modfile.ModulePath(data) } -// rebuildColdRefFacts seeds the durable resolved-reference sidecar once for -// the complete successful repository set. The backend implementation is a -// single set-oriented transaction; keeping the capability probe here avoids a -// per-repository persistence loop and leaves in-memory stores as a no-op. -func (mi *MultiIndexer) rebuildColdRefFacts(ctx context.Context, repoPrefixes []string) error { - if err := ctx.Err(); err != nil { - return err - } - rebuilder, ok := mi.graph.(graph.RefFactsRebuilder) - if !ok { - return nil - } - if err := rebuilder.RebuildRefFactsForRepos(repoPrefixes); err != nil { - return err - } - // The current backend contract is atomic but not interruptible in flight. - // Observing cancellation here prevents completion/global consumers from - // publishing a generation whose ref-fact boundary the caller abandoned. - return ctx.Err() -} - -// indexMultiRepo indexes multiple repos concurrently with repo prefixes. -func (mi *MultiIndexer) indexMultiRepo(repos []config.RepoEntry) (map[string]*IndexResult, error) { - type repoResult struct { - prefix string - result *IndexResult - idx *Indexer - meta *RepoMetadata - err error - } - - // Resolve each repo's prefix serially first: git-worktree instancing, - // the derived-name persistence, and the cross-batch collision guard - // must be deterministic, not raced across the parallel index - // goroutines below. The same loop builds the tracked-module map used - // for cross-repo dependency detection. - type resolvedRepo struct { - entry config.RepoEntry - prefix string - cfg *config.Config - absPath string - identity *RepoIdentity - } - var configLookup config.RepoPathLookup - if mi.configMgr != nil { - configLookup = mi.configMgr.Global().SnapshotRepoPaths() - } - moduleRoots := make(map[string]string) - resolved := make([]resolvedRepo, 0, len(repos)) - seenPrefix := make(map[string]string, len(repos)) // prefix → absPath - var resolveErrors []string - for _, entry := range repos { - absPath, err := filepath.Abs(entry.Path) - if err != nil { - resolveErrors = append(resolveErrors, fmt.Sprintf("resolving path %s: %v", entry.Path, err)) - continue - } - info, err := os.Stat(absPath) - if err != nil { - resolveErrors = append(resolveErrors, fmt.Sprintf("opening repository %s: %v", entry.Path, err)) - continue - } - if !info.IsDir() { - resolveErrors = append(resolveErrors, fmt.Sprintf("opening repository %s: not a directory", entry.Path)) - continue - } - identity, _ := DetectIdentity(absPath) - e := entry - prefix, cfg := mi.resolveTrackPrefix(&e, absPath, identity, configLookup, mi.configuredPrefixCheck()) - // Two distinct checkouts can still land on the same derived prefix - // (e.g. two worktrees declaring the same workspace). resolveTrackPrefix - // only sees already-tracked repos, so guard against collisions within - // this batch too. - if prev, ok := seenPrefix[prefix]; ok { - if pathkey.SamePathIdentity(prev, absPath) { - // Duplicate config entries for one repository must not launch - // concurrent raw workers behind a single held repository lane. - continue - } - prefix += "-" + shortPathHash(absPath) - e.Name = prefix - // This prefix is minted for THIS batch and is not the one the - // lifecycle registered an owner under, so the receipt opened for - // it below finds no owner and the repository is left unwitnessed - // (openSourceWitness). That is the honest direction — the nodes - // this pass writes carry the disambiguated prefix too, so no pin - // taken on the colliding prefix is reading them — but a request - // against the disambiguated prefix gets ErrBaseCorpusUnwitnessed - // rather than an exactness label until an owner is registered for - // the name this batch chose. - } - seenPrefix[prefix] = absPath - resolved = append(resolved, resolvedRepo{entry: e, prefix: prefix, cfg: cfg, absPath: absPath, identity: identity}) - moduleRoots[prefix] = absPath - } - trackedModules := readTrackedGoModules(moduleRoots) - for prefix, mod := range trackedModules { - mi.logger.Debug("tracked repo module", zap.String("repo", prefix), zap.String("module", mod)) - } - - if len(resolved) == 0 { - if len(resolveErrors) == 0 { - return map[string]*IndexResult{}, nil - } - sort.Strings(resolveErrors) - return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(resolveErrors, "; ")) - } - - prefixes := make([]string, 0, len(resolved)) - for _, repo := range resolved { - prefixes = append(prefixes, repo.prefix) - } - // Freeze one batch-mode generation before acquiring any stable repository - // lane. This global gate -> sorted lanes order prevents a queued transition - // writer from owning the gate while waiting for a lane held by this batch. - mi.batchMutationGate.RLock() - defer mi.batchMutationGate.RUnlock() - - // One receipt per OUTPUT OWNER this batch writes, opened after every lane - // is held and before any payload moves. The whole batch is one decision: if - // a newer mutation takes over any of these owners while it runs, the batch - // cannot fulfil that owner's generation. - // - // Owners are deduplicated, and not by prefix. Two tracked entries can name - // one repository root (two config entries with different explicit Names and - // one Path: resolveTrackPrefix honours an explicit Name verbatim, so the - // prefix collision guard above never fires). legacyOutputTargetFor keys the - // owner on the root, so both entries name ONE generation-zero output. - // Issuing two receipts for it would make the second supersede the first and - // refuse a cold index that fully succeeded. - batchTargets := make([]OutputMutationTarget, 0, len(resolved)) - seenOutputOwner := make(map[string]bool, len(resolved)) - for _, repo := range resolved { - target := legacyOutputTargetFor(outputStoreIdentity(mi.graph), repo.prefix, repo.absPath, "prefix:"+repo.prefix) - if seenOutputOwner[target.OwnerKey] { - continue - } - seenOutputOwner[target.OwnerKey] = true - batchTargets = append(batchTargets, target) - } - - var finalResults map[string]*IndexResult - laneErr := mi.withRepositoryMutationLanes(context.Background(), prefixes, func() error { - receipts, receiptErr := mi.outputGenerationAuthority().BeginAll( - context.Background(), OutputEntryIndexMultiRepo, batchTargets) - if receiptErr != nil { - return receiptErr - } - settled := false - defer func() { - if !settled { - receipts.Abandon() - } - }() - // Preventive half: refuse before the cold batch writes anything if any - // owner was already taken over while admission blocked on its gates. - if err := receipts.Current(); err != nil { - return err - } - batchErr := func() error { - // The one topology writer covers parse, publication, deferred/cross-repo - // tails, ref-facts, and global derivation. - batchMode := mi.currentBatchMode() - finishTopologyMutation := reach.BeginTopologyMutation(mi.graph) - defer finishTopologyMutation(true) - - pipelineResults, pipelineErr := func() (map[string]*IndexResult, error) { - resultCh := make(chan repoResult, len(resolved)) - var wg sync.WaitGroup - coordinatedBulk, _ := mi.graph.(graph.CoordinatedBulkLoader) - coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() - defer func() { - if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { - mi.logger.Error("multi-repo bulk-load cleanup failed", zap.Error(err)) - } - } - }() - - for _, rr := range resolved { - wg.Add(1) - go func(r resolvedRepo) { - defer wg.Done() - - idx := mi.newPerRepoIndexerGuardedWithMode(r.cfg.Index, batchMode) - idx.SetRepoPrefix(r.prefix) - entryCopy := r.entry - idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, r.cfg, r.prefix)) - idx.SetProjectID(resolveProjectID(&entryCopy, r.cfg, r.prefix)) - idx.SetTrackedRepoModules(trackedModules) - // Defer the per-repo cross-cutting passes (ResolveAll, - // semantic enrich, contract extract+commit) so they don't - // race against each other across goroutines on the shared - // graph. They run serially below via RunDeferredPasses after - // wg.Wait(). The graph-wide derivation passes run once after - // the loop via the shared global-pass pipeline. - idx.SetDeferResolve(true) - - result, err := idx.indexCtxRaw(WithIndexOrigin(context.Background(), "index_all"), r.absPath) - if err != nil { - idx.Close() - resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s: %w", r.absPath, err)} - return - } - if result == nil { - idx.Close() - resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s returned a nil result", r.absPath)} - return - } - result.RepoPrefix = r.prefix - - meta := &RepoMetadata{ - RepoPrefix: r.prefix, - RootPath: r.absPath, - Identity: r.identity, - LastIndexTime: time.Now(), - FileCount: result.FileCount, - NodeCount: result.NodeCount, - EdgeCount: result.EdgeCount, - ParseErrors: result.Errors, - FileMtimes: idx.publishFileMtimes(), - - IsWorktree: ResolveWorktree(r.absPath).IsWorktree, - } - - resultCh <- repoResult{prefix: r.prefix, result: result, idx: idx, meta: meta} - }(rr) - } - - go func() { - wg.Wait() - close(resultCh) - }() - - results := make(map[string]*IndexResult) - indexErrors := resolveErrors - completed := make([]repoResult, 0, len(resolved)) - - // Drain workers without holding the registry lock. Constructors and future - // worker tails may need read access to MultiIndexer state; pinning mi.mu - // across the channel range turns any such read into a producer/consumer - // deadlock and blocks unrelated registry readers for the whole warmup. - for rr := range resultCh { - if rr.err != nil { - mi.logger.Error("failed to index repo", zap.String("prefix", rr.prefix), zap.Error(rr.err)) - indexErrors = append(indexErrors, rr.err.Error()) - continue - } - completed = append(completed, rr) - results[rr.prefix] = rr.result - } - oldIndexers := make([]*Indexer, 0, len(completed)) - mi.mu.Lock() - for _, rr := range completed { - if old := mi.indexers[rr.prefix]; old != nil && old != rr.idx { - oldIndexers = append(oldIndexers, old) - } - mi.repos[rr.prefix] = rr.meta - mi.indexers[rr.prefix] = rr.idx - } - mi.mu.Unlock() - for _, old := range oldIndexers { - old.Close() - } - if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { - return nil, fmt.Errorf("multi-repo bulk-load finalize: %w", err) - } - coordinatedBulkActive = false - } - for _, rr := range completed { - if err := mi.postPublishGoModules(context.Background(), rr.idx, true); err != nil { - mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) - } - } - - // Do not publish a completed pipeline when no repository reached the - // deferred stages. Besides making the failure deterministic, this prevents - // global passes from deriving edges from a partially-drained failed batch. - if len(indexErrors) > 0 && len(results) == 0 { - sort.Strings(indexErrors) - return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(indexErrors, "; ")) - } - - // Complete cold multi-repo indexing through the same coordinated pipeline - // used by daemon warmup: - // 1. materialise go.mod contracts once, then run one shared base resolve; - // 2. enrich repositories in bounded language-aware batches (large Go - // repositories remain exclusive), committing contracts only after each - // batch drains; - // 3. use the mutation receipt to perform only the exact catch-up needed for - // semantic/contract mutations. - // - // The old loop called idx.RunDeferredPasses with - // skipResolveInDeferred=false, so every repository performed ResolveAll over - // the entire shared graph. At R repositories and E edges that was O(R*E) and - // was the dominant cold-index regression. runDeferredGoMod is generation- - // idempotent, so RunDeferredPassesAll does not repeat the pre-resolve work. - deferCtx := context.Background() - if err := mi.RunPreEnrichResolve(deferCtx, nil, nil); err != nil { - return results, fmt.Errorf("multi-repo pre-enrichment resolve: %w", err) - } - completedIndexers := make([]*Indexer, 0, len(completed)) - for _, rr := range completed { - completedIndexers = append(completedIndexers, rr.idx) - } - deferredResult := mi.finishColdDeferredPasses(deferCtx, completedIndexers) - mi.logger.Info("multi-repo coordinated deferred passes complete", - zap.Int("repos_indexed", len(results)), - zap.Int("repos_failed", len(indexErrors)), - zap.Int("enrich_scheduled", deferredResult.EnrichScheduled), - zap.Bool("exact_cross_repo_complete", deferredResult.ExactCrossRepoComplete)) - - // ResolveAll normally seeds ref_facts after a full resolve. The coordinated - // cold path intentionally bypasses per-repository ResolveAll, so seed the - // successful repository set once after every base, semantic catch-up, and - // cross-repository mutation has settled. Sorting makes the boundary stable - // for tracing/tests; SQLite consumes the whole slice in one transaction. - successfulPrefixes := make([]string, 0, len(results)) - for prefix := range results { - successfulPrefixes = append(successfulPrefixes, prefix) - } - sort.Strings(successfulPrefixes) - if err := mi.rebuildColdRefFacts(deferCtx, successfulPrefixes); err != nil { - return results, fmt.Errorf("multi-repo reference-fact rebuild: %w", err) - } - - // Graph-wide derivation passes run exactly once after every repo - // has been parsed, every per-repo and cross-repo resolver has lifted - // placeholder edges, and contract bridges are in place. RunDeferredPasses - // intentionally skips these so we don't pay an O(global) walk per - // repo (was the dominant cost at R≈100+). - mi.runGlobalGraphPassesTopologyHeld(context.Background(), nil, false) - - return results, nil - }() - finalResults = pipelineResults - return pipelineErr - }() - if batchErr != nil { - return batchErr - } - settled = true - // Fulfilment is refused when a newer mutation took over any of these - // owners while the batch ran; the batch then reports that refusal - // instead of announcing a completed cold index. - if err := receipts.Complete(); err != nil { - return err - } - // Every completed repository's journal is accepted, failed files - // included: the cold pass staged a failure envelope for each of them - // (indexCtxRaw), so the accepted namespace excludes exactly that set. - for prefix, result := range finalResults { - if result != nil { - if err := mi.acceptInstalledContractCoreInputs(context.Background(), prefix); err != nil { - return err - } - } - } - return nil - }) - return finalResults, laneErr -} - // IndexRepo re-indexes a single repo by prefix. Evicts existing data first. func (mi *MultiIndexer) IndexRepo(repoPrefix string) (*IndexResult, error) { mi.mu.RLock() @@ -2731,8 +2285,8 @@ func (mi *MultiIndexer) indexRepoRaw(repoPrefix string) (*IndexResult, error) { } }() // Always stamp the repo prefix, even when this is the only tracked repo. - // The multi-repo cold path (indexMultiRepo) already prefixes - // unconditionally; gating the single-repo re-index on repo count left the + // First registration also prefixes unconditionally; gating re-indexing + // on repo count left the // two paths producing different id / path shapes for the same repo, so // adding a second repo turned the first from bare into prefixed — a lossy // whole-graph rewrite. Uniform prefixing removes the single/multi data-shape @@ -2847,6 +2401,18 @@ func (mi *MultiIndexer) RefreshRepoConfigs() int { // directories; otherwise the whole repo root is scanned. Returns an // error when the prefix is not a tracked repo. func (mi *MultiIndexer) IncrementalReindexRepo(repoPrefix string, paths []string) (*IndexResult, error) { + return mi.IncrementalReindexRepoObserved(repoPrefix, paths, nil) +} + +// IncrementalReindexRepoObserved retains the ordinary reconciliation behavior. +// Observation belongs to the executed coordinator drain, not the reader RPC. +func (mi *MultiIndexer) IncrementalReindexRepoObserved(repoPrefix string, paths []string, observer *ReadRepairObserver) (*IndexResult, error) { + var request *repositoryReadRepairRequest + if observer != nil && observer.Complete != nil { + request = &repositoryReadRepairRequest{ + session: observer.Session, requestIDKey: observer.RequestIDKey, complete: observer.Complete, + } + } for attempt := 0; attempt < 3; attempt++ { mi.mu.RLock() meta, ok := mi.repos[repoPrefix] @@ -2867,7 +2433,7 @@ func (mi *MultiIndexer) IncrementalReindexRepo(repoPrefix string, paths []string if !current { continue } - result, err := coordinator.reconcile(context.Background(), canonical) + result, err := coordinator.reconcileObserved(context.Background(), canonical, request) if err == errRepositoryMutationCoordinatorClosed { continue } @@ -2923,9 +2489,14 @@ func (mi *MultiIndexer) incrementalDiscoverRepoRaw(repoPrefix string, paths []st // full shared resolver, semantic, metadata, and precise derived tail without // submitting back into a coordinated public API. func (mi *MultiIndexer) incrementalPointRepoRaw(repoPrefix, path string) (*IndexResult, error) { + return mi.incrementalPointRepoRawObserved(repoPrefix, path, nil) +} + +func (mi *MultiIndexer) incrementalPointRepoRawObserved(repoPrefix, path string, stamps *PublicationStamps) (*IndexResult, error) { return mi.incrementalReindexRepoRawMode(repoPrefix, []string{path}, incrementalPathMode{ detectDeletions: true, forceExplicitFiles: true, + publication: stamps, }) } @@ -2998,6 +2569,9 @@ func (mi *MultiIndexer) incrementalReindexRepoRawMode(repoPrefix string, paths [ if idx == nil { return nil, fmt.Errorf("repository mutation executor has no live indexer: %s", repoPrefix) } + if mode.readRepair != nil { + mode.readRepair.owner = weak.Make(idx) + } // Refresh the producer under its already-held repository lane. A watcher // admission refresh alone would still leave unignored source excluded by @@ -3009,7 +2583,9 @@ func (mi *MultiIndexer) incrementalReindexRepoRawMode(repoPrefix string, paths [ // The stable per-repository lane is held before this executor runs. Take // the global reachability topology gate second and keep it through the // resolver, metadata, and derived tails to avoid lane/gate inversion. + gate := mode.publication.beginPatchLap(patchGateWait) finishTopologyMutation := reach.BeginTopologyMutation(mi.graph) + gate.end() topologyChanged := true defer func() { finishTopologyMutation(topologyChanged) }() @@ -3035,6 +2611,7 @@ func (mi *MultiIndexer) incrementalReindexRepoRawMode(repoPrefix string, paths [ repoPrefix, result, receipt, batch, mode.exactPointSemantic, ) + mtimes := mode.publication.beginPatchLap(patchMtimes) fileMtimes := idx.publishFileMtimes() mi.mu.Lock() mi.repos[repoPrefix] = &RepoMetadata{ @@ -3056,11 +2633,14 @@ func (mi *MultiIndexer) incrementalReindexRepoRawMode(repoPrefix string, paths [ IsWorktree: meta.IsWorktree, } mi.mu.Unlock() + mtimes.end() idx.observeIncrementalCatchup("derived", result.DerivedInvalidation.Files) + derived := mode.publication.beginDerivedPatchLap() mi.runIncrementalDerivedPassesWithPriorTopologyHeld(context.Background(), map[string]DerivedInvalidationPlan{ repoPrefix: result.DerivedInvalidation, }, result.capabilityPrior) + derived.endDerived() topologyChanged = incrementalTopologyChanged(result) return result, nil diff --git a/internal/indexer/multi_cold_orchestration_test.go b/internal/indexer/multi_cold_orchestration_test.go index 28bc6ba3a..71f0fa479 100644 --- a/internal/indexer/multi_cold_orchestration_test.go +++ b/internal/indexer/multi_cold_orchestration_test.go @@ -98,7 +98,7 @@ func TestFinishColdDeferredPassesDoesNotRepeatFullCrossRepoResolve(t *testing.T) store := &coldOrchestrationProbeStore{Graph: graph.New()} mi := newColdOrchestrationMulti(store, zap.NewNop()) - result := mi.finishColdDeferredPasses(t.Context(), nil) + result := mi.finishFixtureDeferredPasses(t.Context(), nil) assert.True(t, result.ExactCrossRepoComplete) assert.Zero(t, store.unresolvedScans.Load(), @@ -112,7 +112,7 @@ func TestIndexMultiRepoUsesOneCoordinatedBaseResolve(t *testing.T) { mi := newColdOrchestrationMulti(graph.New(), zap.New(core)) repos := coldOrchestrationRepos(t, 8) - results, err := mi.indexMultiRepo(repos) + results, err := mi.indexFixtureRepos(repos) require.NoError(t, err) require.Len(t, results, len(repos)) @@ -139,7 +139,7 @@ func TestIndexMultiRepoAllFailuresDoNotRunCompletionPipeline(t *testing.T) { }) } - results, err := mi.indexMultiRepo(repos) + results, err := mi.indexFixtureRepos(repos) require.Error(t, err) assert.Nil(t, results) assert.Empty(t, logs.FilterMessage("DEFERRED-TIMING master.ResolveAll").All()) @@ -332,7 +332,7 @@ func snapshotColdGraph(store graph.Store, repos []config.RepoEntry) coldGraphSha func TestCoordinatedColdPipelineMatchesSerialReferenceGraph(t *testing.T) { repos := coldOrchestrationRepos(t, 5) coordinated := newColdOrchestrationMulti(graph.New(), zap.NewNop()) - _, err := coordinated.indexMultiRepo(repos) + _, err := coordinated.indexFixtureRepos(repos) require.NoError(t, err) serial := serialColdReference(t, repos) @@ -407,7 +407,7 @@ func TestIndexMultiRepoRefFactsMatchesSingleFullAndUsesOneSetRebuild(t *testing. counting := &coldRefFactsCountingStore{Store: store} multi := newColdOrchestrationMulti(counting, zap.NewNop()) - results, err := multi.indexMultiRepo(requested) + results, err := multi.indexFixtureRepos(requested) require.NoError(t, err) require.Len(t, results, len(repos)) require.Equal(t, 1, counting.rebuildCalls, @@ -449,11 +449,11 @@ func TestColdRefFactRebuildPreservesFailureAndCancellation(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) cancel() - err := mi.rebuildColdRefFacts(ctx, []string{"repo-a", "repo-b"}) + err := mi.rebuildFixtureRefFacts(ctx, []string{"repo-a", "repo-b"}) require.ErrorIs(t, err, context.Canceled) require.Zero(t, probe.rebuildCalls, "cancellation before the boundary must skip the rebuild") - err = mi.rebuildColdRefFacts(t.Context(), []string{"repo-a", "repo-b"}) + err = mi.rebuildFixtureRefFacts(t.Context(), []string{"repo-a", "repo-b"}) require.ErrorIs(t, err, sentinel) require.Equal(t, 1, probe.rebuildCalls) require.Equal(t, []string{"repo-a", "repo-b"}, probe.rebuildPrefixes) @@ -465,7 +465,7 @@ func TestIndexMultiRepoRefFactFailureStopsGlobalConsumers(t *testing.T) { probe := &coldRefFactsProbeStore{Store: graph.New(), rebuildErr: sentinel} mi := newColdOrchestrationMulti(probe, zap.New(core)) - results, err := mi.indexMultiRepo(coldRefFactRepos(t, 2)) + results, err := mi.indexFixtureRepos(coldRefFactRepos(t, 2)) require.ErrorIs(t, err, sentinel) require.Len(t, results, 2, "successful parse results remain available with the completion error") require.Equal(t, 1, probe.rebuildCalls) diff --git a/internal/indexer/multi_fixture_test.go b/internal/indexer/multi_fixture_test.go new file mode 100644 index 000000000..2b21c49cf --- /dev/null +++ b/internal/indexer/multi_fixture_test.go @@ -0,0 +1,464 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/pathkey" + "github.com/zzet/gortex/internal/reach" +) + +// These fixtures retain the atomic cold-cohort shape used by lane, topology, +// output-generation and resolver tests. They are deliberately test-only: +// production startup registers roots through TrackRepoCtx and coordinates its +// deferred stages in the daemon. External fixtures use that production path. +// The cohort fixture names the existing topology output entry, not a separate +// production mutation door. +// finishFixtureDeferredPasses completes the cold-index deferred tail without +// repeating the whole-graph cross-repository resolver. The receipt-guided tail +// already performs either exact cross-repository catch-up or a fail-closed full +// pass; only contract-bridge reconciliation remains afterward. +func (mi *MultiIndexer) finishFixtureDeferredPasses(ctx context.Context, completedIndexers []*Indexer) DeferredPassesResult { + result := mi.beginDeferredPasses(ctx, nil, completedIndexers, true).FinishTailResult() + mi.ReconcileContractEdges() + return result +} + +// IndexAll indexes all active repos concurrently. Each repo gets its own +// Indexer instance with repo-specific config. Returns per-repo IndexResults. +func (mi *MultiIndexer) IndexAll() (map[string]*IndexResult, error) { + return mi.indexFixtureScope("", "") +} + +// indexFixtureScope selects fixture roots with the same workspace/project +// precedence as registered repositories: entry override, workspace config, +// then prefix. Empty filters retain every active fixture root; a typo refuses +// an empty test graph. This is not a production startup selection API. +func (mi *MultiIndexer) indexFixtureScope(workspaceSlug, projectSlug string) (map[string]*IndexResult, error) { + repos := mi.configMgr.ActiveRepos() + if len(repos) == 0 { + return nil, nil + } + if workspaceSlug != "" || projectSlug != "" { + filtered := mi.filterFixtureReposByScope(repos, workspaceSlug, projectSlug) + if len(filtered) == 0 { + return nil, fmt.Errorf("scope filter matched zero of %d active repos (workspace=%q project=%q)", len(repos), workspaceSlug, projectSlug) + } + repos = filtered + } + + // indexFixtureRepos owns the complete coordinated pipeline, including the + // one contract reconciliation required before graph-wide derivation. Do + // not repeat it here: reconciliation evicts and rebuilds a generation and + // a second identical pass is pure global work. + // + // A lone repo takes the same path as any other — it is simply the first + // tracked repo, and its nodes carry its prefix like everyone else's. + return mi.indexFixtureRepos(repos) +} + +// filterFixtureReposByScope returns the subset of repos whose resolved +// workspace and project slugs match the supplied filters. Empty +// filters disable that axis. Loads each repo's `.gortex.yaml` first so +// resolution sees the workspace/project declared there — matching only +// against `RepoEntry.Workspace` would miss repos that declare their +// slug in their own config file (the typical case for first-party +// repos). +func (mi *MultiIndexer) filterFixtureReposByScope(repos []config.RepoEntry, workspaceSlug, projectSlug string) []config.RepoEntry { + if workspaceSlug == "" && projectSlug == "" { + return repos + } + out := make([]config.RepoEntry, 0, len(repos)) + for _, e := range repos { + absPath, err := filepath.Abs(e.Path) + if err != nil { + continue + } + prefix := config.ResolvePrefix(e) + mi.configMgr.LoadWorkspaceConfig(prefix, absPath) + cfg := mi.configMgr.GetRepoConfig(prefix) + entryCopy := e + if workspaceSlug != "" && resolveWorkspaceID(&entryCopy, cfg, prefix) != workspaceSlug { + continue + } + if projectSlug != "" && resolveProjectID(&entryCopy, cfg, prefix) != projectSlug { + continue + } + out = append(out, e) + } + return out +} + +// rebuildFixtureRefFacts seeds the durable resolved-reference sidecar once for +// the complete successful repository set. The backend implementation is a +// single set-oriented transaction; keeping the capability probe here avoids a +// per-repository persistence loop and leaves in-memory stores as a no-op. +func (mi *MultiIndexer) rebuildFixtureRefFacts(ctx context.Context, repoPrefixes []string) error { + if err := ctx.Err(); err != nil { + return err + } + rebuilder, ok := mi.graph.(graph.RefFactsRebuilder) + if !ok { + return nil + } + if err := rebuilder.RebuildRefFactsForRepos(repoPrefixes); err != nil { + return err + } + // The current backend contract is atomic but not interruptible in flight. + // Observing cancellation here prevents completion/global consumers from + // publishing a generation whose ref-fact boundary the caller abandoned. + return ctx.Err() +} + +// indexFixtureRepos indexes multiple repos concurrently with repo prefixes. +func (mi *MultiIndexer) indexFixtureRepos(repos []config.RepoEntry) (map[string]*IndexResult, error) { + type repoResult struct { + prefix string + result *IndexResult + idx *Indexer + meta *RepoMetadata + err error + } + + // Resolve each repo's prefix serially first: git-worktree instancing, + // the derived-name persistence, and the cross-batch collision guard + // must be deterministic, not raced across the parallel index + // goroutines below. The same loop builds the tracked-module map used + // for cross-repo dependency detection. + type resolvedRepo struct { + entry config.RepoEntry + prefix string + cfg *config.Config + absPath string + identity *RepoIdentity + } + var configLookup config.RepoPathLookup + if mi.configMgr != nil { + configLookup = mi.configMgr.Global().SnapshotRepoPaths() + } + moduleRoots := make(map[string]string) + resolved := make([]resolvedRepo, 0, len(repos)) + seenPrefix := make(map[string]string, len(repos)) // prefix → absPath + var resolveErrors []string + for _, entry := range repos { + absPath, err := filepath.Abs(entry.Path) + if err != nil { + resolveErrors = append(resolveErrors, fmt.Sprintf("resolving path %s: %v", entry.Path, err)) + continue + } + info, err := os.Stat(absPath) + if err != nil { + resolveErrors = append(resolveErrors, fmt.Sprintf("opening repository %s: %v", entry.Path, err)) + continue + } + if !info.IsDir() { + resolveErrors = append(resolveErrors, fmt.Sprintf("opening repository %s: not a directory", entry.Path)) + continue + } + identity, _ := DetectIdentity(absPath) + e := entry + prefix, cfg := mi.resolveTrackPrefix(&e, absPath, identity, configLookup, mi.configuredPrefixCheck()) + // Two distinct checkouts can still land on the same derived prefix + // (e.g. two worktrees declaring the same workspace). resolveTrackPrefix + // only sees already-tracked repos, so guard against collisions within + // this batch too. + if prev, ok := seenPrefix[prefix]; ok { + if pathkey.SamePathIdentity(prev, absPath) { + // Duplicate config entries for one repository must not launch + // concurrent raw workers behind a single held repository lane. + continue + } + prefix += "-" + shortPathHash(absPath) + e.Name = prefix + // This prefix is minted for THIS batch and is not the one the + // lifecycle registered an owner under, so the receipt opened for + // it below finds no owner and the repository is left unwitnessed + // (openSourceWitness). That is the honest direction — the nodes + // this pass writes carry the disambiguated prefix too, so no pin + // taken on the colliding prefix is reading them — but a request + // against the disambiguated prefix gets ErrBaseCorpusUnwitnessed + // rather than an exactness label until an owner is registered for + // the name this batch chose. + } + seenPrefix[prefix] = absPath + resolved = append(resolved, resolvedRepo{entry: e, prefix: prefix, cfg: cfg, absPath: absPath, identity: identity}) + moduleRoots[prefix] = absPath + } + trackedModules := readTrackedGoModules(moduleRoots) + for prefix, mod := range trackedModules { + mi.logger.Debug("tracked repo module", zap.String("repo", prefix), zap.String("module", mod)) + } + + if len(resolved) == 0 { + if len(resolveErrors) == 0 { + return map[string]*IndexResult{}, nil + } + sort.Strings(resolveErrors) + return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(resolveErrors, "; ")) + } + + prefixes := make([]string, 0, len(resolved)) + for _, repo := range resolved { + prefixes = append(prefixes, repo.prefix) + } + // Freeze one batch-mode generation before acquiring any stable repository + // lane. This global gate -> sorted lanes order prevents a queued transition + // writer from owning the gate while waiting for a lane held by this batch. + mi.batchMutationGate.RLock() + defer mi.batchMutationGate.RUnlock() + + // One receipt per OUTPUT OWNER this batch writes, opened after every lane + // is held and before any payload moves. The whole batch is one decision: if + // a newer mutation takes over any of these owners while it runs, the batch + // cannot fulfil that owner's generation. + // + // Owners are deduplicated, and not by prefix. Two tracked entries can name + // one repository root (two config entries with different explicit Names and + // one Path: resolveTrackPrefix honours an explicit Name verbatim, so the + // prefix collision guard above never fires). legacyOutputTargetFor keys the + // owner on the root, so both entries name ONE generation-zero output. + // Issuing two receipts for it would make the second supersede the first and + // refuse a cold index that fully succeeded. + batchTargets := make([]OutputMutationTarget, 0, len(resolved)) + seenOutputOwner := make(map[string]bool, len(resolved)) + for _, repo := range resolved { + target := legacyOutputTargetFor(outputStoreIdentity(mi.graph), repo.prefix, repo.absPath, "prefix:"+repo.prefix) + if seenOutputOwner[target.OwnerKey] { + continue + } + seenOutputOwner[target.OwnerKey] = true + batchTargets = append(batchTargets, target) + } + + var finalResults map[string]*IndexResult + laneErr := mi.withRepositoryMutationLanes(context.Background(), prefixes, func() error { + receipts, receiptErr := mi.outputGenerationAuthority().BeginAll( + context.Background(), OutputEntryRepositoryTopology, batchTargets) + if receiptErr != nil { + return receiptErr + } + settled := false + defer func() { + if !settled { + receipts.Abandon() + } + }() + // Preventive half: refuse before the cold batch writes anything if any + // owner was already taken over while admission blocked on its gates. + if err := receipts.Current(); err != nil { + return err + } + batchErr := func() error { + // The one topology writer covers parse, publication, deferred/cross-repo + // tails, ref-facts, and global derivation. + batchMode := mi.currentBatchMode() + finishTopologyMutation := reach.BeginTopologyMutation(mi.graph) + defer finishTopologyMutation(true) + + pipelineResults, pipelineErr := func() (map[string]*IndexResult, error) { + resultCh := make(chan repoResult, len(resolved)) + var wg sync.WaitGroup + coordinatedBulk, _ := mi.graph.(graph.CoordinatedBulkLoader) + coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() + defer func() { + if coordinatedBulkActive { + if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + mi.logger.Error("multi-repo bulk-load cleanup failed", zap.Error(err)) + } + } + }() + + for _, rr := range resolved { + wg.Add(1) + go func(r resolvedRepo) { + defer wg.Done() + + idx := mi.newPerRepoIndexerGuardedWithMode(r.cfg.Index, batchMode) + idx.SetRepoPrefix(r.prefix) + entryCopy := r.entry + idx.SetWorkspaceID(resolveWorkspaceID(&entryCopy, r.cfg, r.prefix)) + idx.SetProjectID(resolveProjectID(&entryCopy, r.cfg, r.prefix)) + idx.SetTrackedRepoModules(trackedModules) + // Defer the per-repo cross-cutting passes (ResolveAll, + // semantic enrich, contract extract+commit) so they don't + // race against each other across goroutines on the shared + // graph. They run serially below via RunDeferredPasses after + // wg.Wait(). The graph-wide derivation passes run once after + // the loop via the shared global-pass pipeline. + idx.SetDeferResolve(true) + + result, err := idx.indexCtxRaw(WithIndexOrigin(context.Background(), "index_all"), r.absPath) + if err != nil { + idx.Close() + resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s: %w", r.absPath, err)} + return + } + if result == nil { + idx.Close() + resultCh <- repoResult{prefix: r.prefix, err: fmt.Errorf("indexing %s returned a nil result", r.absPath)} + return + } + result.RepoPrefix = r.prefix + + meta := &RepoMetadata{ + RepoPrefix: r.prefix, + RootPath: r.absPath, + Identity: r.identity, + LastIndexTime: time.Now(), + FileCount: result.FileCount, + NodeCount: result.NodeCount, + EdgeCount: result.EdgeCount, + ParseErrors: result.Errors, + FileMtimes: idx.publishFileMtimes(), + + IsWorktree: ResolveWorktree(r.absPath).IsWorktree, + } + + resultCh <- repoResult{prefix: r.prefix, result: result, idx: idx, meta: meta} + }(rr) + } + + go func() { + wg.Wait() + close(resultCh) + }() + + results := make(map[string]*IndexResult) + indexErrors := resolveErrors + completed := make([]repoResult, 0, len(resolved)) + + // Drain workers without holding the registry lock. Constructors and future + // worker tails may need read access to MultiIndexer state; pinning mi.mu + // across the channel range turns any such read into a producer/consumer + // deadlock and blocks unrelated registry readers for the whole warmup. + for rr := range resultCh { + if rr.err != nil { + mi.logger.Error("failed to index repo", zap.String("prefix", rr.prefix), zap.Error(rr.err)) + indexErrors = append(indexErrors, rr.err.Error()) + continue + } + completed = append(completed, rr) + results[rr.prefix] = rr.result + } + oldIndexers := make([]*Indexer, 0, len(completed)) + mi.mu.Lock() + for _, rr := range completed { + if old := mi.indexers[rr.prefix]; old != nil && old != rr.idx { + oldIndexers = append(oldIndexers, old) + } + mi.repos[rr.prefix] = rr.meta + mi.indexers[rr.prefix] = rr.idx + } + mi.mu.Unlock() + for _, old := range oldIndexers { + old.Close() + } + if coordinatedBulkActive { + if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + return nil, fmt.Errorf("multi-repo bulk-load finalize: %w", err) + } + coordinatedBulkActive = false + } + for _, rr := range completed { + if err := mi.postPublishGoModules(context.Background(), rr.idx, true); err != nil { + mi.logger.Warn("post-publish Go module follow-up", zap.Error(err)) + } + } + + // Do not publish a completed pipeline when no repository reached the + // deferred stages. Besides making the failure deterministic, this prevents + // global passes from deriving edges from a partially-drained failed batch. + if len(indexErrors) > 0 && len(results) == 0 { + sort.Strings(indexErrors) + return nil, fmt.Errorf("all repos failed to index: %s", strings.Join(indexErrors, "; ")) + } + + // Complete cold multi-repo indexing through the same coordinated pipeline + // used by daemon warmup: + // 1. materialise go.mod contracts once, then run one shared base resolve; + // 2. enrich repositories in bounded language-aware batches (large Go + // repositories remain exclusive), committing contracts only after each + // batch drains; + // 3. use the mutation receipt to perform only the exact catch-up needed for + // semantic/contract mutations. + // + // The old loop called idx.RunDeferredPasses with + // skipResolveInDeferred=false, so every repository performed ResolveAll over + // the entire shared graph. At R repositories and E edges that was O(R*E) and + // was the dominant cold-index regression. runDeferredGoMod is generation- + // idempotent, so RunDeferredPassesAll does not repeat the pre-resolve work. + deferCtx := context.Background() + if err := mi.RunPreEnrichResolve(deferCtx, nil, nil); err != nil { + return results, fmt.Errorf("multi-repo pre-enrichment resolve: %w", err) + } + completedIndexers := make([]*Indexer, 0, len(completed)) + for _, rr := range completed { + completedIndexers = append(completedIndexers, rr.idx) + } + deferredResult := mi.finishFixtureDeferredPasses(deferCtx, completedIndexers) + mi.logger.Info("multi-repo coordinated deferred passes complete", + zap.Int("repos_indexed", len(results)), + zap.Int("repos_failed", len(indexErrors)), + zap.Int("enrich_scheduled", deferredResult.EnrichScheduled), + zap.Bool("exact_cross_repo_complete", deferredResult.ExactCrossRepoComplete)) + + // ResolveAll normally seeds ref_facts after a full resolve. The coordinated + // cold path intentionally bypasses per-repository ResolveAll, so seed the + // successful repository set once after every base, semantic catch-up, and + // cross-repository mutation has settled. Sorting makes the boundary stable + // for tracing/tests; SQLite consumes the whole slice in one transaction. + successfulPrefixes := make([]string, 0, len(results)) + for prefix := range results { + successfulPrefixes = append(successfulPrefixes, prefix) + } + sort.Strings(successfulPrefixes) + if err := mi.rebuildFixtureRefFacts(deferCtx, successfulPrefixes); err != nil { + return results, fmt.Errorf("multi-repo reference-fact rebuild: %w", err) + } + + // Graph-wide derivation passes run exactly once after every repo + // has been parsed, every per-repo and cross-repo resolver has lifted + // placeholder edges, and contract bridges are in place. RunDeferredPasses + // intentionally skips these so we don't pay an O(global) walk per + // repo (was the dominant cost at R≈100+). + mi.runGlobalGraphPassesTopologyHeld(context.Background(), nil, false) + + return results, nil + }() + finalResults = pipelineResults + return pipelineErr + }() + if batchErr != nil { + return batchErr + } + settled = true + // Fulfilment is refused when a newer mutation took over any of these + // owners while the batch ran; the batch then reports that refusal + // instead of announcing a completed cold index. + if err := receipts.Complete(); err != nil { + return err + } + // Every completed repository's journal is accepted, failed files + // included: the cold pass staged a failure envelope for each of them + // (indexCtxRaw), so the accepted namespace excludes exactly that set. + for prefix, result := range finalResults { + if result != nil { + if err := mi.acceptInstalledContractCoreInputs(context.Background(), prefix); err != nil { + return err + } + } + } + return nil + }) + return finalResults, laneErr +} diff --git a/internal/indexer/multi_scoped_test.go b/internal/indexer/multi_scoped_test.go index fc0281449..e019b7cc4 100644 --- a/internal/indexer/multi_scoped_test.go +++ b/internal/indexer/multi_scoped_test.go @@ -14,11 +14,11 @@ import ( "github.com/zzet/gortex/internal/search" ) -// TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML verifies that -// IndexScoped picks up the workspace slug declared in a repo's own +// TestMultiIndexer_FixtureScope_WorkspaceFromGortexYAML verifies that +// indexFixtureScope picks up the workspace slug declared in a repo's own // `.gortex.yaml` — the typical first-party setup. Repos with a // different workspace slug must be excluded. -func TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML(t *testing.T) { +func TestMultiIndexer_FixtureScope_WorkspaceFromGortexYAML(t *testing.T) { repoA := setupRepoDir(t, "repo-a") repoB := setupRepoDir(t, "repo-b") require.NoError(t, os.WriteFile(filepath.Join(repoA, ".gortex.yaml"), @@ -42,7 +42,7 @@ func TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - results, err := mi.IndexScoped("alpha", "") + results, err := mi.indexFixtureScope("alpha", "") require.NoError(t, err) require.Len(t, results, 1) assert.Contains(t, results, "repo-a") @@ -50,11 +50,11 @@ func TestMultiIndexer_IndexScoped_WorkspaceFromGortexYAML(t *testing.T) { assert.Nil(t, mi.GetMetadata("repo-b")) } -// TestMultiIndexer_IndexScoped_RepoEntryOverride verifies that a +// TestMultiIndexer_FixtureScope_RepoEntryOverride verifies that a // global-config RepoEntry.Workspace override beats the repo's own // `.gortex.yaml::workspace` slug — the user pinning OSS / read-only // repos into a workspace without leaving an artifact in them. -func TestMultiIndexer_IndexScoped_RepoEntryOverride(t *testing.T) { +func TestMultiIndexer_FixtureScope_RepoEntryOverride(t *testing.T) { repoA := setupRepoDir(t, "repo-a") require.NoError(t, os.WriteFile(filepath.Join(repoA, ".gortex.yaml"), []byte("workspace: upstream\n"), 0o644)) @@ -75,19 +75,19 @@ func TestMultiIndexer_IndexScoped_RepoEntryOverride(t *testing.T) { mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) // Override wins over .gortex.yaml: "upstream" matches nothing. - _, err = mi.IndexScoped("upstream", "") + _, err = mi.indexFixtureScope("upstream", "") require.Error(t, err) // "local-pin" matches the override. - results, err := mi.IndexScoped("local-pin", "") + results, err := mi.indexFixtureScope("local-pin", "") require.NoError(t, err) require.Len(t, results, 1) } -// TestMultiIndexer_IndexScoped_FallsBackToRepoPrefix verifies the +// TestMultiIndexer_FixtureScope_FallsBackToRepoPrefix verifies the // default: repos with no `.gortex.yaml::workspace` and no // RepoEntry.Workspace use the repo prefix as their workspace slug. -func TestMultiIndexer_IndexScoped_FallsBackToRepoPrefix(t *testing.T) { +func TestMultiIndexer_FixtureScope_FallsBackToRepoPrefix(t *testing.T) { repoA := setupRepoDir(t, "repo-a") repoB := setupRepoDir(t, "repo-b") @@ -107,17 +107,17 @@ func TestMultiIndexer_IndexScoped_FallsBackToRepoPrefix(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - results, err := mi.IndexScoped("repo-b", "") + results, err := mi.indexFixtureScope("repo-b", "") require.NoError(t, err) require.Len(t, results, 1) assert.Contains(t, results, "repo-b") } -// TestMultiIndexer_IndexScoped_ProjectNarrowsInsideWorkspace verifies +// TestMultiIndexer_FixtureScope_ProjectNarrowsInsideWorkspace verifies // that workspace + project filters are AND-combined. Two repos in the // same workspace but different projects → only the matching project // gets indexed. -func TestMultiIndexer_IndexScoped_ProjectNarrowsInsideWorkspace(t *testing.T) { +func TestMultiIndexer_FixtureScope_ProjectNarrowsInsideWorkspace(t *testing.T) { repoA := setupRepoDir(t, "repo-a") repoB := setupRepoDir(t, "repo-b") require.NoError(t, os.WriteFile(filepath.Join(repoA, ".gortex.yaml"), @@ -141,16 +141,16 @@ func TestMultiIndexer_IndexScoped_ProjectNarrowsInsideWorkspace(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - results, err := mi.IndexScoped("shared", "api") + results, err := mi.indexFixtureScope("shared", "api") require.NoError(t, err) require.Len(t, results, 1) assert.Contains(t, results, "repo-a") } -// TestMultiIndexer_IndexScoped_NoMatchErrors verifies that a typo'd +// TestMultiIndexer_FixtureScope_NoMatchErrors verifies that a typo'd // filter surfaces as an error rather than producing a silent empty // graph. -func TestMultiIndexer_IndexScoped_NoMatchErrors(t *testing.T) { +func TestMultiIndexer_FixtureScope_NoMatchErrors(t *testing.T) { repoA := setupRepoDir(t, "repo-a") tmpCfg := filepath.Join(t.TempDir(), "config.yaml") @@ -166,16 +166,16 @@ func TestMultiIndexer_IndexScoped_NoMatchErrors(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("nonexistent", "") + _, err = mi.indexFixtureScope("nonexistent", "") require.Error(t, err) assert.Contains(t, err.Error(), "scope filter matched zero") assert.Equal(t, 0, g.NodeCount(), "no nodes should be added on a zero-match scope") } -// TestMultiIndexer_IndexScoped_EmptyFiltersEqualsIndexAll verifies +// TestMultiIndexer_FixtureScope_EmptyFiltersEqualsIndexAll verifies // that empty/empty filters degrade to IndexAll behaviour: every // active repo is indexed. -func TestMultiIndexer_IndexScoped_EmptyFiltersEqualsIndexAll(t *testing.T) { +func TestMultiIndexer_FixtureScope_EmptyFiltersEqualsIndexAll(t *testing.T) { repoA := setupRepoDir(t, "repo-a") repoB := setupRepoDir(t, "repo-b") require.NoError(t, os.WriteFile(filepath.Join(repoA, ".gortex.yaml"), @@ -197,7 +197,7 @@ func TestMultiIndexer_IndexScoped_EmptyFiltersEqualsIndexAll(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - results, err := mi.IndexScoped("", "") + results, err := mi.indexFixtureScope("", "") require.NoError(t, err) assert.Len(t, results, 2) assert.Contains(t, results, "repo-a") diff --git a/internal/indexer/multi_watcher.go b/internal/indexer/multi_watcher.go index 5e5762aac..4d1316f41 100644 --- a/internal/indexer/multi_watcher.go +++ b/internal/indexer/multi_watcher.go @@ -251,8 +251,8 @@ func (mw *MultiWatcher) createWatcher(prefix string, cfg config.WatchConfig) err w.explicitReindexRaw = func(paths []string) (*IndexResult, error) { return mw.multi.incrementalReindexRepoRawMode(prefix, paths, incrementalPathMode{detectDeletions: true, forceExplicitFiles: true}) } - w.pointReindexRaw = func(filePath string) (*IndexResult, error) { - return mw.multi.incrementalPointRepoRaw(prefix, filePath) + w.pointReindexRaw = func(filePath string, stamps *PublicationStamps) (*IndexResult, error) { + return mw.multi.incrementalPointRepoRawObserved(prefix, filePath, stamps) } mw.watchers[prefix] = w @@ -1112,6 +1112,7 @@ func (mw *MultiWatcher) OnWorktreeChangeContext(cb WorktreeChangeCallback) { // cannot force a synchronous fallback, and a watcher that failed to start is // never reported as accepting work. func (mw *MultiWatcher) EnqueueFileMutation(ctx context.Context, filePath string) (*MutationTicket, error) { + ctx, _ = canonicalSchedulingEntry(ctx) if err := ctx.Err(); err != nil { return nil, err } diff --git a/internal/indexer/output_generation_authority_test.go b/internal/indexer/output_generation_authority_test.go index 1e6a0e747..2db72513f 100644 --- a/internal/indexer/output_generation_authority_test.go +++ b/internal/indexer/output_generation_authority_test.go @@ -298,7 +298,6 @@ func TestEveryProductionMutationEntryPointIsRegistered(t *testing.T) { // same function still names an entry. required := []mutationDoor{ {file: "contract_baseline_reconcile.go", fn: "reconcilePrimaryContractBaseline", door: "withRepositoryMutationLanes", entries: []string{"OutputEntryContractBaseline"}}, - {file: "multi.go", fn: "indexMultiRepo", door: "withRepositoryMutationLanes", entries: []string{"OutputEntryIndexMultiRepo"}}, {file: "multi.go", fn: "IndexRepo", door: "runExclusive", entries: []string{"OutputEntryIndexRepo"}}, {file: "multi.go", fn: "incrementalDiscoverRepo", door: "runExclusive", entries: []string{"OutputEntryIncrementalDiscoverRepo"}}, {file: "repository_topology_batch.go", fn: "coordinateRepositoryTopologyMutation", door: "runExclusiveLaneOnly", entries: []string{"OutputEntryRepositoryTopology"}}, @@ -407,7 +406,6 @@ func outputMutationEntryValue(identifier string) (string, bool) { "OutputEntryReresolveFileScoped": OutputEntryReresolveFileScoped, "OutputEntryDeferredPasses": OutputEntryDeferredPasses, "OutputEntryRepositoryTopology": OutputEntryRepositoryTopology, - "OutputEntryIndexMultiRepo": OutputEntryIndexMultiRepo, "OutputEntryIndexRepo": OutputEntryIndexRepo, "OutputEntryIncrementalDiscoverRepo": OutputEntryIncrementalDiscoverRepo, "OutputEntryGitWatcherFinalize": OutputEntryGitWatcherFinalize, @@ -619,7 +617,7 @@ func TestSupersededWorkIsRefusedBeforeItWrites(t *testing.T) { } // The batch form refuses the same way, before the batch writes anything. - batchOlder, err := authority.Begin(ctx, OutputEntryIndexMultiRepo, target) + batchOlder, err := authority.Begin(ctx, OutputEntryRepositoryTopology, target) if err != nil { t.Fatalf("batch receipt: %v", err) } @@ -627,7 +625,7 @@ func TestSupersededWorkIsRefusedBeforeItWrites(t *testing.T) { if err := batch.Current(); err != nil { t.Fatalf("an uncontested batch must be current: %v", err) } - contender, err := authority.Begin(ctx, OutputEntryIndexMultiRepo, target) + contender, err := authority.Begin(ctx, OutputEntryRepositoryTopology, target) if err != nil { t.Fatalf("batch contender: %v", err) } @@ -1429,9 +1427,9 @@ func TestColdBatchOpensOneReceiptPerRepositoryOwner(t *testing.T) { t.Fatalf("IndexAll: %v", err) } stats := authority.Stats() - if got := stats.Entries[OutputEntryIndexMultiRepo]; got != 2 { + if got := stats.Entries[OutputEntryRepositoryTopology]; got != 2 { t.Fatalf("the cold batch opened %d receipts under %s, want one per repository owner (2)", - got, OutputEntryIndexMultiRepo) + got, OutputEntryRepositoryTopology) } if stats.Superseded != 0 || stats.Settled != stats.Issued { t.Fatalf("every batch receipt must settle cleanly: %+v", stats) diff --git a/internal/indexer/point_edit_writes_test.go b/internal/indexer/point_edit_writes_test.go index 8d8f54267..a5b6c7d65 100644 --- a/internal/indexer/point_edit_writes_test.go +++ b/internal/indexer/point_edit_writes_test.go @@ -196,7 +196,9 @@ func TestPointEditWriteTelemetryFieldsAndLineBudget(t *testing.T) { logs.TakeAll() w, err := NewWatcher(f.mi.indexers["fixture"], config.WatchConfig{}, zap.New(core)) require.NoError(t, err) - w.pointReindexRaw = func(path string) (*IndexResult, error) { return f.mi.incrementalPointRepoRaw("fixture", path) } + w.pointReindexRaw = func(path string, _ *PublicationStamps) (*IndexResult, error) { + return f.mi.incrementalPointRepoRaw("fixture", path) + } source := pointEditText(f.sources["hub.go"], "hub", "add_func") require.NoError(t, os.WriteFile(filepath.Join(f.root, "hub.go"), []byte(source), 0600)) require.NoError(t, w.patchGraphAfterReceiptCheck(filepath.Join(f.root, "hub.go"), ChangeModified)) diff --git a/internal/indexer/publication_phases.go b/internal/indexer/publication_phases.go index 9a566a58b..aeb4ce225 100644 --- a/internal/indexer/publication_phases.go +++ b/internal/indexer/publication_phases.go @@ -7,6 +7,7 @@ import ( "sync" "sync/atomic" "time" + "weak" ) // PublicationPhase names one step between an edit reaching the daemon and the @@ -69,8 +70,18 @@ const ( // coordinator's queue. PublicationTicketEnqueued PublicationPhase = "ticket_enqueued" // PublicationCycleStarted is the coordinator starting the cycle that will - // serve the ticket (the quiet window has closed). + // serve the ticket (the quiet window has closed), or the accepted canonical + // point attempt becoming runnable. It is not a pure debounce duration. PublicationCycleStarted PublicationPhase = "cycle_started" + // These are the accepted canonical point attempt's actual common executor + // boundaries. They exclude before-image inspection and post-executor event + // work; they are not a full patch wall or a split of its nested deep laps. + PublicationPatchExecutionStarted PublicationPhase = "patch_execution_started" + PublicationPatchExecutionFinished PublicationPhase = "patch_execution_finished" + // The precise derived tail follows resolver and semantic work. These marks + // surround that actual call, not the wider point executor or its nested laps. + PublicationPatchDerivedStarted PublicationPhase = "patch_derived_started" + PublicationPatchDerivedFinished PublicationPhase = "patch_derived_finished" // PublicationAdmitted is the build acquiring the shared build lane. PublicationAdmitted PublicationPhase = "admitted" // PublicationPlanned is the build's file set and generation being fixed. @@ -89,6 +100,9 @@ const ( // route it published describes a sample taken after the ticket arrived. // It is terminal. PublicationTicketCompleted PublicationPhase = "ticket_completed" + // PublicationTimerArmStarted is an independently observed native serving + // timer's actual arm start; it is not request admission. + PublicationTimerArmStarted PublicationPhase = "timer_arm_started" // PublicationTicketFailed is the ticket ending without a publication. It // is terminal. PublicationTicketFailed PublicationPhase = "ticket_failed" @@ -157,15 +171,27 @@ func DefaultPublicationPhases() *PublicationPhaseRecorder { return defaultPublic // PublicationPhaseRecord is one edit's (or one fresh request's) path to // publication. type PublicationPhaseRecord struct { - mu sync.Mutex - key string - checkoutID string - source string - ticket uint64 - generation int64 - origin time.Time - marks []publicationMark - terminal bool + mu sync.Mutex + key string + checkoutID string + source string + ticket uint64 + generation int64 + origin time.Time + marks []publicationMark + terminal bool + canonicalBound bool + patchLaps *canonicalPatchEvidence // immutable, exact accepted attempt only + laneWait *canonicalLaneWaitEvidence // separate from executor-contained laps + debounce *canonicalDebounceEvidence + owner weak.Pointer[PublicationPhaseRecorder] // membership only; never retains an expired recorder + acceptedAttempt *PublicationStamps // private exact winner witness, cleared on completion + attemptWindow *canonicalAttemptWindow + admissions *canonicalAdmissionEvidence + coalescedWinner weak.Pointer[PublicationPhaseRecord] // identity only; never retains an evicted attempt + coalescedLinked bool + background *canonicalBackgroundEvidence + backgroundUnavailable *RequestedBackgroundUnavailable } type publicationMark struct { @@ -186,11 +212,15 @@ func (r *PublicationPhaseRecorder) Begin(checkoutID, key, source string, origin } r.mu.Lock() defer r.mu.Unlock() + return r.beginLocked(checkoutID, key, source, origin) +} + +func (r *PublicationPhaseRecorder) beginLocked(checkoutID, key, source string, origin time.Time) *PublicationPhaseRecord { if existing, ok := r.byKey[key]; ok { return existing } record := &PublicationPhaseRecord{ - key: key, checkoutID: checkoutID, source: source, origin: origin, + key: key, checkoutID: checkoutID, source: source, origin: origin, owner: weak.Make(r), marks: []publicationMark{{phase: PublicationReceived, at: origin}}, } if _, known := r.byCheckout[checkoutID]; !known && len(r.byCheckout) >= r.checkouts { @@ -340,6 +370,35 @@ func (p *PublicationPhaseRecord) BindTicket(sequence uint64) { p.mu.Unlock() } +// bindCanonicalTicket is called under the watcher's admission mutex, before +// its debounce timer can wake. A record describes one ticket: accidentally +// reused or terminal context metadata must not attach an old edit to a newer +// generation. This only refuses timing association, never mutation admission. +func (p *PublicationPhaseRecord) bindCanonicalTicket(sequence uint64, at time.Time) bool { + if p == nil || sequence == 0 { + return false + } + p.mu.Lock() + defer p.mu.Unlock() + if p.terminal || p.canonicalBound || at.Before(p.origin) || p.ticket != 0 { + return false + } + p.canonicalBound = true + p.ticket = sequence + p.markLocked(PublicationTicketEnqueued, at) + return true +} + +// Key returns the current recorder identity without allocating a phase snapshot. +func (p *PublicationPhaseRecord) Key() string { + if p == nil { + return "" + } + p.mu.Lock() + defer p.mu.Unlock() + return p.key +} + // SetGeneration records the dirty generation that published this record. func (p *PublicationPhaseRecord) SetGeneration(generationID int64) { if p == nil || generationID <= 0 { @@ -413,6 +472,11 @@ type PublicationPhaseSnapshot struct { Clock string `json:"clock"` Terminal bool `json:"terminal"` Phases []PublicationPhaseOffset `json:"phases"` + patchLaps *canonicalPatchEvidence // private; legacy timeline wire stays unchanged + laneWait *canonicalLaneWaitEvidence + debounce *canonicalDebounceEvidence + attemptWindow *canonicalAttemptWindow + admissions *canonicalAdmissionEvidence } // Snapshot renders the record. @@ -429,6 +493,11 @@ func (p *PublicationPhaseRecord) Snapshot() PublicationPhaseSnapshot { Clock: "monotonic", Terminal: p.terminal, Phases: make([]PublicationPhaseOffset, 0, len(p.marks)), + patchLaps: p.patchLaps, + laneWait: p.laneWait, + debounce: p.debounce, + attemptWindow: p.attemptWindow, + admissions: p.admissions, } for _, mark := range p.marks { offset := mark.at.Sub(p.origin) @@ -477,8 +546,12 @@ func (p *PublicationPhaseRecord) FinishBackgroundWork(completed bool) { // (view selection, admission, the parse gate, the write's own checks) is // exactly where a slow commit's time goes. The record absorbs them when it opens. type PublicationStamps struct { - mu sync.Mutex - marks []publicationMark + mu sync.Mutex + marks []publicationMark + patchLaps *canonicalPatchLaps + laneWait canonicalLaneWait + debounce canonicalDebounceWait + admissions [canonicalAdmissionCount]canonicalAdmissionPair } type publicationStampsKey struct{} @@ -539,8 +612,37 @@ func (p *PublicationPhaseRecord) Absorb(stamps *PublicationStamps) { } stamps.mu.Lock() marks := append([]publicationMark(nil), stamps.marks...) + var patch *canonicalPatchEvidence + var lane *canonicalLaneWaitEvidence + var debounce *canonicalDebounceEvidence + var admissions *canonicalAdmissionEvidence + if stamps.patchLaps != nil { + frozen := stamps.patchLaps.freeze(p.origin) + patch = &frozen + frozenAdmissions := stamps.freezeCanonicalAdmissions(p.origin) + admissions = &frozenAdmissions + } + if !stamps.laneWait.started.IsZero() || stamps.laneWait.invalid { + frozen := stamps.laneWait.freeze(p.origin) + lane = &frozen + } + if !stamps.debounce.started.IsZero() { + frozen := stamps.debounce.freeze(p.origin) + debounce = &frozen + } stamps.mu.Unlock() + p.mu.Lock() + defer p.mu.Unlock() for _, mark := range marks { - p.MarkAt(mark.phase, mark.at) + if !mark.at.IsZero() { + p.markLocked(mark.phase, mark.at) + } + } + if !p.terminal && p.canonicalBound && p.patchLaps == nil { + p.patchLaps = patch + p.laneWait = lane + p.debounce = debounce + p.acceptedAttempt = stamps + p.admissions = admissions } } diff --git a/internal/indexer/raw_edge_settled_completion_test.go b/internal/indexer/raw_edge_settled_completion_test.go index 07d948b67..4e472aebd 100644 --- a/internal/indexer/raw_edge_settled_completion_test.go +++ b/internal/indexer/raw_edge_settled_completion_test.go @@ -37,7 +37,7 @@ func TestRawEdgeProofWatcherSupersessionBootstrapsMetadata(t *testing.T) { firstMtime := time.Now().Add(time.Minute) require.NoError(t, os.Chtimes(path, firstMtime, firstMtime)) var applied *IndexResult - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := f.mi.incrementalPointRepoRaw("fixture", p) if err == nil { applied = result @@ -65,7 +65,7 @@ func TestRawEdgeProofWatcherSupersessionBootstrapsMetadata(t *testing.T) { require.True(t, state.Seq > 0 && state.Settled && !state.Writing) require.True(t, idx.rawEdgesPriorCompleted(owner, f.store.GetFileNodes(owner))) - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := f.mi.incrementalPointRepoRaw("fixture", p) applied = result return result, err @@ -103,7 +103,7 @@ func rawEdgeCompletionOvertakenSave(t *testing.T, f *contractJobFixture, w *Watc writeFile(t, path, source) future := time.Now().Add(time.Minute) require.NoError(t, os.Chtimes(path, future, future)) - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := f.mi.incrementalPointRepoRaw("fixture", p) w.mu.Lock() w.pendingGeneration[p] = 8 @@ -179,7 +179,7 @@ func TestRawEdgeProofWatcherFailedTailDoesNotSettle(t *testing.T) { require.NoError(t, os.Chtimes(path, mtime, mtime)) before := f.store.ContractReceiptWriteStates("fixture", "", []string{owner})[owner] called := false - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { called = true result, err := f.mi.incrementalPointRepoRaw("fixture", p) if err != nil { diff --git a/internal/indexer/repo_load_identity_test.go b/internal/indexer/repo_load_identity_test.go new file mode 100644 index 000000000..921b3b374 --- /dev/null +++ b/internal/indexer/repo_load_identity_test.go @@ -0,0 +1,224 @@ +package indexer + +import ( + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/pathkey" +) + +func TestRepoIdentityColdAliasPublication(t *testing.T) { + base := t.TempDir() + physical := filepath.Join(base, "physical-repository") + alias := filepath.Join(base, "different-configured-name") + require.NoError(t, os.MkdirAll(filepath.Join(physical, ".git"), 0755)) + require.NoError(t, os.Symlink(physical, alias)) + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: alias}} + mi.DeclareRepos(entries) + require.True(t, mi.RepoIdentityPending()) + _, known := mi.PendingRepoForPath(physical) + require.False(t, known, "an unresolved declaration cannot invent ownership") + mi.ExpectRepos(entries) + require.False(t, mi.RepoIdentityPending()) + rec, known := mi.PendingRepoForPath(physical) + require.True(t, known) + require.Equal(t, alias, rec.Root) +} + +func TestRepoIdentityDarwinTmpAliasPublication(t *testing.T) { + if runtime.GOOS != "darwin" { + t.Skip("requires Darwin's /tmp -> /private/tmp alias") + } + canonical, err := filepath.EvalSymlinks("/tmp") + require.NoError(t, err) + if canonical != "/private/tmp" { + t.Skip("host does not expose the /tmp -> /private/tmp alias") + } + root, err := os.MkdirTemp("/tmp", "repo-load-alias-") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(root)) }) + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: root}} + mi.DeclareRepos(entries) + physical := filepath.Join(canonical, filepath.Base(root)) + require.True(t, mi.RepoIdentityPending()) + _, known := mi.PendingRepoForPath(physical) + require.False(t, known) + mi.ExpectRepos(entries) + rec, known := mi.PendingRepoForPath(physical) + require.True(t, known) + require.Equal(t, root, rec.Root) +} + +func TestRepoIdentityPublicationDoesNotOverrideRegistrationOrRedeclaration(t *testing.T) { + for _, transition := range []string{"registration", "redeclaration", "running"} { + t.Run(transition, func(t *testing.T) { + root := t.TempDir() + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: root}} + mi.DeclareRepos(entries) + before := mi.RepoLoads()[0] + resolved, err := resolveRepoLoadIdentity(before) + require.NoError(t, err) + switch transition { + case "registration": + mi.markRepoRegistered("ready", root) + case "redeclaration": + mi.DeclareRepos(nil) + mi.DeclareRepos(entries) + case "running": + mi.StartRepoLoad(root) + } + mi.publishRepoLoadIdentity(before, resolved) + after, ok := mi.RepoLoadFor(root) + require.True(t, ok) + switch transition { + case "registration": + require.Equal(t, RepoLoadReady, after.State) + require.False(t, mi.RepoIdentityPending()) + case "redeclaration": + require.True(t, mi.RepoIdentityPending(), "stale owner result must not resolve a replacement declaration") + case "running": + require.Equal(t, RepoLoadRunning, after.State) + require.False(t, mi.RepoIdentityPending()) + } + }) + } +} + +func TestRepoIdentityUncertainFilesystemRetainsNegativeFence(t *testing.T) { + root := t.TempDir() + // A directory where commondir should be a readable file produces a real + // non-ENOENT I/O failure regardless of the test user's permission privileges. + admin := filepath.Join(t.TempDir(), "admin") + require.NoError(t, os.MkdirAll(filepath.Join(admin, "commondir"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(root, ".git"), []byte("gitdir: "+admin), 0644)) + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: root}} + mi.ExpectRepos(entries) + require.True(t, mi.RepoIdentityPending()) + rec, _ := mi.RepoLoadFor(root) + require.Equal(t, RepoLoadQueued, rec.State) + _, err := resolveRepoLoadIdentity(rec) + require.Error(t, err) + require.False(t, os.IsNotExist(err)) + require.NoError(t, os.Remove(filepath.Join(root, ".git"))) + mi.ExpectRepos(entries) + require.False(t, mi.RepoIdentityPending()) +} + +func TestRepoIdentitySeparateGitDirAndLinkedBoundary(t *testing.T) { + primary, linked := t.TempDir(), t.TempDir() + common := filepath.Join(t.TempDir(), "external-git") + admin := filepath.Join(common, "worktrees", "linked") + require.NoError(t, os.MkdirAll(admin, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(primary, ".git"), []byte("gitdir: "+common), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(linked, ".git"), []byte("gitdir: "+admin), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(admin, "commondir"), []byte("../.."), 0644)) + info := ResolveWorktree(primary) + require.False(t, info.IsWorktree) + require.Equal(t, primary, info.MainRepoPath) + require.True(t, pathkey.EqualPaths(common, info.GitCommonDir)) + mi := newRepoLoadTestMulti(t) + mi.ExpectRepos([]config.RepoEntry{{Path: primary}}) + rec, known := mi.PendingRepoForPath(linked) + require.True(t, known) + require.Equal(t, primary, rec.Root) + nested := filepath.Join(linked, "nested") + require.NoError(t, os.MkdirAll(filepath.Join(nested, ".git"), 0755)) + _, known = mi.PendingRepoForPath(nested) + require.False(t, known, "the nearest unrelated git identity is a boundary") +} + +func TestRepoIdentityRegistrationPublishesCanonicalBinding(t *testing.T) { + // RepoLoadFor reads snapshot keys only; the temporary directory may itself + // be an alias, such as Darwin's /var -> /private/var. + physical, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + alias := filepath.Join(t.TempDir(), "different-name") + require.NoError(t, os.Symlink(physical, alias)) + mi := newRepoLoadTestMulti(t) + mi.DeclareRepos([]config.RepoEntry{{Path: alias}}) + before := mi.RepoLoads()[0] + resolved, err := resolveRepoLoadIdentity(before) + require.NoError(t, err) + mi.markRepoRegistered("registered", alias) + mi.publishRepoLoadIdentity(before, resolved) + require.False(t, mi.RepoIdentityPending()) + rec, found := mi.RepoLoadFor(physical) + require.True(t, found, "registration must publish its own proven canonical key even when the warmup publisher loses the race") + require.Equal(t, RepoLoadReady, rec.State) + require.Equal(t, "registered", rec.Prefix) +} + +func TestRepoIdentityRejectsInvalidSeparateGitDir(t *testing.T) { + for _, kind := range []string{"missing", "file"} { + t.Run(kind, func(t *testing.T) { + root := t.TempDir() + target := filepath.Join(t.TempDir(), "not-a-git-directory") + if kind == "file" { + require.NoError(t, os.WriteFile(target, []byte("not a directory"), 0644)) + } + require.NoError(t, os.WriteFile(filepath.Join(root, ".git"), []byte("gitdir: "+target), 0644)) + info := ResolveWorktree(root) + require.Empty(t, info.GitCommonDir) + require.False(t, info.IsWorktree) + require.Equal(t, root, info.MainRepoPath) + mi := newRepoLoadTestMulti(t) + mi.ExpectRepos([]config.RepoEntry{{Path: root}}) + require.True(t, mi.RepoIdentityPending()) + require.Empty(t, enclosingWorktreeCommonDir(root)) + }) + } +} + +func TestRepoIdentityPermissionAndDeviceErrorsAreNotNegativeEvidence(t *testing.T) { + for _, cause := range []error{syscall.EACCES, syscall.EIO} { + t.Run(cause.Error(), func(t *testing.T) { + mi := newRepoLoadTestMulti(t) + root := t.TempDir() + mi.DeclareRepos([]config.RepoEntry{{Path: root}}) + before := mi.RepoLoads()[0] + failure := &os.PathError{Op: "lstat", Path: root, Err: cause} + after, err := resolveRepoLoadIdentityWith(before, func(path string) (string, error) { + require.Equal(t, root, path) + return root, failure // even an absolute fallback is not proof on error + }) + require.ErrorIs(t, err, cause) + require.True(t, after.identityPending) + require.Equal(t, RepoLoadQueued, after.State) + require.Empty(t, after.canonicalKey) + require.True(t, mi.RepoIdentityPending()) + }) + } +} + +func TestRepoIdentityOldPublisherCannotClearNewRegistrationUncertainty(t *testing.T) { + root := t.TempDir() + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: root}} + mi.DeclareRepos(entries) + before := mi.RepoLoads()[0] + resolved, err := resolveRepoLoadIdentity(before) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(root, ".git"), []byte("unreadable identity format"), 0644)) + mi.markRepoRegistered("registered", root) + require.True(t, mi.RepoIdentityPending()) + mi.publishRepoLoadIdentity(before, resolved) + require.True(t, mi.RepoIdentityPending(), "an older successful observation must not replace a newer uncertain registration identity") + rec, found := mi.RepoLoadFor(root) + require.True(t, found) + require.Equal(t, RepoLoadReady, rec.State, "uncertain alias identity does not revoke authoritative graph registration") + require.Contains(t, rec.Reason, "identity resolution pending") + require.NoError(t, os.Remove(filepath.Join(root, ".git"))) + mi.ExpectRepos(entries) + require.False(t, mi.RepoIdentityPending()) + rec, _ = mi.RepoLoadFor(root) + require.Empty(t, rec.Reason) +} diff --git a/internal/indexer/repo_load_publication_test.go b/internal/indexer/repo_load_publication_test.go new file mode 100644 index 000000000..78a51a974 --- /dev/null +++ b/internal/indexer/repo_load_publication_test.go @@ -0,0 +1,66 @@ +package indexer + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" +) + +func TestRepoLoadAdmissionUsesOneSnapshotAcrossPublication(t *testing.T) { + physical := t.TempDir() + alias := filepath.Join(t.TempDir(), "configured-alias") + require.NoError(t, os.Symlink(physical, alias)) + mi := newRepoLoadTestMulti(t) + entries := []config.RepoEntry{{Path: alias}} + mi.DeclareRepos(entries) + before := mi.repoLoads.load() + mi.ExpectRepos(entries) + // Publication after an ownership miss must not combine the old miss with + // the new all-resolved flag and produce a definitive untracked answer. + pending, classified := before.admissionForPath(physical) + require.True(t, classified) + require.Equal(t, RepoLoadIdentityPending, pending.State) + require.Empty(t, pending.Root) + loading, classified := mi.RepoLoadAdmissionForPath(physical) + require.True(t, classified) + require.Equal(t, RepoLoadQueued, loading.State) + require.Equal(t, alias, loading.Root) + mi.markRepoRegistered("ready", alias) + ready, classified := mi.RepoLoadAdmissionForPath(physical) + require.True(t, classified) + require.Equal(t, RepoLoadReady, ready.State, "final admission must expose a registration that raced positive checks") + _, pendingOwnership := mi.PendingRepoForPath(physical) + require.False(t, pendingOwnership, "existing ownership-only API must not reinterpret ready as loading") +} + +func TestRepoLoadAdmissionRetainsReadyLinkedFamilyIdentity(t *testing.T) { + root, linked := linkedWorktreeLayout(t) + mi := newRepoLoadTestMulti(t) + mi.ExpectRepos([]config.RepoEntry{{Path: root}}) + mi.markRepoRegistered("ready", root) + rec, classified := mi.RepoLoadAdmissionForPath(linked) + require.True(t, classified) + require.Equal(t, RepoLoadReady, rec.State) + require.Equal(t, root, rec.Root) + _, pending := mi.PendingRepoForPath(linked) + require.False(t, pending, "a ready family is identity evidence, not a pending ownership verdict") + nested := filepath.Join(linked, "nested") + require.NoError(t, os.MkdirAll(filepath.Join(nested, ".git"), 0755)) + _, classified = mi.RepoLoadAdmissionForPath(nested) + require.False(t, classified, "a different nearest git identity cannot inherit ready family evidence") + // A specific pending family match keeps its existing priority; adding ready + // identity evidence must not displace it merely because ready was recorded first. + loading := t.TempDir() + admin := filepath.Join(root, ".git", "worktrees", "another") + require.NoError(t, os.MkdirAll(admin, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(admin, "commondir"), []byte("../.."), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(loading, ".git"), []byte("gitdir: "+admin), 0644)) + mi.ExpectRepos([]config.RepoEntry{{Path: root}, {Path: loading, Name: "pending-family"}}) + rec, classified = mi.RepoLoadAdmissionForPath(linked) + require.True(t, classified) + require.Equal(t, RepoLoadQueued, rec.State) + require.Equal(t, loading, rec.Root) +} diff --git a/internal/indexer/repo_load_state.go b/internal/indexer/repo_load_state.go index 080b705d4..a38845d69 100644 --- a/internal/indexer/repo_load_state.go +++ b/internal/indexer/repo_load_state.go @@ -1,6 +1,7 @@ package indexer import ( + "errors" "os" "path/filepath" "slices" @@ -27,6 +28,9 @@ import ( type RepoLoadState string const ( + // RepoLoadIdentityPending is an unmatched request while configured root + // identities are unresolved. It names no repository and grants no scope. + RepoLoadIdentityPending RepoLoadState = "identity_pending" // RepoLoadQueued is a configured repository warmup has not started yet. RepoLoadQueued RepoLoadState = "queued" // RepoLoadRunning is a configured repository warmup is indexing or @@ -43,7 +47,7 @@ const ( // Loading reports a state that has not reached a terminal answer yet. func (s RepoLoadState) Loading() bool { - return s == RepoLoadQueued || s == RepoLoadRunning + return s == RepoLoadQueued || s == RepoLoadRunning || s == RepoLoadIdentityPending } // Ready reports a registered repository. @@ -73,10 +77,15 @@ type RepoLoad struct { // never touches the filesystem. key string canonicalKey string - // commonDir is Root's shared git directory, read once when the record is - // created, so a linked worktree of a loading primary is recognised without - // spawning git on the admission path. + // commonDir is the shared git directory resolved off the serving path. + // A declaration uses only root/.git as a loading-only candidate until + // warmup resolves identity; a request must independently resolve its own + // linked-worktree marker before this can produce a pending refusal. commonDir string + // identityPending distinguishes a lexical boot declaration from the + // filesystem identity warmup fills in. It grants no graph admission. + identityPending bool + identityToken *byte } // Label names the repository in user-facing text: its prefix, or its root @@ -104,13 +113,17 @@ type repoLoadSnapshot struct { index map[string]int // loading counts records in a non-terminal state, so the worktree arm // can skip its filesystem walk once warmup has settled. - loading int + loading int + identityPending bool } func newRepoLoadSnapshot(records []RepoLoad) *repoLoadSnapshot { snap := &repoLoadSnapshot{records: records, index: make(map[string]int, 2*len(records))} for i, rec := range records { snap.index[rec.key] = i + if rec.identityPending { + snap.identityPending = true + } if rec.canonicalKey != "" { snap.index[rec.canonicalKey] = i } @@ -156,18 +169,17 @@ func (t *repoLoadTable) update(mutate func(records []RepoLoad) ([]RepoLoad, bool // canonical spelling and git common directory. Called only off the hot path: // when a start announces its repositories and when one registers. func newRepoLoad(root, prefix string, state RepoLoadState, now time.Time) RepoLoad { - rec := RepoLoad{Root: root, Prefix: prefix, State: state, Since: now, key: pathkey.IdentityKey(root)} - if canonical := pathkey.CanonicalExistingRoot(root); canonical != "" { - if key := pathkey.IdentityKey(canonical); key != rec.key { - rec.canonicalKey = key - } - } - if state.Loading() { - if commonDir := ResolveWorktree(root).GitCommonDir; commonDir != "" { - rec.commonDir = pathkey.CanonicalExistingRoot(commonDir) - } + rec := RepoLoad{Root: root, Prefix: prefix, State: state, Since: now, + key: pathkey.IdentityKey(root), identityPending: true, identityToken: new(byte)} + resolved, err := resolveRepoLoadIdentity(rec) + if err != nil { + rec.Reason = "identity resolution pending: " + err.Error() + return rec } - return rec + // Registration is authoritative about ready state even if its directory + // disappeared since indexing. Identity absence is a separate observation. + resolved.State = state + return resolved } // absRepoRoot spells a configured path the way registration does. @@ -183,14 +195,27 @@ func absRepoRoot(path string) string { return pathkey.NormalizeVolume(abs) } +// DeclareRepos publishes configured roots before the daemon listens without +// touching those roots, resolving symlinks or reading git metadata. A hung +// configured mount must not delay the listener or hide the other declarations. +// ExpectRepos fills in filesystem identity and missing-path state during warmup. +func (mi *MultiIndexer) DeclareRepos(entries []config.RepoEntry) { + mi.expectRepos(entries, false) +} + // ExpectRepos records the configured repositories a start is about to load. // -// Called once before checkout seeding, so a configured repository reads as -// loading from the moment the socket serves, and again when warmup takes its -// own snapshot of the config. A repository already recorded keeps its record; +// Called off the serving path before checkout seeding and again when warmup +// takes its own snapshot of the config. DeclareRepos publishes lexical roots +// before the socket serves; this enriches those declarations without resetting +// progress or a registration that raced with warmup. A repository already recorded keeps its record; // one that is no longer configured and not registered is dropped, so a stale // entry can never read as loading forever. func (mi *MultiIndexer) ExpectRepos(entries []config.RepoEntry) { + mi.expectRepos(entries, true) +} + +func (mi *MultiIndexer) expectRepos(entries []config.RepoEntry, inspect bool) { if mi == nil { return } @@ -202,32 +227,125 @@ func (mi *MultiIndexer) ExpectRepos(entries []config.RepoEntry) { var added []RepoLoad for _, entry := range entries { root := absRepoRoot(entry.Path) - if root == "" || addedKeys[pathkey.IdentityKey(root)] { + key := pathkey.IdentityKey(root) + if root == "" || addedKeys[key] { continue } if i, ok := current.find(root); ok { wanted[i] = true continue } - state := RepoLoadQueued - reason := "" - if info, err := os.Stat(root); err != nil || !info.IsDir() { - state, reason = RepoLoadMissing, "the configured path does not exist" - } - rec := newRepoLoad(root, config.ResolvePrefix(entry), state, now) - rec.Reason = reason - added = append(added, rec) - addedKeys[rec.key] = true + added = append(added, RepoLoad{Root: root, Prefix: config.ResolvePrefix(entry), State: RepoLoadQueued, + Since: now, key: key, commonDir: filepath.Join(root, ".git"), identityPending: true, identityToken: new(byte)}) + addedKeys[key] = true } next := make([]RepoLoad, 0, len(records)+len(added)) for i, rec := range records { - if wanted[i] || rec.State == RepoLoadReady { + if wanted[i] || rec.State.Ready() { next = append(next, rec) } } next = append(next, added...) return next, len(added) > 0 || len(next) != len(records) }) + if !inspect { + return + } + // Only the existing warmup owner resolves identities. No filesystem access + // holds the publication lock; each completed root becomes visible before the + // next root is inspected, even if that next mount never answers. + for _, rec := range mi.repoLoads.load().records { + if !rec.identityPending { + continue + } + resolved, err := resolveRepoLoadIdentity(rec) + if err != nil { + // Retain the owner's diagnostic without turning uncertainty into a + // terminal absent-alias result. No request performs this operation again. + mi.repoLoads.update(func(records []RepoLoad) ([]RepoLoad, bool) { + i, ok := newRepoLoadSnapshot(records).find(rec.Root) + if !ok || !records[i].identityPending || records[i].identityToken != rec.identityToken { + return records, false + } + reason := "identity resolution pending: " + err.Error() + if records[i].Reason == reason { + return records, false + } + records[i].Reason = reason + return records, true + }) + continue + } + mi.publishRepoLoadIdentity(rec, resolved) + } +} + +func resolveRepoLoadIdentity(rec RepoLoad) (RepoLoad, error) { + return resolveRepoLoadIdentityWith(rec, filepath.EvalSymlinks) +} + +// The resolver argument lets source-level I/O failures be exercised without +// depending on host privileges or an unavailable device. Production uses only +// the existing warmup owner's synchronous filesystem resolver. +func resolveRepoLoadIdentityWith(rec RepoLoad, canonicalize func(string) (string, error)) (RepoLoad, error) { + canonical, err := canonicalize(rec.Root) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return rec, err + } + rec.State, rec.Reason = RepoLoadMissing, "the configured path does not exist" + } else { + st, statErr := os.Stat(canonical) + if statErr != nil { + return rec, statErr + } + if !st.IsDir() { + rec.State, rec.Reason = RepoLoadMissing, "the configured path is not a directory" + } else { + rec.canonicalKey = pathkey.IdentityKey(pathkey.NormalizeVolume(canonical)) + info, resolveErr := resolveWorktreeChecked(canonical) + if resolveErr != nil { + return rec, resolveErr + } + rec.commonDir = "" + if info.GitCommonDir != "" { + common, commonErr := filepath.EvalSymlinks(info.GitCommonDir) + if commonErr != nil { + return rec, commonErr + } + rec.commonDir = pathkey.NormalizeVolume(common) + } + } + } + rec.identityPending = false + return rec, nil +} + +func (mi *MultiIndexer) publishRepoLoadIdentity(before, resolved RepoLoad) { + mi.repoLoads.update(func(records []RepoLoad) ([]RepoLoad, bool) { + i, ok := newRepoLoadSnapshot(records).find(before.Root) + if !ok || !records[i].identityPending || records[i].identityToken != before.identityToken { + return records, false + } + rec := &records[i] + rec.canonicalKey, rec.commonDir, rec.identityPending = resolved.canonicalKey, resolved.commonDir, false + if strings.HasPrefix(rec.Reason, "identity resolution pending: ") { + rec.Reason = "" + } + if rec.State == RepoLoadQueued && resolved.State == RepoLoadMissing { + rec.State, rec.Reason, rec.Since = resolved.State, resolved.Reason, time.Now() + } + return records, true + }) +} + +// RepoIdentityPending is a cheap negative-admission fence, never proof that a +// particular path belongs to a configured repository. Authoritative positive +// scope/checkout/router answers must take precedence. It remains true after an +// uncertain filesystem outcome until warmup resolves identity or registration +// itself proves ownership. +func (mi *MultiIndexer) RepoIdentityPending() bool { + return mi != nil && mi.repoLoads.load().identityPending } // StartRepoLoad marks a queued repository as being worked on by warmup. @@ -348,20 +466,29 @@ func (mi *MultiIndexer) markRepoRegistered(prefix, root string) { return } now := time.Now() + registered := newRepoLoad(root, prefix, RepoLoadReady, now) var announced RepoLoad var listeners []func(RepoLoad) mi.repoLoads.update(func(records []RepoLoad) ([]RepoLoad, bool) { listeners = slices.Clone(mi.repoLoads.listeners) if i, ok := newRepoLoadSnapshot(records).find(root); ok { records[i].State, records[i].Reason, records[i].Since = RepoLoadReady, "", now - records[i].commonDir = "" + if registered.canonicalKey != "" { + records[i].canonicalKey = registered.canonicalKey + } + records[i].commonDir = registered.commonDir + records[i].identityPending = registered.identityPending + records[i].identityToken = registered.identityToken + if registered.identityPending { + records[i].Reason = registered.Reason + } if prefix != "" { records[i].Prefix = prefix } announced = records[i] return records, true } - announced = newRepoLoad(root, prefix, RepoLoadReady, now) + announced = registered return append(records, announced), true }) if len(listeners) == 0 { @@ -435,35 +562,75 @@ func (mi *MultiIndexer) PendingRepoForPath(path string) (RepoLoad, bool) { if mi == nil || path == "" { return RepoLoad{}, false } - snap := mi.repoLoads.load() + return mi.repoLoads.load().matchForPath(path, false) +} + +// RepoLoadAdmissionForPath classifies a final negative from one immutable +// snapshot. A matched Ready record is identity proof only: the caller must +// recheck ordinary graph/scope authority or retry, never grant access from it. +// An anonymous identity-pending record names no configured repository. +func (mi *MultiIndexer) RepoLoadAdmissionForPath(path string) (RepoLoad, bool) { + if mi == nil || path == "" { + return RepoLoad{}, false + } + return mi.repoLoads.load().admissionForPath(path) +} + +func (s *repoLoadSnapshot) admissionForPath(path string) (RepoLoad, bool) { + if rec, matched := s.matchForPath(path, true); matched { + return rec, true + } + if s.identityPending { + return RepoLoad{State: RepoLoadIdentityPending}, true + } + return RepoLoad{}, false +} + +func (snap *repoLoadSnapshot) matchForPath(path string, includeReady bool) (RepoLoad, bool) { if len(snap.records) == 0 { return RepoLoad{}, false } abs := absRepoRoot(path) if rec, ok := snap.enclosing(abs); ok { - return rec, !rec.State.Ready() + return rec, includeReady || !rec.State.Ready() + } + // Until the owner resolves aliases, an unmatched request cannot prove + // absence. Do not walk its filesystem merely to manufacture a negative. + if snap.identityPending { + if rec, ok := snap.contained(abs, includeReady); ok { + return rec, true + } + return RepoLoad{}, false } if canonical := pathkey.CanonicalPath(abs); canonical != abs { if rec, ok := snap.enclosing(canonical); ok { - return rec, !rec.State.Ready() + return rec, includeReady || !rec.State.Ready() } } - if rec, ok := snap.contained(abs); ok { + if rec, ok := snap.contained(abs, includeReady); ok { return rec, true } - if snap.loading == 0 { + if snap.loading == 0 && !includeReady { return RepoLoad{}, false } commonDir := enclosingWorktreeCommonDir(abs) if commonDir == "" { return RepoLoad{}, false } + var ready RepoLoad + hasReady := false for _, rec := range snap.records { - if rec.State.Loading() && rec.commonDir != "" && pathkey.EqualPaths(rec.commonDir, commonDir) { + if rec.commonDir == "" || !pathkey.EqualPaths(rec.commonDir, commonDir) { + continue + } + if rec.State.Loading() { return rec, true } + if includeReady && rec.State.Ready() && !hasReady { + ready, hasReady = rec, true + } } - return RepoLoad{}, false + return ready, hasReady } // enclosing finds the nearest recorded root at or above path. @@ -480,21 +647,22 @@ func (s *repoLoadSnapshot) enclosing(path string) (RepoLoad, bool) { } } -// contained finds a not-ready recorded root below path — the shape of an +// contained finds a recorded root below path — the shape of an // agent opened at the directory above its repositories. A loading root wins -// over an unavailable one, because retrying is the useful answer while any -// repository under path can still become ready. -func (s *repoLoadSnapshot) contained(path string) (RepoLoad, bool) { +// over an unavailable or ready one, because retrying is useful while any +// repository under path can still become ready. The ownership-only public +// lookup excludes ready records; final admission includes them for rechecking. +func (s *repoLoadSnapshot) contained(path string, includeReady bool) (RepoLoad, bool) { var found RepoLoad ok := false for _, rec := range s.records { - if rec.State.Ready() || pathkey.EqualPaths(rec.Root, path) || !pathkey.HasPathPrefix(rec.Root, path) { + if (!includeReady && rec.State.Ready()) || pathkey.EqualPaths(rec.Root, path) || !pathkey.HasPathPrefix(rec.Root, path) { continue } if rec.State.Loading() { return rec, true } - if !ok { + if !ok || (found.State.Ready() && !rec.State.Ready()) { found, ok = rec, true } } @@ -506,9 +674,16 @@ func (s *repoLoadSnapshot) contained(path string) (RepoLoad, bool) { // path decides: a main checkout or a submodule is not a linked worktree. func enclosingWorktreeCommonDir(path string) string { for dir := filepath.Clean(path); ; { - if _, err := os.Lstat(filepath.Join(dir, ".git")); err == nil { - info := ResolveWorktree(dir) - if !info.IsWorktree || info.GitCommonDir == "" { + _, err := os.Lstat(filepath.Join(dir, ".git")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return "" + } + if err == nil { + info, resolveErr := resolveWorktreeChecked(dir) + if resolveErr != nil { + return "" + } + if info.GitCommonDir == "" { return "" } // Git writes a worktree's gitdir with symlinks resolved while a diff --git a/internal/indexer/repo_load_state_test.go b/internal/indexer/repo_load_state_test.go index cdcc8ee84..e6c146aca 100644 --- a/internal/indexer/repo_load_state_test.go +++ b/internal/indexer/repo_load_state_test.go @@ -3,6 +3,7 @@ package indexer import ( "context" "errors" + "fmt" "os" "path/filepath" "sync/atomic" @@ -250,3 +251,153 @@ func TestPrimaryRegisteredStartsTheCoordinatorsThatWaitedForIt(t *testing.T) { require.Eventually(t, func() bool { return f.lc.hasCoordinator(checkout.CheckoutID) }, 5*time.Second, 20*time.Millisecond, "the primary's registration must start the coordinator that waited for it") } + +// A selected restart cohort must all be retried by registration, rather than +// only the first checkout noticed before the primary's index was registered. +// These are real Git worktrees and durable catalog identities; this test is not +// the separate reference-host warm-restart workload or its latency evidence. +func TestPrimaryRegisteredStartsAll24WaitingCoordinators(t *testing.T) { + f := newLifecycleFixture(t) + defer f.close() + ctx := context.Background() + main := f.gitRepo("await-cohort-main") + const wanted = 24 + for i := range wanted { + f.worktreeOf(main, fmt.Sprintf("await-cohort-%02d", i)) + } + tracked, err := f.lc.Register(ctx, config.RepoEntry{Path: main, Name: "await-cohort-main"}, TrackSourceCLI) + require.NoError(t, err) + require.NoError(t, tracked.CatalogErr) + + f.restart() + require.NoError(t, f.lc.Seed(ctx)) + require.Nil(t, f.mi.GetMetadata(tracked.Prefix)) + checkouts, err := f.catalog.ListCheckouts(ctx, tracked.FamilyID) + require.NoError(t, err) + require.Len(t, checkouts, wanted+1, "the primary and every linked checkout must survive restart") + for _, checkout := range checkouts { + if checkout.CheckoutID == tracked.CheckoutID { + continue + } + // The same startup edge as an activation reaching ensureCoordinator: + // the catalog identity exists, but its primary is not registered yet. + f.lc.ensureCoordinator(ctx, tracked.GraphID, checkout) + require.False(t, f.lc.hasCoordinator(checkout.CheckoutID)) + } + f.lc.awaitingPrimaryMu.Lock() + waiting := len(f.lc.awaitingPrimary[tracked.GraphID]) + f.lc.awaitingPrimaryMu.Unlock() + require.Equal(t, wanted, waiting) + + f.lc.PrimaryRegistered("unrelated-primary") + require.Zero(t, f.lc.LiveCoordinators(tracked.FamilyID), + "another repository's event must not wake this cohort") + registered := make(chan time.Time, 1) + f.mi.OnRepoRegistered(func(rec RepoLoad) { + if rec.Prefix == tracked.Prefix { + registered <- time.Now() + } + f.lc.PrimaryRegistered(rec.Prefix) + }) + _, err = f.mi.TrackRepoCtx(ctx, config.RepoEntry{Path: main, Name: tracked.Prefix}) + require.NoError(t, err) + var registeredAt time.Time + select { + case registeredAt = <-registered: + case <-time.After(2 * time.Second): + t.Fatal("the real registration event did not drive the retry") + } + remaining := time.Until(registeredAt.Add(2 * time.Second)) + require.Positive(t, remaining, "the retry must start within the registration budget") + require.Eventually(t, func() bool { + return f.lc.LiveCoordinators(tracked.FamilyID) == wanted + }, remaining, 5*time.Millisecond, + "all waiting build loops must be live without a janitor sweep") + t.Logf("%d/%d coordinators live %s after primary registration", wanted, wanted, time.Since(registeredAt)) + assert.Empty(t, f.lc.CoordinatorStartFailures()) + f.lc.awaitingPrimaryMu.Lock() + waiting = len(f.lc.awaitingPrimary[tracked.GraphID]) + f.lc.awaitingPrimaryMu.Unlock() + assert.Zero(t, waiting, "successful retries must drain the waiting cohort") +} + +func TestDeclareRepos_DefersFilesystemIdentityUntilWarmup(t *testing.T) { + mi := newRepoLoadTestMulti(t) + missing := filepath.Join(t.TempDir(), "not-mounted") + entries := []config.RepoEntry{{Path: missing, Name: "pending"}, {Path: missing}} + mi.DeclareRepos(entries) + require.Len(t, mi.RepoLoads(), 1) + rec, ok := mi.RepoLoadFor(missing) + require.True(t, ok) + assert.Equal(t, RepoLoadQueued, rec.State, "declaration must not stat a configured root before Listen") + assert.True(t, rec.identityPending) + assert.Empty(t, rec.Reason) + rec, pending := mi.PendingRepoForPath(filepath.Join(missing, "pkg")) + require.True(t, pending) + assert.Equal(t, "pending", rec.Prefix) + + mi.ExpectRepos(entries) + rec, ok = mi.RepoLoadFor(missing) + require.True(t, ok) + assert.Equal(t, RepoLoadMissing, rec.State, "warmup still diagnoses missing paths") + assert.False(t, rec.identityPending) +} + +func TestDeclareRepos_WarmupEnrichmentPreservesRunningState(t *testing.T) { + mi := newRepoLoadTestMulti(t) + root, worktree := linkedWorktreeLayout(t) + entries := []config.RepoEntry{{Path: root}} + mi.DeclareRepos(entries) + _, pending := mi.PendingRepoForPath(filepath.Join(worktree, "pkg")) + require.False(t, pending, "an unresolved declaration cannot name the worktree's family without request filesystem work") + require.True(t, mi.RepoIdentityPending(), "negative startup admission must remain generically retryable until the owner resolves identity") + + mi.StartRepoLoad(root) + before, _ := mi.RepoLoadFor(root) + mi.ExpectRepos(entries) + after, ok := mi.RepoLoadFor(root) + require.True(t, ok) + assert.Equal(t, RepoLoadRunning, after.State) + assert.Equal(t, before.Since, after.Since) + assert.False(t, after.identityPending) + rec, pending := mi.PendingRepoForPath(worktree) + assert.True(t, pending) + assert.Equal(t, root, rec.Root, "the owner now proves the linked worktree's configured family") + + mi.DeclareRepos(nil) + assert.Empty(t, mi.RepoLoads(), "removed declarations must not remain loading forever") +} + +// The daemon invalidates cached scopes and publishes workspace_readiness from +// OnRepoRegistered. A real file-save mutation must not trigger that boot event. +func TestRepoRegistration_FileSaveDoesNotAnnounceAgain(t *testing.T) { + mi := newRepoLoadTestMulti(t) + root := filepath.Join(t.TempDir(), "saved") + writeRepoLoadSource(t, root) + mi.ExpectRepos([]config.RepoEntry{{Path: root}}) + announced := make(chan RepoLoad, 4) + mi.OnRepoRegistered(func(rec RepoLoad) { announced <- rec }) + res, err := mi.TrackRepoCtx(context.Background(), config.RepoEntry{Path: root}) + require.NoError(t, err) + select { + case <-announced: + case <-time.After(time.Second): + t.Fatal("initial cold registration was not announced") + } + before, ok := mi.RepoLoadFor(root) + require.True(t, ok) + require.Equal(t, RepoLoadReady, before.State) + idx := mi.GetIndexer(res.RepoPrefix) + require.NotNil(t, idx) + require.NoError(t, os.WriteFile(filepath.Join(root, "main.go"), []byte("package main\n\nfunc SavedAfterRegistration() {}\n"), 0o644)) + require.NoError(t, idx.IndexFile("main.go")) + after, ok := mi.RepoLoadFor(root) + require.True(t, ok) + assert.Equal(t, before.Since, after.Since, "a save must not create another registration epoch") + assert.Equal(t, before.State, after.State) + select { + case rec := <-announced: + t.Fatalf("file save emitted a boot readiness/scope-invalidation event: %+v", rec) + case <-time.After(100 * time.Millisecond): + } +} diff --git a/internal/indexer/repository_mutation_coordinator.go b/internal/indexer/repository_mutation_coordinator.go index 094e2a3e4..5aca66c5d 100644 --- a/internal/indexer/repository_mutation_coordinator.go +++ b/internal/indexer/repository_mutation_coordinator.go @@ -13,6 +13,7 @@ import ( "slices" "sort" "sync" + "time" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/graph/store_sqlite" @@ -83,6 +84,7 @@ type repositoryMutationOutcome struct { type repositoryMutationWaiter struct { generation uint64 done chan repositoryMutationOutcome + readRepair *repositoryReadRepairRequest } // repositoryMutationCoordinator is the stable single mutation lane for one repository. @@ -107,6 +109,7 @@ type repositoryMutationCoordinator struct { // coordinators leave it nil; tests set it before admitting concurrent work. batchAdmissionHook func() executor repositoryMutationExecutor + readRepairExecutor func([]string, *ReadRepairObservation) (*IndexResult, error) // outputGeneration wraps ONE executed coalesced batch in an // output-generation receipt. The coalescing lane, not the queuing caller, // is the mutation entry point here: many callers merge into one execution, @@ -127,6 +130,7 @@ type repositoryMutationCoordinator struct { completedGeneration uint64 pending repositoryMutationScope waiters []*repositoryMutationWaiter + readRepairPending bool } func newRepositoryMutationCoordinator(executor repositoryMutationExecutor) *repositoryMutationCoordinator { @@ -138,6 +142,14 @@ func newRepositoryMutationCoordinator(executor repositoryMutationExecutor) *repo func (c *repositoryMutationCoordinator) reconcile( ctx context.Context, paths []string, +) (*IndexResult, error) { + return c.reconcileObserved(ctx, paths, nil) +} + +func (c *repositoryMutationCoordinator) reconcileObserved( + ctx context.Context, + paths []string, + readRepair *repositoryReadRepairRequest, ) (*IndexResult, error) { if ctx == nil { ctx = context.Background() @@ -146,6 +158,10 @@ func (c *repositoryMutationCoordinator) reconcile( return nil, err } waiter := &repositoryMutationWaiter{done: make(chan repositoryMutationOutcome, 1)} + if readRepair != nil && readRepair.complete != nil { + request := *readRepair + waiter.readRepair = &request + } c.mu.Lock() if c.closed { c.mu.Unlock() @@ -158,6 +174,7 @@ func (c *repositoryMutationCoordinator) reconcile( c.requestedGeneration++ waiter.generation = c.requestedGeneration c.waiters = append(c.waiters, waiter) + c.readRepairPending = c.readRepairPending || waiter.readRepair != nil if !c.running { c.running = true c.work.Add(1) @@ -204,13 +221,40 @@ func (c *repositoryMutationCoordinator) drain() { paths := c.pending.take() waiters := c.waiters c.waiters = nil + readRepairPending := c.readRepairPending + c.readRepairPending = false executor := c.executor + readRepairExecutor := c.readRepairExecutor outputGeneration := c.outputGeneration authorityRequired := c.authorityRequired c.mu.Unlock() + var observation *ReadRepairObservation + if readRepairPending { + observation = newRepositoryReadRepairObservation(generation, paths, waiters) + } + if observation != nil { + if readRepairExecutor != nil { + executor = func(paths []string) (*IndexResult, error) { + return readRepairExecutor(paths, observation) + } + } + observation.started = time.Now() + observation.beginPhaseAccounting() + } outcome := executeRepositoryMutationUnderAuthority( outputGeneration, authorityRequired, executor, paths) + if observation != nil { + observation.finished = time.Now() + observation.finishPhaseAccounting() + result := outcome.result + observation.executionSucceeded = outcome.err == nil && result != nil && + !result.FullRetrack && len(result.FailedFiles) == 0 && len(result.Errors) == 0 && + result.DeletedFileCount == 0 && result.QuarantinedFiles == 0 && result.SkippedFiles == 0 + if !observation.executionSucceeded { + observation.acceptedBlobSHA = "" + } + } c.lane <- struct{}{} if batchMutationGate != nil { batchMutationGate.RUnlock() @@ -221,6 +265,7 @@ func (c *repositoryMutationCoordinator) drain() { c.completedGeneration = generation } c.mu.Unlock() + completeRepositoryReadRepair(observation, waiters) for _, waiter := range waiters { waiter.done <- outcome } @@ -365,6 +410,10 @@ func (c *repositoryMutationCoordinator) runExclusiveMode( // Batch admission precedes the repository lane. Otherwise a queued writer // can acquire the batch gate while this call owns the lane, then wait for // that lane itself — a gate/lane cycle that prevents either side advancing. + var batchAdmission canonicalAdmissionSpan + if batchMutationGate != nil || c.batchAdmissionHook != nil { + batchAdmission = PublicationStampsFrom(ctx).beginCanonicalAdmission(canonicalBatchAdmission) + } if batchMutationGate != nil { batchMutationGate.RLock() defer batchMutationGate.RUnlock() @@ -372,14 +421,17 @@ func (c *repositoryMutationCoordinator) runExclusiveMode( if hook := c.batchAdmissionHook; hook != nil { hook() } + batchAdmission.end() if err := ctx.Err(); err != nil { return err } + laneTiming := beginCanonicalLaneWait(ctx) select { case <-ctx.Done(): return ctx.Err() case <-c.lane: + laneTiming.canonicalLaneAcquired() } defer func() { c.lane <- struct{}{} }() if err := ctx.Err(); err != nil { @@ -462,6 +514,11 @@ func (mi *MultiIndexer) repositoryMutationCoordinator(repoPrefix string) *reposi coordinator = newRepositoryMutationCoordinator(func(paths []string) (*IndexResult, error) { return mi.incrementalReindexRepoRaw(repoPrefix, paths) }) + coordinator.readRepairExecutor = func(paths []string, observation *ReadRepairObservation) (*IndexResult, error) { + return mi.incrementalReindexRepoRawMode(repoPrefix, paths, incrementalPathMode{ + detectDeletions: true, readRepair: observation, publication: observation.collector, + }) + } // Attach before publishing the slot: every execution through this stable // lane must participate in the owning MultiIndexer's batch transition. coordinator.batchMutationGate = &mi.batchMutationGate @@ -1041,7 +1098,6 @@ const ( // lane worker, which names OutputEntryRepositoryReconcileLane below. OutputEntryDeferredPasses OutputMutationEntry = "indexer.MultiIndexer.withDeferredRepositoryMutation" OutputEntryRepositoryTopology OutputMutationEntry = "indexer.MultiIndexer.coordinateRepositoryTopologyMutation" - OutputEntryIndexMultiRepo OutputMutationEntry = "indexer.MultiIndexer.indexMultiRepo" OutputEntryIndexRepo OutputMutationEntry = "indexer.MultiIndexer.IndexRepo" OutputEntryIncrementalDiscoverRepo OutputMutationEntry = "indexer.MultiIndexer.incrementalDiscoverRepo" OutputEntryGitWatcherFinalize OutputMutationEntry = "indexer.GitWatcher.finalizeReconcile" @@ -1092,7 +1148,6 @@ var outputMutationEntryKinds = map[OutputMutationEntry]OutputGenerationKind{ OutputEntryReresolveFileScoped: OutputGenerationLegacy, OutputEntryDeferredPasses: OutputGenerationLegacy, OutputEntryRepositoryTopology: OutputGenerationLegacy, - OutputEntryIndexMultiRepo: OutputGenerationLegacy, OutputEntryIndexRepo: OutputGenerationLegacy, OutputEntryIncrementalDiscoverRepo: OutputGenerationLegacy, OutputEntryGitWatcherFinalize: OutputGenerationLegacy, diff --git a/internal/indexer/repository_read_repair.go b/internal/indexer/repository_read_repair.go new file mode 100644 index 000000000..2aab2b1ca --- /dev/null +++ b/internal/indexer/repository_read_repair.go @@ -0,0 +1,165 @@ +package indexer + +import ( + "crypto/sha1" // Git blob identity, not a security decision. + "encoding/hex" + "path/filepath" + "strconv" + "time" + "weak" + + "github.com/zzet/gortex/internal/pathkey" +) + +const repositoryReadRepairCauseCap = 8 + +// ReadRepairObserver requests one bounded observation of an already-needed +// repository reconcile. It cannot change its paths, scheduling or ownership. +type ReadRepairObserver struct { + Session string + RequestIDKey string + Complete func(ReadRepairObservation) +} + +// A reader is an admission cause, never the owner of the work performed by +// drain. In particular a session may reuse a wire ID during the same drain. +// The existing waiter generation distinguishes those admissions. +type repositoryReadRepairCause struct { + session string + requestIDKey string + admissionGeneration uint64 +} + +type repositoryReadRepairRequest struct { + session string + requestIDKey string + complete func(ReadRepairObservation) +} + +// ReadRepairObservation is a value copy scoped to one coordinator +// incarnation. Its generation belongs to the executed drain, not any caller. +// This independent executor window is not additive with a receipt's quiet +// wait, acknowledgement, background window or later no-op watcher attempt. +type ReadRepairObservation struct { + generation uint64 + path string + started time.Time + finished time.Time + causes [repositoryReadRepairCauseCap]repositoryReadRepairCause + causeCount int + causesComplete bool + // Execution success alone is not content-publication proof. A later + // accepted-read receipt must still match the pending commit's exact bytes. + executionSucceeded bool + acceptedBlobSHA string + acceptedAt time.Time + contentAmbiguous bool + // Execution supplies the concrete owner; a prefix lookup at completion + // could instead find a replacement. Do not retain a retired Indexer. + owner weak.Pointer[Indexer] + // The mutable carrier exists only during the observed execution and is + // removed before callbacks; accounting retains a value-only frozen copy. + collector *PublicationStamps + accounting ReadRepairPhaseAccounting +} + +// AcceptedFor verifies the executed incarnation and exact accepted Git blob. +// It deliberately says nothing about the original request's B coverage. +func (o ReadRepairObservation) AcceptedFor(owner *Indexer, blobSHA string) bool { + return owner != nil && o.owner.Value() == owner && o.executionSucceeded && + o.causesComplete && !o.contentAmbiguous && blobSHA != "" && o.acceptedBlobSHA == blobSHA && + o.generation != 0 && !o.started.IsZero() && !o.finished.Before(o.started) && + !o.acceptedAt.IsZero() && !o.acceptedAt.Before(o.started) && !o.acceptedAt.After(o.finished) +} + +// Bounds are the actual execution endpoints, independent of receipt lifetime. +func (o ReadRepairObservation) Bounds() (time.Time, time.Time) { return o.started, o.finished } + +// AcceptedAt is the actual applied-and-fresh acceptance, not callback time. +// Keep the first acceptance so repeated same-byte validation cannot move it +// past a later commit and claim a publication that preceded that commit. +func (o ReadRepairObservation) AcceptedAt() time.Time { return o.acceptedAt } + +// Generation identifies the executed drain within its concrete owner. +func (o ReadRepairObservation) Generation() uint64 { return o.generation } + +// Cause returns at most eight observed reader admissions. Reused wire IDs +// remain distinct through the existing admission generation, not a new ID. +func (o ReadRepairObservation) Cause(i int) (session, requestIDKey string, admissionGeneration uint64, ok bool) { + if i < 0 || i >= o.causeCount { + return "", "", 0, false + } + cause := o.causes[i] + return cause.session, cause.requestIDKey, cause.admissionGeneration, true +} + +func (o *ReadRepairObservation) acceptFreshBlob(path, blobSHA string, applied bool) { + if o == nil || !applied || o.contentAmbiguous || path != o.path || blobSHA == "" { + return + } + if o.acceptedBlobSHA != "" && o.acceptedBlobSHA != blobSHA { + o.acceptedBlobSHA = "" + o.acceptedAt = time.Time{} + o.contentAmbiguous = true + return + } + if o.acceptedBlobSHA == blobSHA { + return + } + o.acceptedBlobSHA = blobSHA + o.acceptedAt = time.Now() +} + +// Derive the ledger's Git blob identity from the exact untransformed buffer +// already read by extraction. This must not reopen the source or substitute +// the distinct SHA-256 content-proof identity used by immutable builds. +func readRepairBlobSHA(src []byte) string { + h := sha1.New() + _, _ = h.Write([]byte("blob " + strconv.Itoa(len(src)) + "\x00")) + _, _ = h.Write(src) + return hex.EncodeToString(h.Sum(nil)) +} + +// newRepositoryReadRepairObservation does not read a clock or allocate on the +// ordinary unobserved path. Full/multipath drains deliberately have no scoped +// observation; they cannot prove publication of this one pending edit. +func newRepositoryReadRepairObservation(generation uint64, paths []string, waiters []*repositoryMutationWaiter) *ReadRepairObservation { + if len(paths) != 1 { + return nil + } + var observation *ReadRepairObservation + for _, waiter := range waiters { + request := waiter.readRepair + if request == nil || request.complete == nil { + continue + } + if observation == nil { + observation = &ReadRepairObservation{ + generation: generation, path: pathkey.Normalize(filepath.ToSlash(paths[0])), causesComplete: true, + } + } + if observation.causeCount == len(observation.causes) { + observation.causesComplete = false + continue + } + observation.causes[observation.causeCount] = repositoryReadRepairCause{ + session: request.session, requestIDKey: request.requestIDKey, + admissionGeneration: waiter.generation, + } + observation.causeCount++ + } + return observation +} + +// Call only after releasing coordinator, lane and batch locks. Each consumer +// receives a fresh value; no retained waiter or mutable observation escapes. +func completeRepositoryReadRepair(observation *ReadRepairObservation, waiters []*repositoryMutationWaiter) { + if observation == nil { + return + } + for _, waiter := range waiters { + if request := waiter.readRepair; request != nil && request.complete != nil { + request.complete(*observation) + } + } +} diff --git a/internal/indexer/repository_read_repair_accounting.go b/internal/indexer/repository_read_repair_accounting.go new file mode 100644 index 000000000..f9df4696e --- /dev/null +++ b/internal/indexer/repository_read_repair_accounting.go @@ -0,0 +1,71 @@ +package indexer + +// ReadRepairPhaseAccounting retains only immutable scalar evidence from the +// coordinator's existing executor window. It has no collector, record or owner +// pointer and cannot extend a receipt's lifetime or rewrite earlier snapshots. +type ReadRepairPhaseAccounting struct { + generation uint64 + windowNS int64 + evidence canonicalAttemptEvidence +} + +func (o *ReadRepairObservation) beginPhaseAccounting() { + o.collector = &PublicationStamps{patchLaps: &canonicalPatchLaps{}} + o.collector.startAttemptControlKind(o.started, patchExecutorControl) + o.collector.markAt(PublicationPatchExecutionStarted, o.started) +} + +func (o *ReadRepairObservation) finishPhaseAccounting() { + s := o.collector + if s == nil { + return + } + s.markAt(PublicationPatchExecutionFinished, o.finished) + s.mu.Lock() + s.patchLaps.finishAttemptControlLocked(o.finished) + o.accounting = ReadRepairPhaseAccounting{ + generation: o.generation, windowNS: o.finished.Sub(o.started).Nanoseconds(), + evidence: s.freezeAttemptEvidence(o.started), + } + s.mu.Unlock() + o.collector = nil // no mutable carrier reaches completion callbacks +} + +// PhaseAccounting is independent of the accepted receipt's A, B and no-op P. +// AcceptedFor must still establish actual owner/bytes before ledger retention. +func (o ReadRepairObservation) PhaseAccounting() ReadRepairPhaseAccounting { return o.accounting } + +// Payload allocates caller-owned projection maps only when status is read. +// Missing publisher phases stay null, never inferred from the enclosing wall. +func (a ReadRepairPhaseAccounting) Payload() map[string]any { + w := canonicalAttemptWindow{endNS: a.windowNS, accounting: a.evidence} + p := w.evaluateAccountingFor(true) + phases, states := map[string]any{}, map[string]string{} + for i, name := range canonicalAttemptLapNames { + kind := canonicalPatchLap(i) + if kind >= canonicalDeepLapCount && kind != patchExecutorControl { + continue + } + if kind == patchExecutorControl { + name = "repair_control" + } + phases[name], states[name] = nil, p.states[i] + if p.states[i] == "measured" { + phases[name] = p.values[i] + } + } + out := map[string]any{ + "scope": "read_assisted_reconcile", "clock": "monotonic", + "owner_generation": a.generation, "sum_state": p.state, + "observed_partition_complete": p.complete, + "phases_ms": phases, "phase_states": states, + "additive_with_requested_background": false, + } + if a.evidence.observed { + out["window_ns"] = a.windowNS // actual frozen bounds remain known even for invalid laps + } + if p.state != "not_observed" && p.state != "invalid_order" { + out["window_ns"], out["observed_subtotal_ns"], out["unattributed_ns"] = a.windowNS, p.sumNS, a.windowNS-p.sumNS + } + return out +} diff --git a/internal/indexer/repository_read_repair_accounting_test.go b/internal/indexer/repository_read_repair_accounting_test.go new file mode 100644 index 000000000..f7e848561 --- /dev/null +++ b/internal/indexer/repository_read_repair_accounting_test.go @@ -0,0 +1,118 @@ +package indexer + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/config" +) + +func controlledReadRepairAccounting() ReadRepairObservation { + origin := time.Now() + o := ReadRepairObservation{generation: 3, started: origin} + o.beginPhaseAccounting() + for kind := patchGateWait; kind <= patchDerived; kind++ { + start := origin.Add(time.Duration(1+2*int(kind)) * time.Millisecond) + end := start.Add(time.Millisecond) + lap := o.collector.beginPatchLapAt(kind, start) + lap.endAt(end) + if kind == patchDerived { + o.collector.markAt(PublicationPatchDerivedStarted, start) + o.collector.markAt(PublicationPatchDerivedFinished, end) + } + } + o.finished = origin.Add(20 * time.Millisecond) + o.finishPhaseAccounting() + return o +} + +func TestReadRepairPhaseAccountingUsesOnlyItsOwnOrderedWindow(t *testing.T) { + o := controlledReadRepairAccounting() + require.Nil(t, o.collector, "mutable carrier must not reach completion callbacks") + p := o.PhaseAccounting().Payload() + require.EqualValues(t, 3, p["owner_generation"]) + require.Equal(t, true, p["observed_partition_complete"]) + require.EqualValues(t, 20e6, p["window_ns"]) + require.EqualValues(t, 20e6, p["observed_subtotal_ns"]) + require.EqualValues(t, 0, p["unattributed_ns"]) + require.Equal(t, false, p["additive_with_requested_background"]) + phases := p["phases_ms"].(map[string]any) + require.Equal(t, 13.0, phases["repair_control"]) + for _, name := range canonicalPatchLapNames { + require.Equal(t, 1.0, phases[name], name) + } + for _, name := range []string{"debounce", "cohort_admission", "batch_admission", "lane_wait", "attempt_setup", "attempt_finalize"} { + require.NotContains(t, phases, name, "pre-R or P work cannot be borrowed") + } + phases["apply"] = 999 + require.Equal(t, 1.0, o.PhaseAccounting().Payload()["phases_ms"].(map[string]any)["apply"], "projection maps cannot mutate retained evidence") + copy := o.PhaseAccounting() + copy.evidence.laps[patchApply] = canonicalLapEvidence{} + require.Equal(t, true, o.PhaseAccounting().Payload()["observed_partition_complete"]) +} + +func TestReadRepairPhaseAccountingRefusesMissingContradictoryAndUnboundedLaps(t *testing.T) { + for _, name := range []string{"missing", "foreign_lane", "outside", "overflow", "duplicate", "unobserved", "gap"} { + t.Run(name, func(t *testing.T) { + a := controlledReadRepairAccounting().PhaseAccounting() + want := "invalid_order" + switch name { + case "missing": + a.evidence.laps[patchExecutorControl].totalNS += a.evidence.laps[patchApply].totalNS + a.evidence.laps[patchApply] = canonicalLapEvidence{} + want = "missing_laps" + case "foreign_lane": + a.evidence.laps[patchRepositoryLane].begun = 1 + case "outside": + a.evidence.laps[patchParse].lastNS = a.windowNS + 1 + case "overflow": + a.evidence.laps[patchExecutorControl].totalNS = 1<<63 - 1 + case "duplicate": + a.evidence.nativeMtimeDuplicate = true + case "unobserved": + a.evidence.observed = false + want = "not_observed" + case "gap": + a.evidence.laps[patchExecutorControl].totalNS-- + want = "unclassified_gap" + } + p := a.Payload() + require.Equal(t, false, p["observed_partition_complete"]) + require.Equal(t, want, p["sum_state"]) + }) + } +} + +func TestReadRepairPhaseAccountingRealCoordinatorExecutorAndInertRefusal(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "main.go") + before := "package fixture\nfunc Value() int { return 1 }\n" + writeFile(t, path, before) + mi, _ := newSqliteMultiIndexer(t, []config.RepoEntry{{Path: root, Name: "fixture"}}) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + _, err := mi.TrackRepo(config.RepoEntry{Path: root, Name: "fixture"}) + require.NoError(t, err) + idx := mi.GetIndexer("fixture") + require.NotNil(t, idx) + after := before + "func Added() int { return Value() }\n" + writeFile(t, path, after) + var observation ReadRepairObservation + observer := &ReadRepairObserver{Session: "reader", RequestIDKey: "n:7", Complete: func(o ReadRepairObservation) { observation = o }} + _, err = mi.IncrementalReindexRepoObserved("fixture", []string{path}, observer) + require.NoError(t, err) + require.True(t, observation.AcceptedFor(idx, readRepairBlobSHA([]byte(after)))) + require.False(t, observation.AcceptedFor(&Indexer{}, readRepairBlobSHA([]byte(after)))) + require.False(t, observation.AcceptedFor(idx, readRepairBlobSHA([]byte("newer bytes")))) + p := observation.PhaseAccounting().Payload() + require.Equal(t, true, p["observed_partition_complete"], "%+v", p) + require.EqualValues(t, observation.finished.Sub(observation.started).Nanoseconds(), p["observed_subtotal_ns"]) + prior := observation + _, err = mi.IncrementalReindexRepoObserved("fixture", []string{path}, observer) + require.NoError(t, err) + require.False(t, observation.AcceptedFor(idx, readRepairBlobSHA([]byte(after))), "same-byte inert validation cannot claim earlier publication") + require.Equal(t, p, prior.PhaseAccounting().Payload(), "later execution cannot mutate an earlier scalar snapshot") +} diff --git a/internal/indexer/repository_read_repair_test.go b/internal/indexer/repository_read_repair_test.go new file mode 100644 index 000000000..98ea2681d --- /dev/null +++ b/internal/indexer/repository_read_repair_test.go @@ -0,0 +1,263 @@ +package indexer + +import ( + "context" + "errors" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + "weak" + + "github.com/zzet/gortex/internal/graph" +) + +func TestReadRepairAcceptanceRequiresConcreteOwnerAndBytes(t *testing.T) { + owner := &Indexer{} + replacement := &Indexer{} + now := time.Now() + observation := ReadRepairObservation{ + generation: 2, started: now, acceptedAt: now, finished: now, + causesComplete: true, executionSucceeded: true, + acceptedBlobSHA: readRepairBlobSHA([]byte("accepted")), owner: weak.Make(owner), + } + if !observation.AcceptedFor(owner, readRepairBlobSHA([]byte("accepted"))) { + t.Fatal("matching actual owner/content unavailable") + } + if observation.AcceptedFor(replacement, observation.acceptedBlobSHA) || observation.AcceptedFor(owner, readRepairBlobSHA([]byte("newer"))) { + t.Fatal("replacement owner or newer content borrowed earlier evidence") + } + for _, invalid := range []time.Time{{}, now.Add(-time.Nanosecond), now.Add(time.Nanosecond)} { + copy := observation + copy.acceptedAt = invalid + if copy.AcceptedFor(owner, observation.acceptedBlobSHA) { + t.Fatal("missing or out-of-window acceptance fabricated publication") + } + } + observation.owner = weak.Pointer[Indexer]{} + if observation.AcceptedFor(owner, observation.acceptedBlobSHA) { + t.Fatal("missing owner fabricated an accepted execution") + } + runtime.KeepAlive(owner) + runtime.KeepAlive(replacement) +} + +func TestRepositoryReadRepairUsesGitBlobIdentity(t *testing.T) { + for _, test := range []struct{ content, oid string }{ + {"", "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"}, + {"hello\n", "ce013625030ba8dba906f756967f9e9ca394464a"}, + } { + if got := readRepairBlobSHA([]byte(test.content)); got != test.oid { + t.Fatalf("Git blob identity = %q, want %q", got, test.oid) + } + } +} + +func TestReadRepairInertSameBytesCannotClaimEarlierPublication(t *testing.T) { + owner := &Indexer{} + now := time.Now() + observation := ReadRepairObservation{ + path: "file.go", generation: 2, started: now, finished: now, + causesComplete: true, executionSucceeded: true, owner: weak.Make(owner), + } + // The second reader can validate the exact bytes an earlier unobserved + // operation published. That fresh receipt is still not an apply receipt. + receipt := fileReadReceipt{readRepairBlobSHA: readRepairBlobSHA([]byte("already published"))} + observation.acceptFreshBlob("file.go", receipt.readRepairBlobSHA, receipt.readRepairApplied) + if observation.acceptedBlobSHA != "" || observation.AcceptedFor(owner, receipt.readRepairBlobSHA) { + t.Fatal("inert same-byte validation claimed an earlier publication") + } + receipt.readRepairApplied = true + observation.acceptFreshBlob("file.go", receipt.readRepairBlobSHA, receipt.readRepairApplied) + observation.finished = time.Now() + if !observation.AcceptedFor(owner, receipt.readRepairBlobSHA) { + t.Fatal("actual accepted apply lost its content proof") + } + runtime.KeepAlive(owner) +} + +func TestReadRepairRealBatchDistinguishesInertFromApply(t *testing.T) { + root := t.TempDir() + path := filepath.Join(root, "value.go") + before := "package fixture\nfunc Value() int { return 1 }\n" + writeFile(t, path, before) + idx := newTestIndexer(graph.New()) + idx.SetRepoPrefix("fixture") + t.Cleanup(idx.Close) + if _, err := idx.Index(root); err != nil { + t.Fatal(err) + } + // Exercise the real probe/inert/fresh-receipt path rather than merely + // checking an aggregate result count after a no-op mtime fast path. + inert := &ReadRepairObservation{path: "value.go"} + plan, _, failed, raced := idx.reindexIncrementalFilesBatched([]string{path}, nil, &reparsePendingEnrichmentBatch{readRepair: inert}, false) + if plan.InertFiles != 1 || len(failed) != 0 || len(raced) != 0 || inert.acceptedBlobSHA != "" { + t.Fatalf("inert batch claimed publication: plan=%+v failed=%v raced=%v proof=%q", plan, failed, raced, inert.acceptedBlobSHA) + } + after := before + "func Added() int { return Value() }\n" + writeFile(t, path, after) + applied := &ReadRepairObservation{path: "value.go"} + plan, _, failed, raced = idx.reindexIncrementalFilesBatched([]string{path}, nil, &reparsePendingEnrichmentBatch{readRepair: applied}, false) + if plan.InertFiles != 0 || len(failed) != 0 || len(raced) != 0 || applied.acceptedBlobSHA != readRepairBlobSHA([]byte(after)) { + t.Fatalf("accepted apply lost its own bytes: plan=%+v failed=%v raced=%v proof=%q", plan, failed, raced, applied.acceptedBlobSHA) + } +} + +func TestRepositoryReadRepairAcceptedContentCannotChangeOwners(t *testing.T) { + observation := &ReadRepairObservation{path: "target.go"} + older := readRepairBlobSHA([]byte("old bytes")) + newer := readRepairBlobSHA([]byte("new bytes")) + observation.acceptFreshBlob("other.go", older, true) + if observation.acceptedBlobSHA != "" { + t.Fatal("another path supplied target content") + } + observation.acceptFreshBlob("target.go", older, true) + immutable := *observation + observation.acceptFreshBlob("target.go", older, true) + if !observation.acceptedAt.Equal(immutable.acceptedAt) { + t.Fatal("same bytes replaced their actual first publication time") + } + observation.acceptFreshBlob("target.go", newer, true) + observation.acceptFreshBlob("target.go", older, true) + if observation.acceptedBlobSHA != "" || !observation.contentAmbiguous || immutable.acceptedBlobSHA != older { + t.Fatal("ambiguous content regained authority or changed an earlier snapshot") + } +} + +func TestRepositoryReadRepairCausesKeepReusedWireIDsDistinct(t *testing.T) { + request := &repositoryReadRepairRequest{session: "session", requestIDKey: "n:7", complete: func(ReadRepairObservation) {}} + waiters := []*repositoryMutationWaiter{ + {generation: 4, readRepair: request}, + {generation: 5, readRepair: request}, + } + observation := newRepositoryReadRepairObservation(5, []string{"file.go"}, waiters) + if observation == nil || observation.generation != 5 || observation.causeCount != 2 || !observation.causesComplete { + t.Fatalf("missing drain/admission observations: %+v", observation) + } + if observation.causes[0].admissionGeneration != 4 || observation.causes[1].admissionGeneration != 5 { + t.Fatalf("reused wire IDs aliased admissions: %+v", observation.causes) + } +} + +func TestReadRepairPathUsesCanonicalRepositoryKey(t *testing.T) { + observation := newRepositoryReadRepairObservation(1, []string{filepath.FromSlash("dir/value.go")}, []*repositoryMutationWaiter{ + {generation: 1, readRepair: &repositoryReadRepairRequest{complete: func(ReadRepairObservation) {}}}, + }) + if observation == nil || observation.path != "dir/value.go" { + t.Fatalf("native scope spelling diverged from receipt key: %+v", observation) + } +} + +func TestRepositoryReadRepairUnobservedPathDoesNotAllocate(t *testing.T) { + paths := []string{"file.go"} + waiters := []*repositoryMutationWaiter{{generation: 1}} + if allocations := testing.AllocsPerRun(100, func() { + if newRepositoryReadRepairObservation(1, paths, waiters) != nil { + panic("unobserved request acquired evidence") + } + }); allocations != 0 { + t.Fatalf("unobserved allocations: %v", allocations) + } +} + +func TestRepositoryReadRepairCauseOverflowAndBroadScopesRemainUnavailable(t *testing.T) { + request := &repositoryReadRepairRequest{complete: func(ReadRepairObservation) {}} + waiters := make([]*repositoryMutationWaiter, repositoryReadRepairCauseCap+1) + for i := range waiters { + waiters[i] = &repositoryMutationWaiter{generation: uint64(i + 1), readRepair: request} + } + observation := newRepositoryReadRepairObservation(uint64(len(waiters)), []string{"file.go"}, waiters) + if observation == nil || observation.causesComplete || observation.causeCount != repositoryReadRepairCauseCap { + t.Fatalf("overflow incorrectly complete: %+v", observation) + } + for _, paths := range [][]string{nil, {"a.go", "b.go"}} { + if newRepositoryReadRepairObservation(1, paths, waiters) != nil { + t.Fatalf("broad scope acquired single-file evidence: %v", paths) + } + } +} + +func TestRepositoryReadRepairCompletionIsOutsideCoordinatorLocks(t *testing.T) { + for _, failed := range []bool{false, true} { + t.Run(map[bool]string{false: "success", true: "failure"}[failed], func(t *testing.T) { + coordinator := newRepositoryMutationCoordinator(func([]string) (*IndexResult, error) { + if failed { + return nil, errors.New("failed execution") + } + return &IndexResult{StaleFileCount: 1}, nil + }) + var batchGate sync.RWMutex + coordinator.batchMutationGate = &batchGate + var snapshot ReadRepairObservation + _, err := coordinator.reconcileObserved(t.Context(), []string{"file.go"}, &repositoryReadRepairRequest{ + complete: func(observation ReadRepairObservation) { + // Both operations would deadlock if completion still owned + // coordinator.mu or the repository mutation lane. + if !coordinator.mu.TryLock() { + t.Error("completion still owns coordinator lock") + return + } + coordinator.mu.Unlock() + if !batchGate.TryLock() { + t.Error("completion still owns batch lock") + return + } + batchGate.Unlock() + if laneErr := coordinator.runExclusive(context.Background(), func() error { return nil }); laneErr != nil { + panic(laneErr) + } + snapshot = observation + observation.causes[0].session = "caller mutation" + }, + }) + coordinator.work.Wait() + if (err != nil) != failed || snapshot.executionSucceeded == failed || snapshot.started.IsZero() || snapshot.finished.Before(snapshot.started) { + t.Fatalf("execution window mismatch: failed=%v err=%v observation=%+v", failed, err, snapshot) + } + if snapshot.causes[0].session != "" { + t.Fatal("mutable completion value escaped") + } + }) + } +} + +func TestRepositoryReadRepairContentRequiresSuccessfulAuthorityAndResult(t *testing.T) { + for _, test := range []struct { + name string + result *IndexResult + authority bool + }{ + {name: "missing_result"}, + {name: "full", result: &IndexResult{FullRetrack: true}}, + {name: "failed", result: &IndexResult{FailedFiles: []string{"file.go"}}}, + {name: "deleted", result: &IndexResult{DeletedFileCount: 1}}, + {name: "quarantined", result: &IndexResult{QuarantinedFiles: 1}}, + {name: "skipped", result: &IndexResult{SkippedFiles: 1}}, + {name: "authority_refusal", result: &IndexResult{StaleFileCount: 1}, authority: true}, + } { + t.Run(test.name, func(t *testing.T) { + coordinator := newRepositoryMutationCoordinator(nil) + coordinator.readRepairExecutor = func(_ []string, observation *ReadRepairObservation) (*IndexResult, error) { + observation.acceptFreshBlob("file.go", readRepairBlobSHA([]byte("accepted bytes")), true) + return test.result, nil + } + if test.authority { + coordinator.outputGeneration = func(fn func(func(*OutputSourceContent)) error) error { + if err := fn(func(*OutputSourceContent) {}); err != nil { + return err + } + return errors.New("superseded authority") + } + } + var snapshot ReadRepairObservation + _, _ = coordinator.reconcileObserved(t.Context(), []string{"file.go"}, &repositoryReadRepairRequest{ + complete: func(observation ReadRepairObservation) { snapshot = observation }, + }) + coordinator.work.Wait() + if snapshot.executionSucceeded || snapshot.acceptedBlobSHA != "" { + t.Fatalf("unsuccessful publisher retained content authority: %+v", snapshot) + } + }) + } +} diff --git a/internal/indexer/scope_for_cwd_test.go b/internal/indexer/scope_for_cwd_test.go index f77cfa75f..c5e32f5f3 100644 --- a/internal/indexer/scope_for_cwd_test.go +++ b/internal/indexer/scope_for_cwd_test.go @@ -49,7 +49,7 @@ func TestScopeForCWD_And_ReposInWorkspace(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") // empty scope → index every configured repo + _, err = mi.indexFixtureScope("", "") // empty scope → index every configured repo require.NoError(t, err) // cwd inside repo-a → workspace "alpha", home repo "repo-a". @@ -236,7 +236,7 @@ func TestScopeForCWD_WorkspaceRoot(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") + _, err = mi.indexFixtureScope("", "") require.NoError(t, err) // Workspace root above exactly one tracked repo: binds with the @@ -321,7 +321,7 @@ func TestContainedReposScope(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") + _, err = mi.indexFixtureScope("", "") require.NoError(t, err) // ScopeForCWD still fails closed on the mixed parent — this change @@ -403,7 +403,7 @@ func TestContainedReposScope_RefusesOverbroadRoots(t *testing.T) { g := graph.New() mi := NewMultiIndexer(g, newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") + _, err = mi.indexFixtureScope("", "") require.NoError(t, err) // Precondition: a genuine parent inside home still binds, so the @@ -455,7 +455,7 @@ func TestScopeForCWD_RefusesOverbroadRootSharedWorkspace(t *testing.T) { require.NoError(t, err) mi := NewMultiIndexer(graph.New(), newTestRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") + _, err = mi.indexFixtureScope("", "") require.NoError(t, err) // Precondition: the shared slug makes the reverse-containment arm diff --git a/internal/indexer/vector_multirepo_test.go b/internal/indexer/vector_multirepo_test.go index c40a81da6..c1484030f 100644 --- a/internal/indexer/vector_multirepo_test.go +++ b/internal/indexer/vector_multirepo_test.go @@ -39,7 +39,7 @@ func TestIndexMultiRepoPublishesCombinedDurableVectorCorpus(t *testing.T) { mi := NewMultiIndexer(store, coldOrchestrationRegistry(), sw, nil, zap.NewNop()) mi.SetEmbedder(&poolEmbedder{}) - results, err := mi.indexMultiRepo(repos) + results, err := mi.indexFixtureRepos(repos) require.NoError(t, err) require.Len(t, results, len(repos)) stats, err := store.VectorCorpusStats(context.Background(), 3) @@ -64,7 +64,7 @@ func TestUntrackRepoRemovesVectorsAndRepublishesAggregateStats(t *testing.T) { sw := search.NewSwappable(initialSearchBackend(store)) mi := NewMultiIndexer(store, coldOrchestrationRegistry(), sw, cm, zap.NewNop()) mi.SetEmbedder(&poolEmbedder{}) - _, err = mi.indexMultiRepo(repos) + _, err = mi.indexFixtureRepos(repos) require.NoError(t, err) prefixA := config.ResolvePrefix(repos[0]) diff --git a/internal/indexer/vector_warm_route_test.go b/internal/indexer/vector_warm_route_test.go index 8fd12b506..fe9a7fb80 100644 --- a/internal/indexer/vector_warm_route_test.go +++ b/internal/indexer/vector_warm_route_test.go @@ -36,7 +36,7 @@ func TestWarmScopedReconcileRestoresDurableVectorCorpus(t *testing.T) { seed := NewMultiIndexer(graph.Store(store), newTestRegistry(), seedSw, cm, zap.NewNop()) seedEmb := &poolEmbedder{} seed.SetEmbedder(seedEmb) - results, err := seed.indexMultiRepo([]config.RepoEntry{entry}) + results, err := seed.indexFixtureRepos([]config.RepoEntry{entry}) require.NoError(t, err) require.Len(t, results, 1) require.Greater(t, seedEmb.calls, int32(0)) diff --git a/internal/indexer/watcher.go b/internal/indexer/watcher.go index e6fe70d49..58c098460 100644 --- a/internal/indexer/watcher.go +++ b/internal/indexer/watcher.go @@ -61,6 +61,21 @@ type MutationTicket struct { Path string Generation uint64 Done <-chan MutationResult + // PublicationRecord is optional, immutable admission metadata. Native + // events have none. The watcher binds it before waking work; the existing + // consumer of Done records completion, never a second channel reader. + PublicationRecord *PublicationPhaseRecord `json:"-"` +} + +// mutationWaiter owns only outstanding ticket state. Publication metadata is +// released with the same completion/stop paths as its existing result channel; +// it is not a separate tracing registry or completed-history collection. +type mutationWaiter struct { + done chan MutationResult + publication *PublicationPhaseRecord + schedulingEntered time.Time + backgroundEligible bool + timerObservation *canonicalPointTimer } // MutationResult is the terminal graph-freshness result for a ticket. @@ -161,19 +176,20 @@ type Watcher struct { excludes *excludes.Matcher // currentExcludePatterns is installed by MultiWatcher for its configured // repository. ConfigManager returns immutable, shared policy snapshots. - currentExcludePatterns func() []string - currentIncludePatterns func() []string - excludePolicyIncludes []string - excludePolicyMu sync.Mutex - excludePolicyPatterns []string - excludePolicyMatcher *excludes.Matcher - events chan GraphChangeEvent - history []GraphChangeEvent - historyMu sync.Mutex - pending map[string]*time.Timer - pendingGeneration map[string]uint64 - mutationWaiters map[string]map[uint64]chan MutationResult - nextGeneration uint64 + currentExcludePatterns func() []string + currentIncludePatterns func() []string + excludePolicyIncludes []string + excludePolicyMu sync.Mutex + excludePolicyPatterns []string + excludePolicyMatcher *excludes.Matcher + events chan GraphChangeEvent + history []GraphChangeEvent + historyMu sync.Mutex + pending map[string]*time.Timer + pendingGeneration map[string]uint64 + mutationWaiters map[string]map[uint64]mutationWaiter + observedMutationGeneration map[string]uint64 // lazy index into outstanding waiters, not timing history + nextGeneration uint64 // mutationRetryDelayFn and pointMutationPatch are deterministic test seams. // Production retries use mutationRetryBackoff and the complete point pipeline. mutationRetryDelayFn func(int) time.Duration @@ -246,7 +262,7 @@ type Watcher struct { discoverReindex watcherBatchReindex // additive directory discovery with the same complete tail // pointReindexRaw runs the complete exact-path pipeline after the watcher // already owns the repository lane. MultiWatcher installs its raw tail here. - pointReindexRaw func(string) (*IndexResult, error) + pointReindexRaw func(string, *PublicationStamps) (*IndexResult, error) // poller is the adaptive-interval fallback that re-checks git // HEAD movement and tracked-file mtimes on a timer, catching the @@ -1780,6 +1796,7 @@ func (w *Watcher) handleEvent(event fswatcher.WatchEvent) { // resolves when this generation or a newer coalesced generation reaches a // terminal graph state. func (w *Watcher) EnqueueFileMutation(ctx context.Context, filePath string) (*MutationTicket, error) { + ctx, schedulingEntered := canonicalSchedulingEntry(ctx) if err := ctx.Err(); err != nil { return nil, err } @@ -1809,7 +1826,7 @@ func (w *Watcher) EnqueueFileMutation(ctx context.Context, filePath string) (*Mu if w.mutationBeforeAdmission != nil { w.mutationBeforeAdmission() } - ticket := w.scheduleFileMutation(absPath, ChangeModified) + ticket := w.scheduleFileMutationEntered(absPath, ChangeModified, PublicationRecordFromContext(ctx), schedulingEntered) if ticket == nil { return nil, errWatcherStopped } @@ -1883,7 +1900,7 @@ func (w *Watcher) EnqueueFileMutations(ctx context.Context, files []string) (map w.pendingGeneration = make(map[string]uint64) } if w.mutationWaiters == nil { - w.mutationWaiters = make(map[string]map[uint64]chan MutationResult) + w.mutationWaiters = make(map[string]map[uint64]mutationWaiter) } tickets := make(map[string]*MutationTicket, len(paths)) for path := range paths { @@ -1896,10 +1913,10 @@ func (w *Watcher) EnqueueFileMutations(ctx context.Context, files []string) (map w.nextGeneration++ generation := w.nextGeneration if w.mutationWaiters[path] == nil { - w.mutationWaiters[path] = make(map[uint64]chan MutationResult) + w.mutationWaiters[path] = make(map[uint64]mutationWaiter) } done := make(chan MutationResult, 1) - w.mutationWaiters[path][generation] = done + w.mutationWaiters[path][generation] = mutationWaiter{done: done} w.pendingGeneration[path] = generation w.stormGenerations[path] = generation w.stormBatch[path] = ChangeModified @@ -1918,13 +1935,53 @@ func (w *Watcher) EnqueueFileMutations(ctx context.Context, files []string) (map // for the same path; every earlier ticket stays attached until the newest patch // produces a terminal graph event. func (w *Watcher) scheduleFileMutation(path string, kind ChangeKind) *MutationTicket { + return w.scheduleFileMutationWithRecord(path, kind, nil) +} + +func (w *Watcher) scheduleFileMutationWithRecord(path string, kind ChangeKind, record *PublicationPhaseRecord) *MutationTicket { + var entered time.Time + if record != nil { + entered = time.Now() + } + return w.scheduleFileMutationEntered(path, kind, record, entered) +} + +func (w *Watcher) scheduleFileMutationEntered(path string, kind ChangeKind, record *PublicationPhaseRecord, entered time.Time) *MutationTicket { w.mu.Lock() if w.stopping { w.mu.Unlock() return nil } + _, hadPending := w.pending[path] + backgroundEligible := record != nil && !hadPending && w.observedMutationGeneration[path] == 0 + var admissionPredecessor uint64 + admissionStopFailed := false + var successorBackground *canonicalBackgroundOwner + var previousBackground *canonicalBackgroundOwner if timer, exists := w.pending[path]; exists { - if timer.Stop() { + previous := w.mutationWaiters[path][w.pendingGeneration[path]].timerObservation + var stopStarted time.Time + if previous != nil && previous.background != nil { + previousBackground = previous.background + stopStarted = time.Now() + } + stopped := timer.Stop() + if record != nil && w.observedMutationGeneration[path] == 0 { + predecessor := w.pendingGeneration[path] + prior := w.mutationWaiters[path][predecessor] + if predecessor != 0 && prior.done != nil && prior.publication == nil && prior.timerObservation == nil { + if stopped { + backgroundEligible = true + admissionPredecessor = predecessor + } else { + admissionStopFailed = true + } + } + } + if !stopStarted.IsZero() { + successorBackground = previous.background.stopped(w, path, w.pendingGeneration[path], previous.successor, stopStarted, time.Now(), stopped, record == nil) + } + if stopped { w.asyncWork.Done() } } @@ -1934,35 +1991,128 @@ func (w *Watcher) scheduleFileMutation(path string, kind ChangeKind) *MutationTi w.pendingGeneration = make(map[string]uint64) } if w.mutationWaiters == nil { - w.mutationWaiters = make(map[string]map[uint64]chan MutationResult) + w.mutationWaiters = make(map[string]map[uint64]mutationWaiter) } if w.mutationWaiters[path] == nil { - w.mutationWaiters[path] = make(map[uint64]chan MutationResult) + w.mutationWaiters[path] = make(map[uint64]mutationWaiter) } done := make(chan MutationResult, 1) - w.mutationWaiters[path][generation] = done w.pendingGeneration[path] = generation - w.armPointMutationTimerLocked(path, kind, generation, 0, time.Duration(w.config.DebounceMs)*time.Millisecond) + ticket := &MutationTicket{Path: path, Generation: generation, Done: done} + if record != nil && record.bindCanonicalTicket(generation, time.Now()) { + ticket.PublicationRecord = record + if w.observedMutationGeneration == nil { + w.observedMutationGeneration = make(map[string]uint64) + } + w.observedMutationGeneration[path] = generation + } + w.mutationWaiters[path][generation] = mutationWaiter{done: done, publication: ticket.PublicationRecord, schedulingEntered: entered, backgroundEligible: backgroundEligible} + armedBackground := w.armPointMutationTimerLocked(path, kind, generation, 0, time.Duration(w.config.DebounceMs)*time.Millisecond, successorBackground, admissionPredecessor, admissionStopFailed) w.mu.Unlock() - return &MutationTicket{Path: path, Generation: generation, Done: done} + if previousBackground != nil { + previousBackground.publishUnavailable() + } + if armedBackground != nil && armedBackground != previousBackground { + armedBackground.publishUnavailable() + } + return ticket } -func (w *Watcher) armPointMutationTimerLocked(path string, kind ChangeKind, generation uint64, retryAttempt int, delay time.Duration) { +func (w *Watcher) armPointMutationTimerLocked(path string, kind ChangeKind, generation uint64, retryAttempt int, delay time.Duration, background *canonicalBackgroundOwner, admissionPredecessor uint64, admissionStopFailed bool) *canonicalBackgroundOwner { var timer *time.Timer + var observation *canonicalPointTimer + if w.mutationWaiters[path][generation].publication != nil { + observation = &canonicalPointTimer{} + if retryAttempt == 0 { + observation.debounce.started = time.Now() + } + } else if requested := w.observedMutationGeneration[path]; requested != 0 && requested < generation { + // Observe this callback, not the predecessor's quiet wait. Native storm + // waiters do not require a scan to establish that no observer exists. + observation = &canonicalPointTimer{} + } + if observation != nil { + waiter := w.mutationWaiters[path][generation] + if retryAttempt == 0 && waiter.publication != nil { + background = newCanonicalBackground(waiter.publication, w, path, generation, waiter.schedulingEntered) + if background != nil { + background.admissionPredecessor = admissionPredecessor + if !waiter.backgroundEligible { + reason := "preexisting_path_work" + if admissionStopFailed { + reason = "preexisting_timer_stop_not_successful" + } + background.invalidate(reason) + } + } + } + observation.background, observation.successor = background, waiter.publication == nil + if background != nil { + armStarted := observation.debounce.started + if armStarted.IsZero() { + armStarted = time.Now() + } + background.armStarted(w, path, generation, observation.successor, armStarted) + if observation.successor && retryAttempt == 0 { + observation.debounce.started = armStarted // same actual native arm reading, never predecessor quiet + } + } + waiter.timerObservation = observation + w.mutationWaiters[path][generation] = waiter + } w.asyncWork.Add(1) timer = time.AfterFunc(delay, func() { + if observation != nil { + observation.entered = time.Now() // callback entry precedes claim/admission + if observation.background != nil { + observation.background.callback(w, path, generation, observation.successor, observation.entered) + } + if !observation.debounce.started.IsZero() { + observation.debounce.entered = observation.entered + } + } defer w.asyncWork.Done() // Timer.Stop can lose a race with a callback that is already queued. // Only the newest timer may consume the pending entry. if !w.claimPendingTimer(path, &timer) { + if observation != nil && observation.background != nil { + observation.background.invalidate("timer_claim_superseded") + observation.background.publishUnavailable() + } return } - w.runPointMutation(path, kind, generation, retryAttempt) + publication := w.runPointMutationObserved(path, kind, generation, retryAttempt, observation) + if publication != nil && observation != nil { + // The mutation returned after its existing completion and cohort-release + // defers. Timing publication does not move Done or resource ownership. + finished := time.Now() + publication.finishPhysicalAttempt(observation.entered, finished, retryAttempt, w.logger) + if observation.background != nil { + observation.background.finish(publication, observation.entered, finished, retryAttempt) + } + } else if observation != nil && observation.background != nil { + observation.background.invalidate("serving_attempt_unavailable") + observation.background.publishUnavailable() + } }) w.pending[path] = timer + if observation != nil && observation.background != nil { + observation.background.armFinished(w, path, generation, observation.successor, time.Now()) + return observation.background + } + return nil } func (w *Watcher) runPointMutation(path string, kind ChangeKind, generation uint64, retryAttempt int) { + w.runPointMutationObserved(path, kind, generation, retryAttempt, nil) +} + +func (w *Watcher) runPointMutationObserved(path string, kind ChangeKind, generation uint64, retryAttempt int, observation *canonicalPointTimer) (publication *pointPublicationAttempt) { + publication = w.beginPointPublicationObserved(path, generation, retryAttempt, observation) + if publication != nil && observation != nil { + publication.stamps.debounce = observation.debounce // immutable callback-owned pair, exact generation only + publication.stamps.startAttemptControl(observation.entered) + } if w.pointMutationClaimed != nil { w.pointMutationClaimed(path) } @@ -1971,7 +2121,12 @@ func (w *Watcher) runPointMutation(path string, kind ChangeKind, generation uint // otherwise puts every path into the patch path at once, contending for the // repository mutation lane. Admission pressure retains this generation and // its waiters instead of turning a bounded wait into a terminal receipt. + var cohortAdmission canonicalAdmissionSpan + if publication != nil { + cohortAdmission = publication.stamps.beginCanonicalAdmission(canonicalCohortAdmission) + } release, admissionErr := w.admitMutationWork(path) + cohortAdmission.end() if admissionErr != nil { if errors.Is(admissionErr, errWatcherStopped) { w.completeMutationWaitersIfCurrent(path, generation, errWatcherStopped) @@ -2015,13 +2170,13 @@ func (w *Watcher) runPointMutation(path string, kind ChangeKind, generation uint if w.mutationBeforeComplete != nil { w.mutationBeforeComplete(path, generation) } - w.completeMutationWaitersWithFanout(path, generation, patchErr, fanout) + w.completeMutationWaitersObserved(path, generation, patchErr, fanout, true, publication) } }() if w.pointMutationPatch != nil { patchErr = w.pointMutationPatch(path, kind, generation) } else { - patchErr = w.patchGraphObservingFanout(path, kind, generation, &fanout) + patchErr = w.patchGraphWithReceiptStateObserved(path, kind, generation, false, &fanout, publication) } var storageErr *store_sqlite.StorageError if errors.As(patchErr, &storageErr) { @@ -2051,11 +2206,18 @@ func (w *Watcher) runPointMutation(path string, kind ChangeKind, generation uint patchErr = errMutationSuperseded complete = false } + return } func (w *Watcher) schedulePointMutationRetry(path string, kind ChangeKind, generation uint64, attempt int) bool { w.mu.Lock() - defer w.mu.Unlock() + var invalidated *canonicalBackgroundOwner + defer func() { + w.mu.Unlock() + if invalidated != nil { + invalidated.publishUnavailable() + } + }() if w.stopping || w.pendingGeneration[path] != generation { return false } @@ -2064,7 +2226,11 @@ func (w *Watcher) schedulePointMutationRetry(path string, kind ChangeKind, gener if _, exists := w.pending[path]; exists { return true } - w.armPointMutationTimerLocked(path, kind, generation, attempt, w.mutationRetryDelay(attempt)) + if observation := w.mutationWaiters[path][generation].timerObservation; observation != nil && observation.background != nil { + invalidated = observation.background + invalidated.invalidate("retry_or_backoff") + } + w.armPointMutationTimerLocked(path, kind, generation, attempt, w.mutationRetryDelay(attempt), nil, 0, false) return true } @@ -2078,29 +2244,28 @@ func (w *Watcher) completeMutationWaitersIfCurrent(path string, appliedGeneratio return w.completeMutationWaitersGuarded(path, appliedGeneration, err, DerivedFanoutCompleteness{}, true) } -// completeMutationWaitersWithFanout is the arm that actually ran the graph -// patch, so it is the only one that has a derived-fan-out observation to -// publish. Every other completion arm reports the zero value, which reads as -// "not observed" rather than as a complete fan-out. -func (w *Watcher) completeMutationWaitersWithFanout( +func (w *Watcher) completeMutationWaitersGuarded( path string, appliedGeneration uint64, err error, fanout DerivedFanoutCompleteness, + requireCurrent bool, ) bool { - return w.completeMutationWaitersGuarded(path, appliedGeneration, err, fanout, true) + return w.completeMutationWaitersObserved(path, appliedGeneration, err, fanout, requireCurrent, nil) } -func (w *Watcher) completeMutationWaitersGuarded( +func (w *Watcher) completeMutationWaitersObserved( path string, appliedGeneration uint64, err error, fanout DerivedFanoutCompleteness, requireCurrent bool, + publication *pointPublicationAttempt, ) bool { type completion struct { - requested uint64 - done chan MutationResult + requested uint64 + done chan MutationResult + publication *PublicationPhaseRecord } // One choke point for every completion arm, present and future: a mutation // that reports an error never publishes a derived-fan-out verdict. Reindexed @@ -2115,20 +2280,44 @@ func (w *Watcher) completeMutationWaitersGuarded( w.mu.Unlock() return false } + _, hasAppliedWaiter := w.mutationWaiters[path][appliedGeneration] + // The successful executor's finishGeneration may already have removed the + // pending ordinal. Its exact waiter must still exist, with no newer owner. + currentIndependent := hasAppliedWaiter && w.pendingGeneration[path] <= appliedGeneration if requireCurrent && w.pendingGeneration[path] == appliedGeneration { delete(w.pendingGeneration, path) } var completions []completion - for generation, done := range w.mutationWaiters[path] { + var accepted *PublicationPhaseRecord + for generation, waiter := range w.mutationWaiters[path] { if generation <= appliedGeneration { - completions = append(completions, completion{requested: generation, done: done}) + if publication != nil && err == nil && !w.stopping && generation == appliedGeneration && waiter.publication == publication.record { + accepted = waiter.publication + } + if publication != nil && publication.independent && err == nil && !w.stopping && + generation < appliedGeneration && waiter.publication != nil && currentIndependent { + accepted = publication.record + } + completions = append(completions, completion{requested: generation, done: waiter.done, publication: waiter.publication}) delete(w.mutationWaiters[path], generation) } } if len(w.mutationWaiters[path]) == 0 { delete(w.mutationWaiters, path) } + if w.observedMutationGeneration[path] <= appliedGeneration { + delete(w.observedMutationGeneration, path) + } w.mu.Unlock() + if accepted != nil { + accepted.Absorb(publication.stamps) + if publication.independent { + accepted.Mark(PublicationTicketCompleted) + for _, completion := range completions { + completion.publication.linkCoalescedWinner(accepted, completion.requested, appliedGeneration) + } + } + } for _, completion := range completions { completion.done <- MutationResult{ @@ -2147,11 +2336,12 @@ func (w *Watcher) failMutationWaiters(err error) { w.mu.Lock() waiters := w.mutationWaiters w.mutationWaiters = nil + w.observedMutationGeneration = nil w.mu.Unlock() for _, byGeneration := range waiters { - for generation, done := range byGeneration { - done <- MutationResult{RequestedGeneration: generation, Err: err} - close(done) + for generation, waiter := range byGeneration { + waiter.done <- MutationResult{RequestedGeneration: generation, Err: err} + close(waiter.done) } } } @@ -2464,12 +2654,12 @@ func (w *Watcher) completeStormMutationWaiters( if pathErr != nil { pathFanout = DerivedFanoutCompleteness{} } - for requestedGeneration, done := range w.mutationWaiters[path] { + for requestedGeneration, waiter := range w.mutationWaiters[path] { if requestedGeneration > appliedGeneration { continue } completions = append(completions, completion{ - done: done, + done: waiter.done, result: MutationResult{ RequestedGeneration: requestedGeneration, AppliedGeneration: appliedGeneration, @@ -2483,6 +2673,9 @@ func (w *Watcher) completeStormMutationWaiters( if len(w.mutationWaiters[path]) == 0 { delete(w.mutationWaiters, path) } + if w.observedMutationGeneration[path] <= appliedGeneration { + delete(w.observedMutationGeneration, path) + } } w.mu.Unlock() @@ -2589,6 +2782,17 @@ func (w *Watcher) patchGraphWithReceiptState( generation uint64, receiptChecked bool, fanout *DerivedFanoutCompleteness, +) error { + return w.patchGraphWithReceiptStateObserved(path, kind, generation, receiptChecked, fanout, nil) +} + +func (w *Watcher) patchGraphWithReceiptStateObserved( + path string, + kind ChangeKind, + generation uint64, + receiptChecked bool, + fanout *DerivedFanoutCompleteness, + publication *pointPublicationAttempt, ) error { if !w.generationCurrent(path, generation) { return errMutationSuperseded @@ -2599,12 +2803,13 @@ func (w *Watcher) patchGraphWithReceiptState( // IndexRepo replacement uses the same lane and cannot change underneath // the raw patch. laneCtx, cancelLane := w.mutationLaneContext() + laneCtx = publication.withContext(laneCtx) err := w.indexer.coordinateRepositoryMutation(laneCtx, OutputEntryWatcherPatchGraph, func() error { idx := w.currentMutationIndexer() if idx == nil { return errWatcherIndexerMissing } - return w.patchGraphWithReceiptStateRawModern(idx, path, kind, generation, receiptChecked, &pending, fanout) + return w.patchGraphWithReceiptStateRawModern(laneCtx, idx, path, kind, generation, receiptChecked, &pending, fanout) }) cancelLane() // User callbacks are deliberately outside the repository lane. Their @@ -2642,20 +2847,21 @@ func pointMutationClassification( // reindexPointPathRaw runs after the caller has acquired the repository lane. // MultiWatcher supplies the cross-repository resolver and derived tail; a // standalone Watcher falls back to the equivalent single-repository tail. -func (w *Watcher) reindexPointPathRaw(idx *Indexer, path string) (*IndexResult, error) { +func (w *Watcher) reindexPointPathRaw(idx *Indexer, path string, stamps *PublicationStamps) (*IndexResult, error) { if w.pointReindexRaw != nil { - return w.pointReindexRaw(path) + return w.pointReindexRaw(path, stamps) } if idx == nil || idx.rootPath == "" { return nil, errWatcherIndexerMissing } - return idx.incrementalPointWatcherPath(idx.rootPath, path) + return idx.incrementalPointWatcherPathObserved(idx.rootPath, path, stamps) } // patchGraphWithReceiptStateRawModern replaces the historical hand-built point // patch with the same receipt, resolver, and exact derived-invalidation pipeline // used by bounded reconciliation. The caller already owns the repository lane. func (w *Watcher) patchGraphWithReceiptStateRawModern( + ctx context.Context, idx *Indexer, path string, kind ChangeKind, @@ -2664,6 +2870,12 @@ func (w *Watcher) patchGraphWithReceiptStateRawModern( pending *symbolChangeNotification, fanout *DerivedFanoutCompleteness, ) error { + stamps := PublicationStampsFrom(ctx) + if stamps.switchAttemptControl(patchInspection) { + // Registered before the existing patchMu unlock: early-return inspection + // includes its real unlock, then outer mutation finalization continues. + defer stamps.switchAttemptControl(patchAttemptFinalize) + } if !w.generationCurrent(path, generation) { return errMutationSuperseded } @@ -2680,6 +2892,7 @@ func (w *Watcher) patchGraphWithReceiptStateRawModern( // startup replay already checked the persisted receipt, so it bypasses this // cheap in-memory mtime test and forces the exact observed path. if kind == ChangeModified && !receiptChecked && idx.fileMtimeMatches(path) { + stamps.recordNativeMtimeDuplicate() return nil } @@ -2725,7 +2938,9 @@ func (w *Watcher) patchGraphWithReceiptStateRawModern( // "complete" over a catch-up that was cut. observation := beginDerivedFanoutObservation(idx) defer observation.close() - result, err := w.reindexPointPathRaw(idx, path) + stamps.switchAttemptControlAt(patchExecutorControl, markPointPublication(ctx, PublicationPatchExecutionStarted)) + result, err := w.reindexPointPathRaw(idx, path, PublicationStampsFrom(ctx)) + stamps.switchAttemptControlAt(patchAttemptFinalize, markPointPublication(ctx, PublicationPatchExecutionFinished)) observed := observation.close() if err != nil { // A failed patch publishes no fan-out verdict. Whatever the bounded @@ -2824,13 +3039,26 @@ func (w *Watcher) patchGraphWithReceiptStateRawModern( default: } if w.logger != nil { - w.logger.Info("graph patch complete", - zap.String("file", path), - zap.String("kind", string(kind)), - zap.String("delta_class", classification), - zap.Int64("ms", ev.DurationMs), - zap.Int("incident_edges", priorEdges), - zap.Inline(patchWriteLog{split: idx.writeCounterMark().Since(writes)})) + if laps, observed := PublicationStampsFrom(ctx).snapshotPatchLog(); observed { + w.logger.Info("graph patch complete", + zap.String("file", path), + zap.String("kind", string(kind)), + zap.String("delta_class", classification), + zap.Int64("ms", ev.DurationMs), + zap.Int("incident_edges", priorEdges), + zap.Inline(patchWriteLog{split: idx.writeCounterMark().Since(writes)}), + zap.Inline(laps)) + } else { + // Keep the ordinary native log path free of new timing snapshots, + // field slices or lazy encoders. Only a bound attempt opts in. + w.logger.Info("graph patch complete", + zap.String("file", path), + zap.String("kind", string(kind)), + zap.String("delta_class", classification), + zap.Int64("ms", ev.DurationMs), + zap.Int("incident_edges", priorEdges), + zap.Inline(patchWriteLog{split: idx.writeCounterMark().Since(writes)})) + } } if superseded { // Every core write of the patch landed before the newer generation diff --git a/internal/indexer/watcher_duplicate_notification_test.go b/internal/indexer/watcher_duplicate_notification_test.go index 60a9a63a6..18824b41f 100644 --- a/internal/indexer/watcher_duplicate_notification_test.go +++ b/internal/indexer/watcher_duplicate_notification_test.go @@ -62,7 +62,7 @@ func Original() {} // is never written while a patch goroutine reads it. var armed atomic.Bool var once sync.Once - w.pointReindexRaw = func(p string) (*IndexResult, error) { + w.pointReindexRaw = func(p string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, p) if armed.Load() { // The window between "graph mutated" and "is my generation diff --git a/internal/indexer/watcher_enqueue_test.go b/internal/indexer/watcher_enqueue_test.go index 2230c8023..c74d052cf 100644 --- a/internal/indexer/watcher_enqueue_test.go +++ b/internal/indexer/watcher_enqueue_test.go @@ -114,7 +114,7 @@ func TestWatcherEnqueueFileMutationWaitsForCompletePointTail(t *testing.T) { close(releaseResolve) } }() - watcher.pointReindexRaw = func(filePath string) (*IndexResult, error) { + watcher.pointReindexRaw = func(filePath string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, filePath) resolveEntered <- filePath <-releaseResolve diff --git a/internal/indexer/watcher_publication.go b/internal/indexer/watcher_publication.go new file mode 100644 index 000000000..884530a7c --- /dev/null +++ b/internal/indexer/watcher_publication.go @@ -0,0 +1,71 @@ +package indexer + +import ( + "context" + "time" +) + +// pointPublicationAttempt is local to one runnable point attempt. Failed, +// retried, stopped and superseded attempts never publish this collector into +// a receipt. The guarded existing waiter completion accepts only its exact +// winning record, outside the watcher lock and before Done delivery. +type pointPublicationAttempt struct { + record *PublicationPhaseRecord + stamps *PublicationStamps + independent bool +} + +func (w *Watcher) beginPointPublication(path string, generation uint64) *pointPublicationAttempt { + return w.beginPointPublicationObserved(path, generation, 0, nil) +} + +func (w *Watcher) beginPointPublicationObserved(path string, generation uint64, retry int, observation *canonicalPointTimer) *pointPublicationAttempt { + w.mu.Lock() + var record *PublicationPhaseRecord + var predecessor *PublicationPhaseRecord + if !w.stopping && w.pendingGeneration[path] == generation { + record = w.mutationWaiters[path][generation].publication + if record == nil && observation != nil { + if requested := w.observedMutationGeneration[path]; requested != 0 && requested < generation { + predecessor = w.mutationWaiters[path][requested].publication + } + } + } + w.mu.Unlock() + independent := false + if record == nil && predecessor != nil { + origin := observation.entered + if retry == 0 && !observation.debounce.started.IsZero() && !observation.debounce.started.After(observation.entered) { + origin = observation.debounce.started + } + record = predecessor.beginCoalescedAttempt(generation, retry, origin) + independent = record != nil + if independent && origin.Equal(observation.debounce.started) { + record.MarkAt(PublicationTimerArmStarted, origin) + } + } + if record == nil { + return nil // no tracing allocations on native/uninstrumented paths + } + stamps := record.newCanonicalPatchStamps() + stamps.markAt(PublicationCycleStarted, time.Now()) + return &pointPublicationAttempt{record: record, stamps: stamps, independent: independent} +} + +func (p *pointPublicationAttempt) withContext(ctx context.Context) context.Context { + if p == nil { + return ctx + } + // Keep the existing lane cancellation and deadline. Only the attempt's + // collector travels; attaching a live record would publish rejected spans. + return context.WithValue(ctx, publicationStampsKey{}, p.stamps) +} + +func markPointPublication(ctx context.Context, phase PublicationPhase) time.Time { + if stamps := PublicationStampsFrom(ctx); stamps != nil { + at := time.Now() + stamps.markAt(phase, at) + return at + } + return time.Time{} +} diff --git a/internal/indexer/watcher_publication_binding_test.go b/internal/indexer/watcher_publication_binding_test.go new file mode 100644 index 000000000..331c64ce1 --- /dev/null +++ b/internal/indexer/watcher_publication_binding_test.go @@ -0,0 +1,126 @@ +package indexer + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" +) + +func TestCanonicalMutationBindsPublicationBeforeWake(t *testing.T) { + w := newMutationRetryTestWatcher(t) + dir := t.TempDir() + path := filepath.Join(dir, "edit.go") + writeTestFile(t, path, "package edited\n") + w.indexer = newTestIndexer(graph.New()) + w.indexer.SetRootPath(dir) + record := NewPublicationPhaseRecorder(1, 1).Begin("", "commit", PublicationSourceMCPEdit, time.Now()) + claimed := make(chan PublicationPhaseSnapshot, 1) + w.pointMutationClaimed = func(string) { claimed <- record.Snapshot() } + w.pointMutationPatch = func(string, ChangeKind, uint64) error { return nil } + + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NotNil(t, ticket) + require.Same(t, record, ticket.PublicationRecord) + select { + case snapshot := <-claimed: + require.Equal(t, ticket.Generation, snapshot.Ticket, "the demand-woken callback must already know the original ticket") + require.Contains(t, phaseOffsets(snapshot), PublicationTicketEnqueued) + case <-time.After(5 * time.Second): + t.Fatal("mutation never reached the existing claim seam") + } + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.Equal(t, ticket.Generation, result.RequestedGeneration) + require.Equal(t, ticket.Generation, result.AppliedGeneration) + require.False(t, record.Snapshot().Terminal, "only Done's existing consumer owns terminal receipt observation") +} + +func TestCanonicalMutationCoalescingKeepsOriginalRecordIdentity(t *testing.T) { + w := newMutationRetryTestWatcher(t) + // Keep the existing debounce asleep; complete the exact coalesced frontier + // explicitly, so the oracle needs neither sleeps nor timer-order luck. + w.config.DebounceMs = int(time.Hour / time.Millisecond) + recorder := NewPublicationPhaseRecorder(2, 1) + older := recorder.Begin("", "older", PublicationSourceMCPEdit, time.Now()) + newer := recorder.Begin("", "newer", PublicationSourceMCPEdit, time.Now()) + first := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, older) + second := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, newer) + // Reusing a context record must not retarget the old timeline, even though + // the actual mutation is still admitted and supersedes both earlier ones. + third := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, older) + require.Same(t, older, first.PublicationRecord) + require.Same(t, newer, second.PublicationRecord) + require.Nil(t, third.PublicationRecord) + require.Equal(t, first.Generation, older.Snapshot().Ticket) + require.Equal(t, second.Generation, newer.Snapshot().Ticket) + require.True(t, w.completeMutationWaitersIfCurrent("edit.go", third.Generation, nil)) + for _, ticket := range []*MutationTicket{first, second, third} { + result := awaitMutationRetryResult(t, ticket) + require.Equal(t, ticket.Generation, result.RequestedGeneration) + require.Equal(t, third.Generation, result.AppliedGeneration) + require.NoError(t, result.Err) + } + require.Equal(t, first.Generation, older.Snapshot().Ticket) + require.Equal(t, second.Generation, newer.Snapshot().Ticket) +} + +func TestCanonicalMutationRejectsTerminalTimingAssociation(t *testing.T) { + w := newMutationRetryTestWatcher(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + record := NewPublicationPhaseRecorder(1, 1).Begin("", "settled", PublicationSourceMCPEdit, time.Now()) + record.Mark(PublicationTicketFailed) + ticket := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, record) + require.NotNil(t, ticket) + require.Nil(t, ticket.PublicationRecord) + require.Zero(t, record.Snapshot().Ticket) + require.True(t, w.completeMutationWaitersIfCurrent("edit.go", ticket.Generation, nil)) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) +} + +func TestCanonicalMutationCannotReuseRecordAcrossEqualSequences(t *testing.T) { + firstWatcher, secondWatcher := newMutationRetryTestWatcher(t), newMutationRetryTestWatcher(t) + firstWatcher.config.DebounceMs = int(time.Hour / time.Millisecond) + secondWatcher.config.DebounceMs = int(time.Hour / time.Millisecond) + record := NewPublicationPhaseRecorder(1, 1).Begin("", "single-use", PublicationSourceMCPEdit, time.Now()) + first := firstWatcher.scheduleFileMutationWithRecord("first.go", ChangeModified, record) + second := secondWatcher.scheduleFileMutationWithRecord("second.go", ChangeModified, record) + require.Equal(t, first.Generation, second.Generation, "sequences are local to their watcher") + require.Same(t, record, first.PublicationRecord) + require.Nil(t, second.PublicationRecord, "equal sequence numbers do not establish the same physical admission") + require.True(t, firstWatcher.completeMutationWaitersIfCurrent("first.go", first.Generation, nil)) + require.True(t, secondWatcher.completeMutationWaitersIfCurrent("second.go", second.Generation, nil)) + require.NoError(t, awaitMutationRetryResult(t, first).Err) + require.NoError(t, awaitMutationRetryResult(t, second).Err) +} + +func TestMutationWaiterReleasesPublicationWithExistingLifetime(t *testing.T) { + w := newMutationRetryTestWatcher(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + recorder := NewPublicationPhaseRecorder(2, 1) + record := recorder.Begin("", "outstanding", PublicationSourceMCPEdit, time.Now()) + ticket := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, record) + w.mu.Lock() + retained := w.mutationWaiters["edit.go"][ticket.Generation].publication + w.mu.Unlock() + require.Same(t, record, retained) + require.True(t, w.completeMutationWaitersIfCurrent("edit.go", ticket.Generation, nil)) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + w.mu.Lock() + remaining := len(w.mutationWaiters) + w.mu.Unlock() + require.Zero(t, remaining, "completion has no separate publication history to prune") + + stopped := recorder.Begin("", "stopped", PublicationSourceMCPEdit, time.Now()) + stoppedTicket := w.scheduleFileMutationWithRecord("stopped.go", ChangeModified, stopped) + require.NoError(t, w.Stop()) + require.ErrorIs(t, awaitMutationRetryResult(t, stoppedTicket).Err, errWatcherStopped) + w.mu.Lock() + remaining = len(w.mutationWaiters) + w.mu.Unlock() + require.Zero(t, remaining, "stop releases the same pending ownership") +} diff --git a/internal/indexer/watcher_publication_execution_test.go b/internal/indexer/watcher_publication_execution_test.go new file mode 100644 index 000000000..1c9858367 --- /dev/null +++ b/internal/indexer/watcher_publication_execution_test.go @@ -0,0 +1,420 @@ +package indexer + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func pointPublicationFixture(t *testing.T) (string, *Indexer, *Watcher) { + t.Helper() + root, idx, w := inertTestWatcher(t, "edit.go", "package p\n\nfunc Before() {}\n") + // Explicit queue admission exercises the real executor without starting an + // fsnotify loop. Stop must still join every timer/patch and close its handle. + w.degradedNoFsnotify = true + t.Cleanup(func() { + if !w.mutationAdmissionStopped() { + require.NoError(t, w.Stop()) + } + }) + path := filepath.Join(root, "edit.go") + bumpMtime(t, path, "package p\n\nfunc After() {}\n") + return path, idx, w +} + +func newPointPublication(t *testing.T, name string) *PublicationPhaseRecord { + t.Helper() + // Keep room for the separately owned physical successor. Eviction refusal + // has its own bounded-recorder control below. + recorder := NewPublicationPhaseRecorder(2, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + return recorder.Begin("", name, PublicationSourceMCPEdit, time.Now()) +} + +func requirePointExecution(t *testing.T, record *PublicationPhaseRecord, want bool) { + t.Helper() + offsets := phaseOffsets(record.Snapshot()) + for _, phase := range []PublicationPhase{PublicationCycleStarted, PublicationPatchExecutionStarted, PublicationPatchExecutionFinished, + PublicationPatchDerivedStarted, PublicationPatchDerivedFinished} { + if want { + require.Contains(t, offsets, phase) + } else { + require.NotContains(t, offsets, phase, "rejected/shared attempts must not publish receipt-owned spans") + } + } + if want { + require.NotNil(t, record.Snapshot().laneWait, "only this winning attempt owns its actual lane pair") + require.NotNil(t, record.Snapshot().patchLaps, "real standalone/MultiWatcher paths must carry the same local scalar collector") + require.LessOrEqual(t, offsets[PublicationPatchExecutionStarted], offsets[PublicationPatchDerivedStarted]) + require.LessOrEqual(t, offsets[PublicationPatchDerivedStarted], offsets[PublicationPatchDerivedFinished]) + require.LessOrEqual(t, offsets[PublicationPatchDerivedFinished], offsets[PublicationPatchExecutionFinished]) + timing := record.Snapshot().EditTimings() + require.Equal(t, "measured", timing.States["derived"]) + require.NotNil(t, timing.Milliseconds["derived"]) + } else { + require.Nil(t, record.Snapshot().laneWait, "rejected/shared attempts must not borrow the winner's lane wait") + require.Nil(t, record.Snapshot().debounce, "rejected/shared attempts must not borrow the winner's quiet timer") + require.Nil(t, record.Snapshot().attemptWindow, "rejected/shared attempts must not publish the winner's physical window") + require.Nil(t, record.Snapshot().admissions, "rejected/shared attempts must not borrow admission measurements") + require.Nil(t, record.Snapshot().patchLaps, "rejected attempts must not transfer any scalar evidence") + } +} + +func awaitPublicationSignal(t *testing.T, signal <-chan struct{}) { + t.Helper() + select { + case <-signal: + case <-time.After(5 * time.Second): + t.Fatal("point execution did not reach its controlled boundary") + } +} + +func TestCanonicalPublicationObservesRealStandaloneExecution(t *testing.T) { + path, idx, w := pointPublicationFixture(t) + record := newPointPublication(t, "standalone") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.NotNil(t, nodeByName(t, idx, "edit.go", "After"), "a real parse/apply must have occurred") + requirePointExecution(t, record, true) + require.Equal(t, ticket.Generation, record.Snapshot().Ticket) + require.Zero(t, record.Snapshot().DirtyGenerationID) +} + +func TestCanonicalPublicationObservesMultiWatcherExecutor(t *testing.T) { + mw, mi, root, _ := setupMultiWatcherTest(t) + for _, w := range mw.watchers { + w.degradedNoFsnotify = true + t.Cleanup(func() { + if !w.mutationAdmissionStopped() { + require.NoError(t, w.Stop()) + } + }) + } + w := mw.watchers["repo-a"] + require.NotNil(t, w.pointReindexRaw, "production MultiWatcher must install the daemon's MultiIndexer executor") + path := filepath.Join(root, "main.go") + bumpMtime(t, path, "package main\n\nfunc ChangedByDaemonPath() {}\n") + record := newPointPublication(t, "multi") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + require.NotNil(t, nodeByName(t, mi.GetIndexer("repo-a"), "repo-a/main.go", "ChangedByDaemonPath")) + requirePointExecution(t, record, true) +} + +func TestCanonicalPublicationCommitsOnlyGuardedWinningAttempt(t *testing.T) { + for _, nativeSuccessor := range []bool{false, true} { + t.Run(fmt.Sprintf("native_successor=%t", nativeSuccessor), func(t *testing.T) { + path, _, w := pointPublicationFixture(t) + beforeComplete, release := make(chan struct{}), make(chan struct{}) + var releaseOnce atomic.Bool + t.Cleanup(func() { + if releaseOnce.CompareAndSwap(false, true) { + close(release) + } + }) + w.mutationBeforeComplete = func(_ string, generation uint64) { + if generation == 1 { + close(beforeComplete) + <-release + } + } + older := newPointPublication(t, "older") + first, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), older), path) + require.NoError(t, err) + awaitPublicationSignal(t, beforeComplete) + requirePointExecution(t, older, false) + bumpMtime(t, path, "package p\n\nfunc Newer() {}\n") + newer := newPointPublication(t, "newer") + var second *MutationTicket + if nativeSuccessor { + // The normal fsnotify path has no request context. A native + // successor must resolve the older ticket without borrowing its + // record for the newer physical work. + second = w.scheduleFileMutation(path, ChangeModified) + require.NotNil(t, second) + } else { + second, err = w.EnqueueFileMutation(WithPublicationRecord(t.Context(), newer), path) + require.NoError(t, err) + } + if releaseOnce.CompareAndSwap(false, true) { + close(release) + } + for _, ticket := range []*MutationTicket{first, second} { + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.Equal(t, ticket.Generation, result.RequestedGeneration) + require.Equal(t, second.Generation, result.AppliedGeneration) + } + require.Greater(t, second.Generation, first.Generation) + requirePointExecution(t, older, false) + requirePointExecution(t, newer, !nativeSuccessor) + require.Equal(t, first.Generation, older.Snapshot().Ticket) + if nativeSuccessor { + require.Eventually(t, func() bool { + winner := older.CoalescedPublication() + return winner != nil && winner.Timings.Window.PatchWallMS != nil + }, 5*time.Second, time.Millisecond) + winner := older.CoalescedPublication() + require.Equal(t, first.Generation, winner.RequestedGeneration) + require.Equal(t, second.Generation, winner.AppliedGeneration) + require.NotEqual(t, older.Key(), winner.Publication.Key) + require.Nil(t, winner.Timings.Milliseconds["debounce"]) + require.Equal(t, "not_established", winner.BackgroundCoverage) + } + }) + } +} + +func TestCanonicalPublicationDiscardsRetryAttemptSpans(t *testing.T) { + path, idx, w := pointPublicationFixture(t) + failedAt := make(chan time.Time, 1) + secondEntered, releaseSecond := make(chan struct{}), make(chan struct{}) + var released atomic.Bool + t.Cleanup(func() { + if released.CompareAndSwap(false, true) { + close(releaseSecond) + } + }) + var calls atomic.Int32 + w.mutationRetryDelayFn = func(int) time.Duration { return time.Millisecond } + w.pointReindexRaw = func(file string, stamps *PublicationStamps) (*IndexResult, error) { + if calls.Add(1) == 1 { + failedAt <- time.Now() + return nil, fmt.Errorf("busy: %w", context.DeadlineExceeded) + } + close(secondEntered) + <-releaseSecond + return idx.incrementalPointWatcherPathObserved(idx.rootPath, file, stamps) + } + origin := time.Now() + recorder := NewPublicationPhaseRecorder(1, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + record := recorder.Begin("", "retry", PublicationSourceMCPEdit, origin) + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + awaitPublicationSignal(t, secondEntered) + firstFailed := <-failedAt + requirePointExecution(t, record, false) + if released.CompareAndSwap(false, true) { + close(releaseSecond) + } + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + requirePointExecution(t, record, true) + require.EqualValues(t, 2, calls.Load()) + require.Nil(t, record.Snapshot().debounce, "winning retry backoff is not the initial debounce and cannot borrow the failed attempt's pair") + require.Equal(t, "not_observed", record.Snapshot().EditTimings().States["debounce"]) + require.Same(t, recorder, record.owner.Value(), "late completion requires retained recorder membership") + require.Eventually(t, func() bool { return record.Snapshot().attemptWindow != nil }, 2*time.Second, time.Millisecond) + require.Equal(t, 1, record.Snapshot().attemptWindow.retry) + require.Equal(t, ticket.Generation, record.Snapshot().EditTimings().Window.PatchAccounting.AttemptTicket) + require.Equal(t, 1, record.Snapshot().EditTimings().Window.PatchAccounting.AttemptRetry) + require.GreaterOrEqual(t, record.Snapshot().attemptWindow.startNS, firstFailed.Sub(origin).Nanoseconds(), "physical wall belongs only to the winning retry") + require.GreaterOrEqual(t, phaseOffsets(record.Snapshot())[PublicationPatchExecutionStarted], firstFailed.Sub(origin).Nanoseconds()) + require.GreaterOrEqual(t, record.Snapshot().laneWait.startNS, firstFailed.Sub(origin).Nanoseconds(), "retry must retain only the winning attempt's wait") +} + +func TestCanonicalPublicationDiscardsStoppedInFlightAttempt(t *testing.T) { + path, idx, w := pointPublicationFixture(t) + entered, release := make(chan struct{}), make(chan struct{}) + var released atomic.Bool + t.Cleanup(func() { + if released.CompareAndSwap(false, true) { + close(release) + } + }) + w.pointReindexRaw = func(file string, stamps *PublicationStamps) (*IndexResult, error) { + close(entered) + <-release + return idx.incrementalPointWatcherPathObserved(idx.rootPath, file, stamps) + } + record := newPointPublication(t, "stop") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + awaitPublicationSignal(t, entered) + stopped := make(chan error, 1) + go func() { stopped <- w.Stop() }() + require.ErrorIs(t, awaitMutationRetryResult(t, ticket).Err, errWatcherStopped) + requirePointExecution(t, record, false) + if released.CompareAndSwap(false, true) { + close(release) + } + select { + case err := <-stopped: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("stop did not join the released point attempt") + } + requirePointExecution(t, record, false) +} + +func TestCanonicalPublicationDiscardsFailedExecutionSpans(t *testing.T) { + path, _, w := pointPublicationFixture(t) + failure := errors.New("point executor rejected the patch") + var calls atomic.Int32 + w.pointReindexRaw = func(string, *PublicationStamps) (*IndexResult, error) { + calls.Add(1) + return nil, failure + } + record := newPointPublication(t, "failed") + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + require.ErrorIs(t, awaitMutationRetryResult(t, ticket).Err, failure) + require.EqualValues(t, 1, calls.Load(), "permanent failures must not be retried") + requirePointExecution(t, record, false) +} + +func TestPointPublicationPreservesLaneContextAndNativeAbstention(t *testing.T) { + w := newMutationRetryTestWatcher(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + ticket := w.scheduleFileMutation("native.go", ChangeModified) + require.Nil(t, w.beginPointPublication("native.go", ticket.Generation)) + require.True(t, w.completeMutationWaitersIfCurrent("native.go", ticket.Generation, nil)) + require.NoError(t, awaitMutationRetryResult(t, ticket).Err) + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + attempt := &pointPublicationAttempt{record: newPointPublication(t, "context"), stamps: &PublicationStamps{}} + observed := attempt.withContext(ctx) + wantDeadline, _ := ctx.Deadline() + gotDeadline, ok := observed.Deadline() + require.True(t, ok) + require.Equal(t, wantDeadline, gotDeadline) + cancel() + require.ErrorIs(t, observed.Err(), context.Canceled) + require.Nil(t, phaseRecordFrom(observed), "a collector context must not live-publish into a receipt") +} + +func TestCanonicalPublicationBypassedDerivedTailRemainsUnobserved(t *testing.T) { + path, idx, _ := pointPublicationFixture(t) + idx.deferGlobalPasses.Store(true) + stamps := &PublicationStamps{} + record := newPointPublication(t, "bypassed") + // This is the existing non-forced reconcile mode: even with a collector, + // its explicit deferred tail must not be fabricated as measured zero. + _, err := idx.incrementalWatcherPaths(idx.rootPath, []string{path}, incrementalPathMode{ + detectDeletions: true, + publication: stamps, + }) + require.NoError(t, err) + require.NotNil(t, nodeByName(t, idx, "edit.go", "After")) + record.Absorb(stamps) + offsets := phaseOffsets(record.Snapshot()) + require.NotContains(t, offsets, PublicationPatchDerivedStarted) + require.NotContains(t, offsets, PublicationPatchDerivedFinished) + timing := record.Snapshot().EditTimings() + require.Nil(t, timing.Milliseconds["derived"]) + require.Equal(t, "not_observed", timing.States["derived"]) +} + +func TestCanonicalNativeWinnerServesMultipleObservedPredecessors(t *testing.T) { + path, _, w := pointPublicationFixture(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + recorder := NewPublicationPhaseRecorder(8, 1) + t.Cleanup(func() { runtime.KeepAlive(recorder) }) + records := []*PublicationPhaseRecord{ + recorder.Begin("", "first", PublicationSourceMCPEdit, time.Now()), + recorder.Begin("", "second", PublicationSourceMCPEdit, time.Now()), + } + var tickets []*MutationTicket + for _, record := range records { + ticket, err := w.EnqueueFileMutation(WithPublicationRecord(t.Context(), record), path) + require.NoError(t, err) + tickets = append(tickets, ticket) + } + native := w.scheduleFileMutation(path, ChangeModified) + require.NotNil(t, native) + w.mu.Lock() + w.pending[path].Reset(time.Millisecond) + w.mu.Unlock() + for _, ticket := range append(tickets, native) { + result := awaitMutationRetryResult(t, ticket) + require.NoError(t, result.Err) + require.Equal(t, native.Generation, result.AppliedGeneration) + } + for i, record := range records { + requirePointExecution(t, record, false) + require.Eventually(t, func() bool { + winner := record.CoalescedPublication() + return winner != nil && winner.Timings.Window.PatchWallMS != nil + }, 5*time.Second, time.Millisecond) + winner := record.CoalescedPublication() + require.Equal(t, tickets[i].Generation, winner.RequestedGeneration) + require.Equal(t, native.Generation, winner.AppliedGeneration) + require.Nil(t, winner.Timings.Milliseconds["debounce"]) + } + require.Equal(t, records[0].CoalescedPublication().Publication.Key, records[1].CoalescedPublication().Publication.Key) + // Neither a retained edit nor its weak link can resurrect an evicted winner. + for i := 0; i < 8; i++ { + recorder.Begin("", fmt.Sprintf("replacement-%d", i), PublicationSourceMCPEdit, time.Now()) + } + for _, record := range records { + require.Nil(t, record.CoalescedPublication()) + } +} + +func TestCoalescedAttemptRequiresRetainedOwnerAndOneShotLink(t *testing.T) { + origin := time.Now() + recorder := NewPublicationPhaseRecorder(4, 1) + record := recorder.Begin("", "edit", PublicationSourceMCPEdit, origin) + require.True(t, record.bindCanonicalTicket(1, origin)) + winner := record.beginCoalescedAttempt(2, 0, origin.Add(time.Millisecond)) + require.NotNil(t, winner) + require.Nil(t, record.beginCoalescedAttempt(2, 0, origin.Add(2*time.Millisecond)), "wall-clock readings cannot rebind a callback identity") + // Admission alone cannot manufacture a serving relationship. + record.linkCoalescedWinner(winner, 1, 2) + require.Nil(t, record.CoalescedPublication()) + stamps := winner.newCanonicalPatchStamps() + winner.Absorb(stamps) + record.linkCoalescedWinner(winner, 1, 2) + linked := record.CoalescedPublication() + require.NotNil(t, linked) + require.Nil(t, linked.Timings.Window.PatchWallMS, "unfinished physical work remains unavailable") + other := record.beginCoalescedAttempt(3, 0, origin.Add(2*time.Millisecond)) + require.NotNil(t, other) + other.Absorb(other.newCanonicalPatchStamps()) + record.linkCoalescedWinner(other, 1, 3) + require.Equal(t, uint64(2), record.CoalescedPublication().AppliedGeneration, "a served ticket cannot be rebound") + require.Equal(t, uint64(2), linked.AppliedGeneration, "earlier returned values stay immutable") + // A retained pointer to an evicted record cannot create another attempt. + for i := 0; i < 4; i++ { + recorder.Begin("", fmt.Sprintf("evict-%d", i), PublicationSourceMCPEdit, origin) + } + require.Nil(t, record.beginCoalescedAttempt(4, 0, origin.Add(3*time.Millisecond))) + require.Nil(t, record.CoalescedPublication()) + absent := &PublicationPhaseRecord{} + require.Nil(t, absent.beginCoalescedAttempt(1, 0, origin)) + require.Nil(t, absent.CoalescedPublication()) + runtime.KeepAlive(recorder) +} + +func TestCanonicalObservationIndexFollowsOutstandingWaiters(t *testing.T) { + w := newMutationRetryTestWatcher(t) + w.config.DebounceMs = int(time.Hour / time.Millisecond) + for i := 0; i < 100; i++ { + w.scheduleFileMutation(fmt.Sprintf("native-%d.go", i), ChangeModified) + } + require.Nil(t, w.observedMutationGeneration, "native storms do not allocate observation metadata") + record := newPointPublication(t, "indexed") + observed := w.scheduleFileMutationWithRecord("edit.go", ChangeModified, record) + require.Equal(t, observed.Generation, w.observedMutationGeneration["edit.go"]) + native := w.scheduleFileMutation("edit.go", ChangeModified) + require.Equal(t, observed.Generation, w.observedMutationGeneration["edit.go"], "native admission never rebinds the observed owner") + require.True(t, w.completeMutationWaitersIfCurrent("edit.go", native.Generation, nil)) + require.Empty(t, w.observedMutationGeneration) + require.NoError(t, awaitMutationRetryResult(t, observed).Err) + require.NoError(t, awaitMutationRetryResult(t, native).Err) + other := w.scheduleFileMutationWithRecord("other.go", ChangeModified, newPointPublication(t, "stopped-index")) + require.NotEmpty(t, w.observedMutationGeneration) + w.failMutationWaiters(errWatcherStopped) + require.Nil(t, w.observedMutationGeneration) + require.ErrorIs(t, awaitMutationRetryResult(t, other).Err, errWatcherStopped) +} diff --git a/internal/indexer/watcher_retry_test.go b/internal/indexer/watcher_retry_test.go index 701a6ec5a..9647d931a 100644 --- a/internal/indexer/watcher_retry_test.go +++ b/internal/indexer/watcher_retry_test.go @@ -143,7 +143,7 @@ func TestPointMutationRepanicsArbitraryPanicWithoutCompletingTicket(t *testing.T done := make(chan MutationResult, 1) w.mu.Lock() w.pendingGeneration[path] = 1 - w.mutationWaiters = map[string]map[uint64]chan MutationResult{path: {1: done}} + w.mutationWaiters = map[string]map[uint64]mutationWaiter{path: {1: {done: done}}} w.mu.Unlock() wantPanic := &struct{ label string }{label: "programmer panic"} w.pointMutationPatch = func(string, ChangeKind, uint64) error { @@ -222,7 +222,7 @@ func TestStormMutationRetryMergesNewerState(t *testing.T) { secondDone := make(chan MutationResult, 1) w.mu.Lock() w.pendingGeneration[path] = 1 - w.mutationWaiters = map[string]map[uint64]chan MutationResult{path: {1: firstDone}} + w.mutationWaiters = map[string]map[uint64]mutationWaiter{path: {1: {done: firstDone}}} w.mu.Unlock() w.stormMu.Lock() w.stormBatch[path] = ChangeCreated @@ -239,7 +239,7 @@ func TestStormMutationRetryMergesNewerState(t *testing.T) { w.stormMu.Unlock() w.mu.Lock() w.pendingGeneration[path] = 2 - w.mutationWaiters[path][2] = secondDone + w.mutationWaiters[path][2] = mutationWaiter{done: secondDone} w.mu.Unlock() return nil, fmt.Errorf("busy: %w", context.DeadlineExceeded) } @@ -329,7 +329,7 @@ func TestStormPermanentCompletionDefersToSuccessor(t *testing.T) { secondDone := make(chan MutationResult, 1) w.mu.Lock() w.pendingGeneration[path] = 1 - w.mutationWaiters = map[string]map[uint64]chan MutationResult{path: {1: firstDone}} + w.mutationWaiters = map[string]map[uint64]mutationWaiter{path: {1: {done: firstDone}}} w.mu.Unlock() w.stormMu.Lock() w.stormBatch[path] = ChangeModified @@ -359,7 +359,7 @@ func TestStormPermanentCompletionDefersToSuccessor(t *testing.T) { <-beforeComplete w.mu.Lock() w.pendingGeneration[path] = 2 - w.mutationWaiters[path][2] = secondDone + w.mutationWaiters[path][2] = mutationWaiter{done: secondDone} w.mu.Unlock() w.stormMu.Lock() w.stormBatch[path] = ChangeDeleted @@ -393,7 +393,7 @@ func TestStormStoragePanicCompletesDetachedWaitersWithTypedError(t *testing.T) { done := make(chan MutationResult, 1) w.mu.Lock() w.pendingGeneration[path] = 1 - w.mutationWaiters = map[string]map[uint64]chan MutationResult{path: {1: done}} + w.mutationWaiters = map[string]map[uint64]mutationWaiter{path: {1: {done: done}}} w.mu.Unlock() w.stormMu.Lock() w.stormBatch[path] = ChangeModified @@ -427,7 +427,7 @@ func TestStormDrainRepanicsArbitraryPanicWithoutCompletingWaiters(t *testing.T) done := make(chan MutationResult, 1) w.mu.Lock() w.pendingGeneration[path] = 1 - w.mutationWaiters = map[string]map[uint64]chan MutationResult{path: {1: done}} + w.mutationWaiters = map[string]map[uint64]mutationWaiter{path: {1: {done: done}}} w.mu.Unlock() w.stormMu.Lock() w.stormBatch[path] = ChangeModified diff --git a/internal/indexer/watcher_storm_batch_test.go b/internal/indexer/watcher_storm_batch_test.go index 367172bdf..701d61767 100644 --- a/internal/indexer/watcher_storm_batch_test.go +++ b/internal/indexer/watcher_storm_batch_test.go @@ -147,7 +147,7 @@ func TestWatcherStormDrainWaitsForClaimedPointPatchLane(t *testing.T) { close(releasePoint) } }() - watcher.pointReindexRaw = func(filePath string) (*IndexResult, error) { + watcher.pointReindexRaw = func(filePath string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, filePath) close(pointTailEntered) <-releasePoint diff --git a/internal/indexer/watcher_test.go b/internal/indexer/watcher_test.go index 305875ae0..02640a045 100644 --- a/internal/indexer/watcher_test.go +++ b/internal/indexer/watcher_test.go @@ -413,7 +413,7 @@ func TestWatcher_DirScanWaitsForPointMutationRepositoryLane(t *testing.T) { close(releasePoint) } }() - watcher.pointReindexRaw = func(filePath string) (*IndexResult, error) { + watcher.pointReindexRaw = func(filePath string, _ *PublicationStamps) (*IndexResult, error) { result, err := idx.incrementalPointWatcherPath(idx.rootPath, filePath) close(pointTailEntered) <-releasePoint diff --git a/internal/indexer/workspace_prefix_snapshot_test.go b/internal/indexer/workspace_prefix_snapshot_test.go index 3dd37e0b2..c625d776c 100644 --- a/internal/indexer/workspace_prefix_snapshot_test.go +++ b/internal/indexer/workspace_prefix_snapshot_test.go @@ -16,11 +16,11 @@ import ( func TestIndexMultiRepoWithoutConfigManager(t *testing.T) { mi := newColdOrchestrationMulti(graph.New(), zap.NewNop()) t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) - results, err := mi.indexMultiRepo(nil) + results, err := mi.indexFixtureRepos(nil) require.NoError(t, err) assert.Empty(t, results) repos := coldOrchestrationRepos(t, 1) - results, err = mi.indexMultiRepo(repos) + results, err = mi.indexFixtureRepos(repos) require.NoError(t, err) require.Len(t, results, 1) assert.NotNil(t, mi.GetMetadata(config.ResolvePrefix(repos[0]))) diff --git a/internal/indexer/worktree.go b/internal/indexer/worktree.go index cef1f9641..596f204d7 100644 --- a/internal/indexer/worktree.go +++ b/internal/indexer/worktree.go @@ -2,6 +2,7 @@ package indexer import ( "context" + "errors" "fmt" "hash/fnv" "os" @@ -43,6 +44,14 @@ type WorktreeInfo struct { // repository have multiple checkouts, each assigned to its own // workspace and indexed from its own branch / working tree. func ResolveWorktree(path string) WorktreeInfo { + info, _ := resolveWorktreeChecked(path) + return info +} + +// resolveWorktreeChecked keeps I/O uncertainty distinct from a non-git root. +// Startup identity publication must not treat a best-effort fallback as proof +// that no configured alias can own a request. +func resolveWorktreeChecked(path string) (WorktreeInfo, error) { abs, err := filepath.Abs(path) if err != nil { abs = path @@ -52,35 +61,53 @@ func ResolveWorktree(path string) WorktreeInfo { gitPath := filepath.Join(abs, ".git") st, err := os.Stat(gitPath) if err != nil { - return info // not a git repository + if errors.Is(err, os.ErrNotExist) { + return info, nil + } + return info, err } if st.IsDir() { info.GitCommonDir = gitPath - return info // the main checkout + return info, nil // the main checkout } // `.git` is a file: a linked worktree or a submodule. Read the // gitdir indirection. content, err := os.ReadFile(gitPath) if err != nil { - return info + return info, err } line := strings.TrimSpace(string(content)) wtGitDir := strings.TrimSpace(strings.TrimPrefix(line, "gitdir:")) if wtGitDir == "" || wtGitDir == line { - return info // malformed .git file + return info, fmt.Errorf("malformed gitdir in %s", gitPath) } if !filepath.IsAbs(wtGitDir) { wtGitDir = filepath.Join(abs, wtGitDir) } + gitInfo, err := os.Stat(wtGitDir) + if err != nil { + return info, err + } + if !gitInfo.IsDir() { + return info, fmt.Errorf("gitdir %s is not a directory", wtGitDir) + } + // Only a worktree's gitdir carries a `commondir` file; a // submodule's does not. Its absence means "not a worktree". commonRaw, err := os.ReadFile(filepath.Join(wtGitDir, "commondir")) if err != nil { - return info + if errors.Is(err, os.ErrNotExist) { + info.GitCommonDir = filepath.Clean(wtGitDir) + return info, nil // separate gitdir or submodule: still its own primary + } + return info, err } common := strings.TrimSpace(string(commonRaw)) + if common == "" { + return info, fmt.Errorf("empty commondir in %s", wtGitDir) + } if !filepath.IsAbs(common) { common = filepath.Join(wtGitDir, common) } @@ -92,7 +119,7 @@ func ResolveWorktree(path string) WorktreeInfo { if filepath.Base(common) == ".git" { info.MainRepoPath = filepath.Dir(common) } - return info + return info, nil } // WorktreeInstanceName decides the repo prefix that absPath should be diff --git a/internal/mcp/analyze_scope_test.go b/internal/mcp/analyze_scope_test.go index 79484cc8d..b44060715 100644 --- a/internal/mcp/analyze_scope_test.go +++ b/internal/mcp/analyze_scope_test.go @@ -82,8 +82,7 @@ func newAnalyzeServer(t *testing.T, flagOn bool, repos ...analyzeRepoSpec) (*Ser reg := testRegistry() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) diff --git a/internal/mcp/batch_acknowledgement.go b/internal/mcp/batch_acknowledgement.go new file mode 100644 index 000000000..f368fbd9f --- /dev/null +++ b/internal/mcp/batch_acknowledgement.go @@ -0,0 +1,143 @@ +package mcp + +import ( + "context" + "crypto/sha256" + "strconv" + "strings" + + "github.com/zzet/gortex/internal/indexer" +) + +// bindBatchAcknowledgement belongs exclusively to runBatchTransaction's first +// execute arm. Duplicate/status/recovery calls must not bind their own clocks. +// No timing pointer or payload is retained by the unbounded transaction map. +func (s *Server) bindBatchAcknowledgement(ctx context.Context, state *batchTransactionState) { + ack := editAcknowledgementFrom(ctx, s) + if ack == nil { + return + } + state.mu.Lock() + if state.acknowledgementReceipt != "" { + state.mu.Unlock() + return + } + id := "batch-ack-" + strconv.FormatUint(ack.id, 10) + transactionID, checkoutID, incarnation := state.receipt.TransactionID, state.receipt.CheckoutID, state.receipt.Incarnation + state.acknowledgementReceipt = id + state.mu.Unlock() + // No state lock crosses the existing ledger's eviction or record locks. + // A racing reader between scalar reservation and this put reports missing + // evidence, never an invented wall. No work is awakened by this helper. + s.mutationCommits.put(&mutationCommitRecord{ + kind: batchAcknowledgementCommit, id: id, startedAt: ack.entered, + acknowledgement: ack, batchOwner: sha256.Sum256([]byte(transactionID)), + checkoutID: checkoutID, checkoutIncarnation: incarnation, + }) +} + +// A batch timing receipt is a separate explicit status variant. It carries no +// physical disk/graph verdict and never routes through byReceipt/replay/source +// evidence. The durable transaction remains the authority for those outcomes. +func (s *Server) batchAcknowledgementReceiptPayload(ctx context.Context, id string) (map[string]any, bool) { + if !strings.HasPrefix(id, "batch-ack-") { + return nil, false + } + checkoutID, incarnation := mutationCheckoutScope(ctx) + ledger := &s.mutationCommits + ledger.mu.Lock() + record := ledger.byID[id] + var ack *editAcknowledgement + if record != nil && record.kind == batchAcknowledgementCommit && + mutationCheckoutScopeMatches(checkoutID, incarnation, record.checkoutID, record.checkoutIncarnation) { + record.mu.RLock() + ack = record.acknowledgement + record.mu.RUnlock() + } + ledger.mu.Unlock() + payload := map[string]any{ + "found": ack != nil, "receipt": id, "receipt_kind": "batch_acknowledgement", + "guidance": "timing only; query the batch transaction_id for its durable source and graph outcomes", + } + timings := (indexer.PublicationPhaseSnapshot{}).EditTimings() + payload["phases_ms"], payload["phase_states"], payload["phase_timing"] = timings.Milliseconds, timings.States, timings.Window + for name := range payload["phase_states"].(map[string]string) { + payload["phase_states"].(map[string]string)[name] = "not_applicable_shared_rpc" + } + var snapshot *editAcknowledgementSnapshot + if ack != nil { + copy := ack.snapshot() + snapshot = © + } + attachAcknowledgementPayload(payload, snapshot, "unavailable_ledger_entry_or_scope") + return payload, true +} + +// The response-only envelope never enters cloneBatchReceipt or the manifest. +// A reference can outlive its bounded payload; mutation_status reports that +// absence explicitly rather than borrowing a retry/status call's own clock. +type batchAcknowledgementResponse struct { + batchTransactionReceipt + AcknowledgementReceipt string `json:"acknowledgement_receipt,omitempty"` + AcknowledgementState string `json:"acknowledgement_state"` + FileObservations []batchFileObservationReference `json:"file_observations,omitempty"` +} + +func (s *Server) batchAcknowledgementResponse(receipt batchTransactionReceipt) batchAcknowledgementResponse { + result := batchAcknowledgementResponse{batchTransactionReceipt: receipt, AcknowledgementState: "unavailable_reference"} + if value, ok := s.batchTransactions.Load(receipt.TransactionID); ok { + if state, ok := value.(*batchTransactionState); ok { + state.mu.RLock() + result.AcknowledgementReceipt = state.acknowledgementReceipt + state.mu.RUnlock() + if result.AcknowledgementReceipt != "" { + result.AcknowledgementState = "reference_only_query_mutation_status" + } + } + } + result.FileObservations = s.batchFileObservationReferences(receipt) + return result +} + +func (r batchAcknowledgementResponse) referenceLine() string { + if r.AcknowledgementReceipt != "" { + return "acknowledgement_receipt=" + r.AcknowledgementReceipt + "\n" + } + return "acknowledgement_state=" + r.AcknowledgementState + "\n" +} + +type batchAcknowledgementSnapshot struct { + Reference string + State string + Snapshot *editAcknowledgementSnapshot +} + +func (s *Server) batchAcknowledgement(state *batchTransactionState) batchAcknowledgementSnapshot { + state.mu.RLock() + id, transactionID := state.acknowledgementReceipt, state.receipt.TransactionID + state.mu.RUnlock() + result := batchAcknowledgementSnapshot{Reference: id, State: "unavailable_reference"} + if id == "" { + return result // restarted/raw/legacy transactions do not acquire a new origin + } + owner := sha256.Sum256([]byte(transactionID)) + ledger := &s.mutationCommits + ledger.mu.Lock() + record := ledger.byID[id] + var ack *editAcknowledgement + if record != nil && record.kind == batchAcknowledgementCommit && record.batchOwner == owner { + record.mu.RLock() + ack = record.acknowledgement + record.mu.RUnlock() + } + ledger.mu.Unlock() + result.State = "unavailable_ledger_entry" + if ack == nil { + return result + } + // Borrow only long enough to produce this owned snapshot. No callback, + // formatter or encoder executes while the ledger/record locks are held. + snapshot := ack.snapshot() + result.State, result.Snapshot = "observed", &snapshot + return result +} diff --git a/internal/mcp/batch_acknowledgement_test.go b/internal/mcp/batch_acknowledgement_test.go new file mode 100644 index 000000000..1dc19f127 --- /dev/null +++ b/internal/mcp/batch_acknowledgement_test.go @@ -0,0 +1,159 @@ +package mcp + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBatchAcknowledgementOriginalExecutionDuplicateEvictionAndRestart(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + file := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + var writes atomic.Int64 + s.batchWriteOverride = func(path string, content []byte, mode os.FileMode) error { + writes.Add(1) + return durableAtomicWriteFile(path, content, mode) + } + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "original-ack-owner", "edits": []any{ + map[string]any{"op": "edit_file", "path": file, "old_string": "before", "new_string": "after"}, + }} + h := s.boundToolHandler(s.handleAtomicBatchEdit) + result, err := h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + require.Equal(t, "after\n", readAtomicBatchFixture(t, file)) + value, ok := s.batchTransactions.Load("original-ack-owner") + require.True(t, ok) + state := value.(*batchTransactionState) + first := s.batchAcknowledgement(state) + require.Equal(t, "observed", first.State) + require.Equal(t, "measured", first.Snapshot.ReplyState) + require.True(t, strings.HasPrefix(first.Reference, "batch-ack-")) + result, err = h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + require.EqualValues(t, 1, writes.Load(), "duplicate does not execute another physical batch") + require.Equal(t, first, s.batchAcknowledgement(state), "duplicate reply cannot replace the original RPC wall") + + manifest, err := os.ReadFile(batchManifestPath("original-ack-owner")) + require.NoError(t, err) + require.NotContains(t, string(manifest), first.Reference) + require.NotContains(t, string(manifest), "acknowledgement", "no new journal field/version") + restarted := &Server{} + loaded, action, err := restarted.loadOrCreateBatchTransaction(t.Context(), "original-ack-owner", "") + require.NoError(t, err) + require.NotEqual(t, "execute", action) + require.Equal(t, "unavailable_reference", restarted.batchAcknowledgement(loaded).State) + + for i := range maxMutationCommits { + s.mutationCommits.put(&mutationCommitRecord{id: fmt.Sprintf("batch-cap-%d", i), startedAt: time.Now()}) + } + require.Equal(t, "unavailable_ledger_entry", s.batchAcknowledgement(state).State) + result, err = h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + require.Equal(t, first.Reference, s.batchAcknowledgement(state).Reference) + require.Nil(t, s.batchAcknowledgement(state).Snapshot, "an old scalar cannot resurrect evicted timing") + require.EqualValues(t, 1, writes.Load()) +} + +func TestBatchAcknowledgementCannotBecomePhysicalMutationOrSourceEvidence(t *testing.T) { + assertBatchObservationCannotBecomePhysical(t, batchAcknowledgementCommit) +} + +func TestBatchFileObservationCannotBecomePhysicalMutationOrSourceEvidence(t *testing.T) { + assertBatchObservationCannotBecomePhysical(t, batchFileObservationCommit) +} + +func assertBatchObservationCannotBecomePhysical(t *testing.T, kind mutationCommitKind) { + t.Helper() + s := newMutationServer() + _, ack, _ := s.withEditAcknowledgement(t.Context()) + record := &mutationCommitRecord{ + kind: kind, id: "batch-observation-control", startedAt: time.Now(), acknowledgement: ack, + // Even a wrongly supplied physical-looking field must not enroll the + // distinct kind in recovery, replay or source-authority paths. + key: "caller-key", fingerprint: "fingerprint", relPath: "pretend.go", absPath: filepath.Join(t.TempDir(), "pretend.go"), + reindexReceipt: "mutation-123", owner: s, + } + s.mutationCommits.put(record) + _, found := s.mutationCommits.byReceipt(record.id) + require.False(t, found) + _, found = s.mutationCommits.byMutationID(record.key) + require.False(t, found) + _, found = s.mutationCommits.byFreshnessReceipt(record.reindexReceipt) + require.False(t, found) + _, found = s.mutationCommits.recentForPath(record.relPath) + require.False(t, found) + require.Empty(t, s.mutationCommits.recent(maxMutationCommits)) + s.pendingSourceFiles.register(record) + require.Empty(t, s.pendingSourceFiles.versions) + note := &mutationCommitNote{} + note.observe(record) + require.Empty(t, note.verdict().Status) + record.markCommitted("pretend", 10) + record.markFailed(errors.New("not a physical failure")) + record.markNotApplied(errors.New("not a physical refusal")) + record.recordGraph(mutationReindexOutcome{Receipt: "mutation-999", Reindexed: true}) + record.markReceiptLost(record.reindexReceipt) + record.markSourceRecoveredForReceipt(record.reindexReceipt) + record.retainResponse(map[string]any{"pretend": true}) + _, replayed := record.replayResponse() + require.False(t, replayed) + _, _, replayed = s.replayMutation(record.key, record.fingerprint) + require.False(t, replayed) + require.Empty(t, record.diskStatus()) + require.Equal(t, mutationCommitSnapshot{}, record.snapshot()) + require.False(t, mutationCommitInScope(t.Context(), record)) + require.Equal(t, map[string]any{"found": false, "reason": "not_physical_commit"}, s.mutationStatusPayload(record)) + require.Nil(t, s.openCommittedPublicationRecord(t.Context(), record)) + require.Nil(t, retainedMutationAcknowledgement(record)) + payload := map[string]any{} + attachMutationCommit(payload, record) + require.Empty(t, payload) + require.Empty(t, record.disk) + require.Empty(t, record.graph) + require.False(t, record.pendingSourceRecovered) +} + +func TestBatchAcknowledgementConcurrentReservationAndOwnerIsolation(t *testing.T) { + s := newMutationServer() + state := &batchTransactionState{receipt: batchTransactionReceipt{TransactionID: "owner"}} + firstCtx, first, _ := s.withEditAcknowledgement(t.Context()) + secondCtx, second, _ := s.withEditAcknowledgement(t.Context()) + var joined sync.WaitGroup + joined.Add(2) + go func() { defer joined.Done(); s.bindBatchAcknowledgement(firstCtx, state) }() + go func() { defer joined.Done(); s.bindBatchAcknowledgement(secondCtx, state) }() + joined.Wait() + got := s.batchAcknowledgement(state) + require.Equal(t, "observed", got.State) + require.Contains(t, []uint64{first.id, second.id}, got.Snapshot.RPCID) + s.mutationCommits.mu.Lock() + retained := len(s.mutationCommits.ordered) + s.mutationCommits.mu.Unlock() + require.Equal(t, 1, retained, "one reserved scalar must produce one bounded owner") + foreign := &batchTransactionState{receipt: batchTransactionReceipt{TransactionID: "other"}, acknowledgementReceipt: got.Reference} + require.Equal(t, "unavailable_ledger_entry", s.batchAcknowledgement(foreign).State) + + // Existing ledger expiration also clears the batch payload. A subsequent + // duplicate/late bind never replaces its original reserved scalar. + s.mutationCommits.mu.Lock() + record := s.mutationCommits.byID[got.Reference] + record.mu.Lock() + record.startedAt = time.Now().Add(-2 * mutationCommitRetention) + record.mu.Unlock() + s.mutationCommits.mu.Unlock() + s.mutationCommits.put(&mutationCommitRecord{id: "expire-trigger", startedAt: time.Now()}) + s.bindBatchAcknowledgement(secondCtx, state) + require.Equal(t, "unavailable_ledger_entry", s.batchAcknowledgement(state).State) +} diff --git a/internal/mcp/batch_checkout.go b/internal/mcp/batch_checkout.go index 19c19d31a..8f1bf820c 100644 --- a/internal/mcp/batch_checkout.go +++ b/internal/mcp/batch_checkout.go @@ -159,7 +159,9 @@ func (s *Server) refreshCheckoutBatchGraph(ctx context.Context, state *batchTran err = fmt.Errorf("checkout coordinator does not support atomic batch publication") } else { pin := handoffRequestView(ctx, viewmetrics.HandoffCheckoutRefresh) + lap := s.beginAcknowledgementLap(ctx, editAckTicketAdmission) ticket, ticketErr := scheduler.EnqueueBatchRefresh(context.WithoutCancel(ctx), batchCheckoutFiles(receipt)) + lap.finish() err = ticketErr if err == nil && (ticket == nil || ticket.Ticket == nil || ticket.CheckoutID != receipt.CheckoutID || ticket.Incarnation != receipt.Incarnation || !pathkey.EqualPaths(ticket.Root, receipt.CheckoutRoot)) { err = fmt.Errorf("batch publication ticket does not belong to committed checkout") diff --git a/internal/mcp/batch_file_observation.go b/internal/mcp/batch_file_observation.go new file mode 100644 index 000000000..af51be310 --- /dev/null +++ b/internal/mcp/batch_file_observation.go @@ -0,0 +1,233 @@ +package mcp + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "strconv" + "strings" + + "github.com/zzet/gortex/internal/indexer" + "time" + + "github.com/zzet/gortex/internal/pathkey" +) + +// This first measured shape is one existing-file after-image. Other batch +// operations retain their ordinary semantics without acquiring invented file +// clocks. The durable transaction, never this bounded observation, is source +// authority; duplicate/recovery arms do not call this helper. +func (s *Server) beginBatchFileObservation(ctx context.Context, receipt batchTransactionReceipt, buffers map[string]*batchFileBuffer, paths []string) *mutationCommitRecord { + ack := editAcknowledgementFrom(ctx, s) + if ack == nil || len(paths) != 1 || len(receipt.Files) != 1 || len(receipt.Results) != 1 || receipt.Results[0].Op != "edit_file" { + return nil + } + buffer := buffers[paths[0]] + if buffer == nil || !buffer.existsBefore || !buffer.existsAfter || receipt.Files[0].Path != buffer.absPath { + return nil + } + record := &mutationCommitRecord{ + kind: batchFileObservationCommit, owner: s, + id: "batch-file-" + strconv.FormatUint(mutationCommitSequence.Add(1), 10), + tool: "batch_edit", key: receipt.TransactionID, fingerprint: receipt.Fingerprint, + absPath: buffer.absPath, relPath: buffer.relPath, + startedAt: ack.entered, acknowledgement: ack, + disk: mutationDiskInFlight, graph: mutationGraphPending, + batchOwner: sha256.Sum256([]byte(receipt.TransactionID)), + checkoutID: receipt.CheckoutID, checkoutIncarnation: receipt.Incarnation, + } + s.mutationCommits.put(record) + return record +} + +// Capture directly around the selected real forward writer. Journal and +// rollback writers keep their original separate calls. No bytes or writer +// closure are retained in the ledger, and an unobserved batch keeps its writer. +func (s *Server) observedBatchFileWriter(record *mutationCommitRecord, writer func(string, []byte, os.FileMode) error) func(string, []byte, os.FileMode) error { + if record == nil { + return writer + } + return func(path string, content []byte, mode os.FileMode) error { + started := time.Now() + err := writer(path, content, mode) + returned := time.Now() + blob, after := "", [32]byte{} + if err == nil { + blob, after = gitBlobSHA(content), sha256.Sum256(content) + } + ledger := &s.mutationCommits + ledger.mu.Lock() + record.mu.Lock() + if ledger.byID[record.id] == record && record.owner == s && record.kind == batchFileObservationCommit { + if !record.diskWriteStartedAt.IsZero() || !pathkey.EqualPaths(record.absPath, path) { + record.disk, record.errText = mutationDiskFailed, "unavailable_repeated_or_foreign_writer" + } else { + record.diskWriteStartedAt, record.committedAt = started, returned + if err != nil { + record.disk, record.errText = mutationDiskFailed, err.Error() + } else { + record.newSHA, record.batchAfterSHA256, record.bytesWritten = blob, after, len(content) + } + } + } + record.mu.Unlock() + ledger.mu.Unlock() + return err // observation never changes writer/durability behavior + } +} + +// A successful writer return alone is insufficient: directory durability and +// the actual transaction verdict must also succeed. Keep that verdict separate +// from committedAt's real per-file writer-return clock. Failure/rollback cannot +// become publication ownership, and late finalization cannot resurrect eviction. +func (s *Server) finishBatchFileObservation(record *mutationCommitRecord, receipt batchTransactionReceipt) { + if record == nil { + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[record.id] != record || record.owner != s || record.kind != batchFileObservationCommit || + record.batchOwner != sha256.Sum256([]byte(receipt.TransactionID)) || record.fingerprint != receipt.Fingerprint { + return + } + if record.disk == mutationDiskCommitted { + return + } + if receipt.Status != "committed" || receipt.DiskStatus != "committed" { + record.disk, record.graph, record.graphRecorded = mutationDiskFailed, mutationGraphFailed, true + record.errText = "batch_not_committed" + return + } + if len(receipt.Files) != 1 || receipt.Files[0].Path != record.absPath || receipt.Files[0].AfterAbsent || receipt.Files[0].BeforeAbsent || + receipt.Files[0].AfterSHA256 != hex.EncodeToString(record.batchAfterSHA256[:]) || record.disk != mutationDiskInFlight || + record.diskWriteStartedAt.IsZero() || record.committedAt.IsZero() || record.committedAt.Before(record.diskWriteStartedAt) || record.newSHA == "" { + record.disk, record.graph, record.graphRecorded = mutationDiskFailed, mutationGraphFailed, true + record.errText = "unavailable_writer_or_final_file_identity" + return + } + record.disk = mutationDiskCommitted + record.commitSeq = mutationDiskCommitSequence.Add(1) +} + +// batchPublicationObservation selects only the original measured transaction's +// retained file record. Recovery and duplicate admissions cannot manufacture a +// new file clock or replace a ticket already witnessed by that record. +func (s *Server) batchPublicationObservation(receipt batchTransactionReceipt, path string) *mutationCommitRecord { + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + for i := len(ledger.ordered) - 1; i >= 0; i-- { + record := ledger.ordered[i] + record.mu.RLock() + match := record.kind == batchFileObservationCommit && record.owner == s && + ledger.byID[record.id] == record && record.key == receipt.TransactionID && + record.fingerprint == receipt.Fingerprint && pathkey.EqualPaths(record.absPath, path) && + record.disk == mutationDiskCommitted && record.commitSeq != 0 && record.reindexReceipt == "" + record.mu.RUnlock() + if match { + return record + } + } + return nil +} + +// Keep the observational ticket local to this record; ordinary physical +// mutation lookup/recovery indexes must not acquire batch authority. +func (s *Server) bindBatchFilePublication(record *mutationCommitRecord, outcome mutationReindexOutcome) { + if record == nil || outcome.Receipt == "" || outcome.Generation == 0 { + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[record.id] != record || record.owner != s || record.kind != batchFileObservationCommit || + record.disk != mutationDiskCommitted || record.reindexReceipt != "" { + return + } + record.reindexReceipt, record.reindexGeneration = outcome.Receipt, outcome.Generation + record.graph, record.graphRecorded = graphStatusFor(outcome), true +} + +// This explicit timing-only variant never substitutes for the durable batch +// status. In particular a successful writer return is not a transaction ACK. +func (s *Server) batchFileObservationPayload(ctx context.Context, id string) (map[string]any, bool) { + if !strings.HasPrefix(id, "batch-file-") { + return nil, false + } + payload := map[string]any{"found": false, "receipt": id, "receipt_kind": "batch_file_observation", + "guidance": "timing only; query transaction_id for durable batch and per-file publication outcomes"} + checkout, incarnation := mutationCheckoutScope(ctx) + ledger := &s.mutationCommits + ledger.mu.Lock() + record := ledger.byID[id] + var publication *indexer.PublicationPhaseRecord + var committed time.Time + var ticket string + var repair *mutationReadRepairObservation + if record != nil { + record.mu.RLock() + if record.kind == batchFileObservationCommit && record.owner == s && + mutationCheckoutScopeMatches(checkout, incarnation, record.checkoutID, record.checkoutIncarnation) { + payload["found"], payload["transaction_id"], payload["path"] = true, record.key, record.relPath + payload["batch_disk_verdict"] = record.disk + if record.newSHA != "" { + payload["after_sha256"] = hex.EncodeToString(record.batchAfterSHA256[:]) + payload["new_sha"] = record.newSHA + } + payload["publication_receipt"], payload["publication_generation"] = record.reindexReceipt, record.reindexGeneration + payload["writer_return_observed"] = !record.committedAt.IsZero() + publication, committed, ticket = record.publication, record.committedAt, record.reindexReceipt + if record.readRepair != nil { + copy := *record.readRepair + repair = © + } + } + record.mu.RUnlock() + } + ledger.mu.Unlock() + if ticket != "" { + if outcome, ok := s.mutationReceiptState(ticket); ok && outcome.Generation != 0 { + payload["applied_generation"] = outcome.AppliedGeneration + payload["publication_pending"] = outcome.Pending + payload["publication_fresh"] = outcome.Reindexed && outcome.Err == nil && !outcome.Pending + if outcome.Err != nil { + payload["publication_error"] = outcome.Err.Error() + } + } + } + attachMutationPhaseTimings(payload, ticket, committed, indexer.PublicationSourceMCPEdit, publication) + if repair != nil { + payload["read_assisted_publication"] = repair.payload() + } + payload["acknowledgement_state"] = "not_applicable_shared_transaction_rpc" + return payload, true +} + +type batchFileObservationReference struct { + Path string `json:"path"` + Receipt string `json:"receipt"` +} + +func (s *Server) batchFileObservationReferences(transaction batchTransactionReceipt) []batchFileObservationReference { + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + for i := len(ledger.ordered) - 1; i >= 0; i-- { + record := ledger.ordered[i] + record.mu.RLock() + match := record.kind == batchFileObservationCommit && record.owner == s && ledger.byID[record.id] == record && + record.key == transaction.TransactionID && record.fingerprint == transaction.Fingerprint + reference := batchFileObservationReference{Path: record.relPath, Receipt: record.id} + record.mu.RUnlock() + if match { + return []batchFileObservationReference{reference} + } + } + return nil +} diff --git a/internal/mcp/batch_file_publication_test.go b/internal/mcp/batch_file_publication_test.go new file mode 100644 index 000000000..fedab9d81 --- /dev/null +++ b/internal/mcp/batch_file_publication_test.go @@ -0,0 +1,294 @@ +package mcp + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/zzet/gortex/internal/indexer" + "os" + "path/filepath" + "testing" + "time" + "weak" + + "github.com/stretchr/testify/require" +) + +func TestBatchFilePublicationBindsOnlyOriginalCommittedObservation(t *testing.T) { + s := newMutationServer() + path := filepath.Join(t.TempDir(), "fixture.go") + transaction := batchTransactionReceipt{TransactionID: "transaction-original", Fingerprint: "offered-content"} + record := &mutationCommitRecord{kind: batchFileObservationCommit, owner: s, id: "batch-file-original", key: transaction.TransactionID, fingerprint: transaction.Fingerprint, absPath: path, disk: mutationDiskCommitted, commitSeq: 1} + s.mutationCommits.put(record) + require.Same(t, record, s.batchPublicationObservation(transaction, path)) + foreign := transaction + foreign.TransactionID = "transaction-other" + require.Nil(t, s.batchPublicationObservation(foreign, path)) + foreign = transaction + foreign.Fingerprint = "different-offer" + require.Nil(t, s.batchPublicationObservation(foreign, path)) + require.Nil(t, s.batchPublicationObservation(transaction, path+".other")) + record.disk = mutationDiskInFlight + require.Nil(t, s.batchPublicationObservation(transaction, path), "writer return alone cannot supply durable batch authority") + record.disk = mutationDiskCommitted + s.bindBatchFilePublication(record, mutationReindexOutcome{Receipt: "publication-original", Generation: 7, Pending: true}) + require.Equal(t, "publication-original", record.reindexReceipt) + require.EqualValues(t, 7, record.reindexGeneration) + require.Nil(t, s.batchPublicationObservation(transaction, path), "recovery cannot open another original timeline") + s.bindBatchFilePublication(record, mutationReindexOutcome{Receipt: "publication-retry", Generation: 8, Pending: true}) + require.Equal(t, "publication-original", record.reindexReceipt) + _, found := s.mutationCommits.byFreshnessReceipt("publication-original") + require.False(t, found, "observation cannot gain physical freshness lookup authority") +} + +func TestBatchFileObservationStatusSeparatesWriterReturnFromDurableVerdict(t *testing.T) { + s := newMutationServer() + transaction := batchTransactionReceipt{TransactionID: "transaction-wire", Fingerprint: "offered"} + record := &mutationCommitRecord{kind: batchFileObservationCommit, owner: s, id: "batch-file-wire", key: transaction.TransactionID, fingerprint: transaction.Fingerprint, relPath: "fixture.ts", disk: mutationDiskInFlight} + s.mutationCommits.put(record) + references := s.batchFileObservationReferences(transaction) + require.Equal(t, []batchFileObservationReference{{Path: "fixture.ts", Receipt: record.id}}, references) + references[0].Receipt = "changed-client-copy" + require.Equal(t, record.id, s.batchFileObservationReferences(transaction)[0].Receipt) + payload, handled := s.batchFileObservationPayload(t.Context(), record.id) + require.True(t, handled) + require.Equal(t, true, payload["found"]) + require.Equal(t, "in_flight", payload["batch_disk_verdict"]) + require.Equal(t, false, payload["writer_return_observed"]) + require.Equal(t, "not_applicable_shared_transaction_rpc", payload["acknowledgement_state"]) + require.NotContains(t, payload, "mutation_id") + require.NotContains(t, payload, "graph_status_terminal") + record.disk = mutationDiskFailed + later, _ := s.batchFileObservationPayload(t.Context(), record.id) + require.Equal(t, "failed", later["batch_disk_verdict"]) + require.Equal(t, "in_flight", payload["batch_disk_verdict"], "an earlier scalar snapshot stays frozen") + foreign := transaction + foreign.TransactionID = "other" + require.Empty(t, s.batchFileObservationReferences(foreign)) +} + +func TestBatchFileObservationRealHandlerCommitAndFailedWriter(t *testing.T) { + for _, fail := range []bool{false, true} { + name := "commit" + if fail { + name = "failed-writer" + } + t.Run(name, func(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = -1 + file := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + transactionID := "file-observation-" + name + var insideWriter bool + s.batchWriteOverride = func(path string, content []byte, mode os.FileMode) error { + if insideWriter { + return durableAtomicWriteFile(path, content, mode) + } + value, ok := s.batchTransactions.Load(transactionID) + require.True(t, ok) + snapshot := value.(*batchTransactionState).snapshot() + refs := s.batchFileObservationReferences(snapshot) + require.Len(t, refs, 1) + status, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + require.Equal(t, "in_flight", status["batch_disk_verdict"], "actual writer cannot certify the eventual durable verdict") + insideWriter = true + if fail { + return errors.New("deliberate forward writer failure") + } + return durableAtomicWriteFile(path, content, mode) + } + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": transactionID, "edits": []any{ + map[string]any{"op": "edit_file", "path": file, "old_string": "before\n", "new_string": "after\n"}, + }} + h := s.boundToolHandler(s.handleAtomicBatchEdit) + _, err := h(t.Context(), req) + require.NoError(t, err) + require.True(t, insideWriter) + value, _ := s.batchTransactions.Load(transactionID) + receipt := value.(*batchTransactionState).snapshot() + refs := s.batchFileObservationReferences(receipt) + require.Len(t, refs, 1) + status, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + if fail { + require.Equal(t, "failed", status["batch_disk_verdict"]) + require.NotContains(t, status, "publication_phases") + return + } + require.Equal(t, "committed", status["batch_disk_verdict"]) + sum := sha256.Sum256([]byte("after\n")) + require.Equal(t, hex.EncodeToString(sum[:]), status["after_sha256"]) + require.Equal(t, "after\n", readAtomicBatchFixture(t, file)) + require.Equal(t, true, status["writer_return_observed"]) + manifest, err := os.ReadFile(batchManifestPath(transactionID)) + require.NoError(t, err) + require.NotContains(t, string(manifest), refs[0].Receipt, "timing is not durable transaction authority") + }) + } +} + +func TestBatchFileObservationRealRollbackRejectsSuccessfulWriter(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + dir := t.TempDir() + file := writeAtomicBatchFixture(t, dir, "target.txt", "before\n") + var failedSync bool + s.batchDurabilityOverride = &batchDurabilityOps{syncDirectory: func(path string) error { + if filepath.Clean(path) == filepath.Clean(dir) && !failedSync { + failedSync = true + return errors.New("forward directory sync refused") + } + return syncBatchDirectory(path) + }} + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "file-observation-rollback", "edits": []any{ + map[string]any{"op": "edit_file", "path": file, "old_string": "before\n", "new_string": "after\n"}, + }} + _, err := s.boundToolHandler(s.handleAtomicBatchEdit)(t.Context(), req) + require.NoError(t, err) + value, _ := s.batchTransactions.Load("file-observation-rollback") + transaction := value.(*batchTransactionState).snapshot() + require.Equal(t, "rolled_back", transaction.DiskStatus) + require.Equal(t, "before\n", readAtomicBatchFixture(t, file)) + refs := s.batchFileObservationReferences(transaction) + require.Len(t, refs, 1) + payload, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + require.Equal(t, true, payload["writer_return_observed"], "successful forward return really occurred") + require.Equal(t, "failed", payload["batch_disk_verdict"], "rolled-back bytes never become a committed batch clock") + require.NotContains(t, payload, "publication_phases") + require.Empty(t, payload["publication_receipt"]) +} + +func TestBatchFileObservationRealReadmissionCannotReplaceOriginalTicket(t *testing.T) { + done := make(chan indexer.MutationResult) + defer close(done) + s := newAtomicBatchTestServer(t, mutationTestWatcher{generation: 71, done: done}) + s.mutationReindexWait = time.Nanosecond + file := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "file-observation-readmission", "edits": []any{ + map[string]any{"op": "edit_file", "path": file, "old_string": "before\n", "new_string": "after\n"}, + }} + _, err := s.boundToolHandler(s.handleAtomicBatchEdit)(t.Context(), req) + require.NoError(t, err) + value, _ := s.batchTransactions.Load("file-observation-readmission") + state := value.(*batchTransactionState) + original := state.snapshot() + require.Equal(t, "pending", original.GraphStatus) + refs := s.batchFileObservationReferences(original) + require.Len(t, refs, 1) + before, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + originalTicket := original.Files[0].ReindexReceipt + require.NotEmpty(t, originalTicket) + require.Equal(t, originalTicket, before["publication_receipt"]) + s.mutationReceipts.Delete(originalTicket) + replacement, err := s.batchTransactionStatus(t.Context(), original.TransactionID) + require.NoError(t, err) + require.NotEmpty(t, replacement.Files[0].ReindexReceipt) + require.NotEqual(t, originalTicket, replacement.Files[0].ReindexReceipt, "durable transaction legitimately readmits lost publication") + after, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + require.Equal(t, originalTicket, after["publication_receipt"], "readmission cannot borrow a later ticket for original writer timing") + require.Equal(t, before["publication_generation"], after["publication_generation"]) +} + +func TestBatchFileObservationActualAdmissionRefusesEarlyPublisher(t *testing.T) { + for _, immediate := range []bool{false, true} { + name := "pending-actual-ticket" + if immediate { + name = "completed-before-admission-return" + } + t.Run(name, func(t *testing.T) { + watcher := &canonicalPublicationWatcher{hold: !immediate} + s := newAtomicBatchTestServer(t, watcher) + s.mutationReindexWait = time.Nanosecond + defer func() { + if !immediate { + for _, done := range watcher.results { + close(done) + } + } + }() + file := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + idx := &indexer.Indexer{} + earlyChecked := false + s.batchWriteOverride = func(path string, content []byte, mode os.FileMode) error { + require.Nil(t, s.readRepairObserver(t.Context(), path, idx), "publisher during writer cannot acquire eventual durable transaction ownership") + earlyChecked = true + return durableAtomicWriteFile(path, content, mode) + } + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "early-publisher-" + name, "edits": []any{map[string]any{"op": "edit_file", "path": file, "old_string": "before\n", "new_string": "after\n"}}} + _, err := s.boundToolHandler(s.handleAtomicBatchEdit)(t.Context(), req) + require.NoError(t, err) + require.True(t, earlyChecked) + require.Len(t, watcher.records, 1) + require.NotNil(t, watcher.records[0], "real scheduler receives original batch file timeline") + value, _ := s.batchTransactions.Load("early-publisher-" + name) + tx := value.(*batchTransactionState).snapshot() + refs := s.batchFileObservationReferences(tx) + require.Len(t, refs, 1) + payload, _ := s.batchFileObservationPayload(t.Context(), refs[0].Receipt) + require.Equal(t, tx.Files[0].ReindexReceipt, payload["publication_receipt"]) + if immediate { + // A completed requested publisher cannot be relabeled as a later reader's + // independently measured execution merely because the bytes now match. + require.Eventually(t, func() bool { return s.readRepairObserver(t.Context(), file, idx) == nil }, time.Second, time.Millisecond) + require.NotContains(t, payload, "read_assisted_publication") + } else { + witness, ok := s.captureReadRepairWitness(file, idx) + require.True(t, ok, "only the actual committed+admitted pending ticket can capture a subsequent reader") + require.Equal(t, tx.Files[0].ReindexReceipt, witness.receipt) + require.Equal(t, tx.Files[0].ReindexGeneration, witness.generation) + } + }) + } +} + +func TestBatchWriterReturnedCandidateRequiresFinalRetainedOriginalOwner(t *testing.T) { + for _, state := range []string{"in_flight", "rollback", "evicted", "replaced", "superseded", "changed_bytes", "changed_ticket"} { + t.Run(state, func(t *testing.T) { + s := newMutationServer() + idx := &indexer.Indexer{} + now := time.Now() + record := &mutationCommitRecord{kind: batchFileObservationCommit, owner: s, id: "batch-file-candidate", key: "transaction-original", fingerprint: "offered-original", absPath: "/repo/file.go", disk: mutationDiskInFlight, startedAt: now, committedAt: now, newSHA: gitBlobSHA([]byte("offered bytes"))} + s.mutationCommits.put(record) + candidate, ok := s.captureReadRepairWitness(record.absPath, idx) + require.True(t, ok, "real writer return is sufficient only for weak capture") + _, ready := candidate.completionWitness(s, record, idx) + require.False(t, ready, "writer return cannot certify final transaction or future ticket") + if state == "in_flight" { + return + } + publication := indexer.NewPublicationPhaseRecorder(2, 1).Begin("", "publication-original", indexer.PublicationSourceMCPEdit, now) + admission := &mutationReceipt{id: "publication-original", generation: 7, path: record.absPath} + s.mutationReceipts.Store(admission.id, admission) + record.disk, record.commitSeq = mutationDiskCommitted, 1 + record.publication, record.readRepairAdmission = publication, weak.Make(admission) + record.reindexReceipt, record.reindexGeneration = admission.id, admission.generation + bound, ready := candidate.completionWitness(s, record, idx) + require.True(t, ready, "negative must start from a fully ready same-record commit and actual identity") + switch state { + case "rollback": + record.disk = mutationDiskFailed + case "evicted": + s.mutationCommits.mu.Lock() + delete(s.mutationCommits.byID, record.id) + s.mutationCommits.mu.Unlock() + case "replaced": + s.mutationCommits.mu.Lock() + s.mutationCommits.byID[record.id] = &mutationCommitRecord{} + s.mutationCommits.mu.Unlock() + case "superseded": + s.mutationCommits.put(&mutationCommitRecord{kind: batchFileObservationCommit, owner: s, id: "batch-file-later", absPath: record.absPath, startedAt: time.Now(), disk: mutationDiskFailed}) + case "changed_bytes": + record.newSHA = gitBlobSHA([]byte("foreign bytes")) + case "changed_ticket": + candidate = bound + other := &mutationReceipt{id: "publication-later", generation: 8, path: record.absPath} + s.mutationReceipts.Store(other.id, other) + record.readRepairAdmission = weak.Make(other) + record.reindexReceipt, record.reindexGeneration = other.id, other.generation + } + _, ready = candidate.completionWitness(s, record, idx) + require.False(t, ready, "late validation borrowed invalid transaction/bytes/identity", state) + }) + } +} diff --git a/internal/mcp/batch_one_file_accounting_contract_test.go b/internal/mcp/batch_one_file_accounting_contract_test.go new file mode 100644 index 000000000..b1944cb53 --- /dev/null +++ b/internal/mcp/batch_one_file_accounting_contract_test.go @@ -0,0 +1,96 @@ +package mcp + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "sync/atomic" + "testing" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +// Exercise the real public facade/transaction serializer for the explicit +// whole-file edit_file fixture. A request-shape fixture alone cannot prove +// that its durable file, shared RPC and graph references join as advertised. +func TestBatchOneFileWholeImagePublicAcknowledgementContract(t *testing.T) { + t.Setenv(batchTransactionDirEnv, filepath.Join(t.TempDir(), "transactions")) + s, root := setupTestServer(t) + s.facades.capture(mcpgo.NewTool("batch_edit"), s.handleBatchEdit) + var schedules, writes atomic.Int64 + s.watcher = mutationTestWatcher{generation: 41, scheduled: &schedules} + s.batchWriteOverride = func(path string, content []byte, mode os.FileMode) error { + writes.Add(1) + return durableAtomicWriteFile(path, content, mode) + } + source := filepath.Join(root, "batch-full-image.go") + before := []byte("package fixture\nfunc Callee() int { return 1 }\n// anchor\n") + after := []byte("package fixture\nfunc Callee() int { return 1 }\n// anchor\nfunc Added() int { return Callee() }\n") + require.NoError(t, os.WriteFile(source, before, 0600)) + abs, relative, err := s.resolveFilePath(t.Context(), source) + require.NoError(t, err) + req := editOperationRequest("edit", "batch") + req.Params.Arguments = map[string]any{"operation": "batch", "changes": []any{map[string]any{"op": "edit_file", "path": source, "old_string": string(before), "new_string": string(after)}}, "options": map[string]any{"transaction_id": "whole-file-transaction"}, "output": map[string]any{"format": "json", "max_bytes": 0}} + h := s.boundToolHandler(func(ctx context.Context, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "edit", req) + }) + call := func() map[string]any { + result, err := h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError, "%v", result.Content) + return unmarshalResult(t, result) + } + first := call() + require.Equal(t, "whole-file-transaction", first["transaction_id"]) + require.Equal(t, "committed", first["status"]) + require.Equal(t, "committed", first["disk_status"]) + ack, ok := first["acknowledgement_receipt"].(string) + require.True(t, ok) + require.NotEmpty(t, ack) + require.Equal(t, "reference_only_query_mutation_status", first["acknowledgement_state"]) + observations := first["file_observations"].([]any) + require.Len(t, observations, 1) + observation := observations[0].(map[string]any) + require.Equal(t, relative, observation["path"]) + observationID := observation["receipt"].(string) + require.NotEqual(t, ack, observationID) + status, handled := s.batchFileObservationPayload(t.Context(), observationID) + require.True(t, handled) + require.Equal(t, "whole-file-transaction", status["transaction_id"]) + require.Equal(t, "committed", status["batch_disk_verdict"]) + files := first["files"].([]any) + results := first["results"].([]any) + require.Len(t, files, 1) + require.Len(t, results, 1) + file := files[0].(map[string]any) + result := results[0].(map[string]any) + require.Equal(t, abs, file["path"]) + require.Equal(t, relative, file["relative_path"]) + hash := sha256.Sum256(after) + require.Equal(t, hex.EncodeToString(hash[:]), file["after_sha256"]) + require.Equal(t, "edit_file", result["op"]) + require.Equal(t, "applied", result["status"]) + require.Equal(t, relative, result["path"]) + graph, ok := file["reindex_receipt"].(string) + require.True(t, ok) + require.NotEmpty(t, graph) + require.NotEqual(t, ack, graph) + require.NotEqual(t, observationID, graph) + require.Equal(t, graph, status["publication_receipt"]) + require.NotEqual(t, first["transaction_id"], graph) + require.NotEqual(t, first["transaction_id"], ack) + require.Equal(t, file["reindex_receipt"], result["reindex_receipt"]) + require.EqualValues(t, 41, file["reindex_generation"]) + require.Equal(t, file["reindex_generation"], result["reindex_generation"]) + actual, err := os.ReadFile(source) + require.NoError(t, err) + require.Equal(t, after, actual) + second := call() + require.Equal(t, first["acknowledgement_receipt"], second["acknowledgement_receipt"]) + require.Equal(t, first["files"], second["files"]) + require.EqualValues(t, 1, writes.Load(), "replay does not regenerate whole-file intent") + require.EqualValues(t, 1, schedules.Load(), "replay does not replace the original graph ticket") +} diff --git a/internal/mcp/batch_read_repair_admission_test.go b/internal/mcp/batch_read_repair_admission_test.go new file mode 100644 index 000000000..37c588f35 --- /dev/null +++ b/internal/mcp/batch_read_repair_admission_test.go @@ -0,0 +1,186 @@ +package mcp + +import ( + "context" + "errors" + "os" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/search" +) + +func TestBatchActualReaderCapturedBeforeFinalVerdictCompletesAfterAdmission(t *testing.T) { + testBatchReaderAcrossAdmission(t, false) +} +func TestBatchActualReaderCompletingBeforeAdmissionRemainsUnavailable(t *testing.T) { + testBatchReaderAcrossAdmission(t, true) +} +func testBatchReaderAcrossAdmission(t *testing.T, finishBeforeAdmission bool) { + root := setupMiniRepo(t, "batch-reader-admission") + path := filepath.Join(root, "main.go") + before, err := os.ReadFile(path) + require.NoError(t, err) + after := string(before) + "\nfunc AddedByBatchReader() { Hello() }\n" + gc := &config.GlobalConfig{Repos: []config.RepoEntry{{Path: root, Name: "fixture"}}} + gc.SetConfigPath(filepath.Join(t.TempDir(), "config.yaml")) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(gc.ConfigPath()) + require.NoError(t, err) + entered, resume := make(chan struct{}), make(chan struct{}) + var armed atomic.Bool + var enterOnce, releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(resume) }) } + core, _ := observer.New(zap.InfoLevel) + logger := zap.New(core, zap.Hooks(func(entry zapcore.Entry) error { + if armed.Load() && entry.Message == "indexer: reconcile phase started" { + enterOnce.Do(func() { close(entered) }) + <-resume + } + return nil + })) + g := graph.New() + mi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + registerMCPFixtureRepos(t, mi, cm) + idx := mi.GetIndexer("fixture") + require.NotNil(t, idx) + watcher := &canonicalPublicationWatcher{hold: true} + s := newAtomicBatchTestServer(t, watcher) + s.mutationReindexWait = time.Nanosecond + type execution struct { + observation indexer.ReadRepairObservation + err error + } + done := make(chan execution, 1) + var actual execution + started, joined := false, false + t.Cleanup(func() { + release() + if started && !joined { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("batch reader was not joined") + } + } + for _, ch := range watcher.results { + close(ch) + } + }) + transaction := "batch-reader-before-admission" + captured := false + var captureErr error + failCapture := func(message string) error { captureErr = errors.New(message); return captureErr } + var record *mutationCommitRecord + s.batchDurabilityOverride = &batchDurabilityOps{syncDirectory: func(dir string) error { + if filepath.Clean(dir) != filepath.Clean(root) || captured { + return syncBatchDirectory(dir) + } + captured = true + value, ok := s.batchTransactions.Load(transaction) + if !ok { + return failCapture("missing original transaction during target durability") + } + refs := s.batchFileObservationReferences(value.(*batchTransactionState).snapshot()) + if len(refs) != 1 { + return failCapture("missing exact file observation during target durability") + } + s.mutationCommits.mu.Lock() + record = s.mutationCommits.byID[refs[0].Receipt] + s.mutationCommits.mu.Unlock() + + if record == nil { + return failCapture("missing retained batch file record") + } + record.mu.RLock() + writerKnown := record.disk == mutationDiskInFlight && record.newSHA == gitBlobSHA([]byte(after)) && !record.committedAt.IsZero() && record.publication == nil + record.mu.RUnlock() + if !writerKnown { + return failCapture("fixture did not reach actual writer-return-known pre-verdict/pre-admission boundary") + } + observed := s.readRepairObserver(t.Context(), path, idx) + if observed == nil { + return failCapture("reader cannot capture the same writer-return-known record before final batch verdict/admission") + } + + armed.Store(true) + started = true + go func() { + var observation indexer.ReadRepairObservation + _, repairErr := mi.IncrementalReindexRepoObserved("fixture", []string{path}, &indexer.ReadRepairObserver{Complete: func(o indexer.ReadRepairObservation) { observation = o; observed.Complete(o) }}) + done <- execution{observation, repairErr} + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + return failCapture("actual reader did not start before batch verdict") + } + if finishBeforeAdmission { + release() + select { + case actual = <-done: + joined = true + case <-time.After(5 * time.Second): + return failCapture("early actual reader did not join") + } + if actual.err != nil || !actual.observation.AcceptedFor(idx, gitBlobSHA([]byte(after))) { + return failCapture("early executor did not really apply offered bytes") + } + } + return syncBatchDirectory(dir) + }} + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": transaction, "edits": []any{map[string]any{"op": "edit_file", "path": path, "old_string": string(before), "new_string": after}}} + result, err := s.boundToolHandler(s.handleAtomicBatchEdit)(t.Context(), req) + require.NoError(t, err) + require.NoError(t, captureErr) + require.False(t, result.IsError, "%v", result.Content) + require.True(t, captured) + record.mu.RLock() + disk, publication, admission := record.disk, record.publication, record.readRepairAdmission.Value() + writerReturned := record.committedAt + ticket, generation := record.reindexReceipt, record.reindexGeneration + earlyRetained := record.readRepair + record.mu.RUnlock() + require.Equal(t, mutationDiskCommitted, disk) + require.NotNil(t, publication) + require.NotNil(t, admission) + if finishBeforeAdmission { + require.Nil(t, earlyRetained, "actual callback before final verdict/admission cannot be retroactively borrowed") + require.True(t, actual.observation.AcceptedFor(idx, gitBlobSHA([]byte(after)))) + runtime.KeepAlive(idx) + return + } + release() + select { + case actual = <-done: + joined = true + case <-time.After(5 * time.Second): + t.Fatal("actual reader did not finish after admission") + } + require.NoError(t, actual.err) + require.True(t, actual.observation.AcceptedFor(idx, gitBlobSHA([]byte(after))), "actual executor must apply the offered after-image") + require.False(t, actual.observation.AcceptedAt().Before(writerReturned)) + require.NotEmpty(t, g.FindNodesByName("AddedByBatchReader")) + record.mu.RLock() + retained := record.readRepair + record.mu.RUnlock() + require.NotNil(t, retained, "successful final verdict and actual admission must validate the earlier exact candidate") + require.Equal(t, ticket, retained.requestedReceipt) + require.Equal(t, generation, retained.requestedGeneration) + require.Equal(t, gitBlobSHA([]byte(after)), retained.acceptedBlobSHA) + runtime.KeepAlive(idx) +} diff --git a/internal/mcp/batch_transaction.go b/internal/mcp/batch_transaction.go index f4971add2..93f1734ce 100644 --- a/internal/mcp/batch_transaction.go +++ b/internal/mcp/batch_transaction.go @@ -95,6 +95,9 @@ type batchTransactionState struct { recoveryMu sync.Mutex mu sync.RWMutex receipt batchTransactionReceipt + // A scalar reference only: the bounded mutation ledger owns timing data. + // This is neither durable journal metadata nor a pointer to an RPC history. + acknowledgementReceipt string } // cloneBatchReceipt returns a receipt that shares no slice or map storage with @@ -1017,7 +1020,7 @@ func applyBatchPlans(plans []plannedBatchEdit, buffers map[string]*batchFileBuff return results, -1, nil } -func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, requestedID string) (batchTransactionReceipt, error) { +func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, requestedID string) (result batchTransactionReceipt, retErr error) { fingerprint := batchEditFingerprint(edits) transactionID, err := normalizeBatchTransactionID(requestedID, fingerprint) if err != nil { @@ -1037,7 +1040,18 @@ func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, s.refreshBatchGraph(ctx, state) return state.snapshot(), nil } - + s.bindBatchAcknowledgement(ctx, state) + var fileObservation *mutationCommitRecord + + // Only the original new transaction owns source effects. Duplicate/status + // and recovery replies above must not count another logical edit. This + // defer stays on the actual handler, including non-cancellable commits + // that finish after its caller has received a deadline response. + defer func() { + s.finishBatchFileObservation(fileObservation, result) + committed := result.Status == "committed" && result.DiskStatus == "committed" + observeEditSourceTerminal(ctx, committed, committed && (result.Error != "" || retErr != nil), result.DiskStatus == "rolled_back") + }() receipt := state.snapshot() plans := s.planBatchTransaction(ctx, edits, true) for i, plan := range plans { @@ -1116,6 +1130,8 @@ func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, if s.batchRemoveOverride != nil { remover = s.batchRemoveOverride } + fileObservation = s.beginBatchFileObservation(ctx, receipt, buffers, orderedPaths) + writer = s.observedBatchFileWriter(fileObservation, writer) finishCommitFailure := func(failedPath, message string) batchTransactionReceipt { status, rollbackErr := s.rollbackBatchReceipt(receipt, ctx) if rollbackErr != nil { @@ -1145,15 +1161,16 @@ func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, if targetErr := s.validateBatchCreateTarget(ctx, path, buffer.relPath); targetErr != nil { commitErr = targetErr } else { - commitErr = writer(path, buffer.content, buffer.mode) + commitErr = s.writeAcknowledgedBatchFile(ctx, writer, path, buffer.content, buffer.mode) } } else { - commitErr = writer(path, buffer.content, buffer.mode) + commitErr = s.writeAcknowledgedBatchFile(ctx, writer, path, buffer.content, buffer.mode) } if commitErr != nil { message := fmt.Sprintf("could not commit %s: %v", buffer.relPath, commitErr) return finishCommitFailure(buffer.relPath, message), nil } + observeEditSourceReplacement(ctx, buffer.original, buffer.content, buffer.existsBefore, true) } for _, path := range orderedPaths { buffer := buffers[path] @@ -1163,16 +1180,18 @@ func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, if err := guardCheckoutMutationPath(ctx, path); err != nil { return finishCommitFailure(buffer.relPath, err.Error()), nil } - if commitErr := remover(path); commitErr != nil { + if commitErr := s.removeAcknowledgedBatchFile(ctx, remover, path); commitErr != nil { message := fmt.Sprintf("could not commit %s: %v", buffer.relPath, commitErr) return finishCommitFailure(buffer.relPath, message), nil } + observeEditSourceEffect(ctx, buffer.existsBefore, true) } - if syncErr := s.syncBatchDirectories(batchPathDirectories(orderedPaths)...); syncErr != nil { + if syncErr := s.syncAcknowledgedBatchDirectories(ctx, batchPathDirectories(orderedPaths)...); syncErr != nil { failedPath := buffers[orderedPaths[len(orderedPaths)-1]].relPath message := "could not persist committed files: " + syncErr.Error() return finishCommitFailure(failedPath, message), nil } + observeEditSourceComplete(ctx) // all forward effects and directory durability completed for i := range receipt.Results { receipt.Results[i].Status = "applied" @@ -1184,6 +1203,7 @@ func (s *Server) runBatchTransaction(ctx context.Context, edits []batchEditItem, receipt.Error = "disk committed; terminal journal update failed: " + persistErr.Error() } state.publish(receipt, false) + s.finishBatchFileObservation(fileObservation, receipt) for _, plan := range plans { session := s.sessionFor(ctx) @@ -1246,16 +1266,28 @@ func (s *Server) batchTransactionStatus(ctx context.Context, transactionID strin return state.snapshot(), nil } -func (s *Server) beginBatchGraphRefresh(absPath string) mutationReindexOutcome { - ctx := context.Background() +func (s *Server) beginBatchGraphRefresh(observationCtx context.Context, absPath string, transaction batchTransactionReceipt) mutationReindexOutcome { + ctx := context.Background() // preserve the original scheduler's lifetime if watcher := s.currentWatcher(); watcher != nil { if scheduler, ok := watcher.(mutationScheduler); ok { - ticket, err := scheduler.EnqueueFileMutation(ctx, absPath) + lap := s.beginAcknowledgementLap(observationCtx, editAckTicketAdmission) + var record *mutationCommitRecord + if editAcknowledgementFrom(observationCtx, s) != nil { + record = s.batchPublicationObservation(transaction, absPath) + } + var publication *indexer.PublicationPhaseRecord + if record != nil { + publication = s.openOwnedPublicationRecord(observationCtx, record) + } + ticket, err := scheduler.EnqueueFileMutation(indexer.WithPublicationRecord(ctx, publication), absPath) + lap.finish() if err != nil { return mutationReindexOutcome{Err: err} } if ticket != nil { receipt := s.trackMutationTicket(ticket) + s.bindBatchFilePublication(record, receipt.outcome(true)) + s.bindReadRepairAdmission(absPath, ticket.PublicationRecord, receipt) return receipt.outcome(true) } } @@ -1263,8 +1295,39 @@ func (s *Server) beginBatchGraphRefresh(absPath string) mutationReindexOutcome { return mutationReindexOutcome{Reindexed: s.reindexFile(absPath)} } +// batchReceiptsPublished closes once every tracked receipt of files has +// published, so a passive wait still queued for slow-lane capacity can stop +// queueing: nothing is left to wait for. It stops watching when ctx ends. +func (s *Server) batchReceiptsPublished(ctx context.Context, files []batchTransactionFile) <-chan struct{} { + var receipts []*mutationReceipt + for _, file := range files { + if file.ReindexReceipt == "" { + continue + } + if value, ok := s.mutationReceipts.Load(file.ReindexReceipt); ok { + if receipt, ok := value.(*mutationReceipt); ok { + receipts = append(receipts, receipt) + } + } + } + published := make(chan struct{}) + go func() { + for _, receipt := range receipts { + select { + case <-receipt.done: + case <-ctx.Done(): + return + } + } + close(published) + }() + return published +} + func (s *Server) waitBatchGraphReceipts(ctx context.Context, files []batchTransactionFile) { deadline := time.Now().Add(s.mutationWaitDuration()) + waitCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() var resumeIntent func() for _, file := range files { if file.ReindexReceipt == "" { @@ -1286,6 +1349,7 @@ func (s *Server) waitBatchGraphReceipts(ctx context.Context, files []batchTransa if remaining <= 0 { return } + waitLap := s.beginAcknowledgementLap(ctx, editAckReindexWait) if resumeIntent == nil { // All disk writes and admissions (including synchronous fallback) // finished before this passive wait. Restore ordinary priority @@ -1293,12 +1357,25 @@ func (s *Server) waitBatchGraphReceipts(ctx context.Context, files []batchTransa resumeIntent = suspendSourceMutationWriteIntent(ctx) defer resumeIntent() // Only waiting from here on: let other requests use this - // dispatch slot. - s.releaseDispatchSlotForWait(ctx) + // dispatch slot. Publications that all land first need none. + if !s.releaseDispatchSlotForWait(waitCtx, s.batchReceiptsPublished(waitCtx, files)) { + waitLap.finish() + return + } + } + remaining = time.Until(deadline) + if remaining <= 0 { + waitLap.finish() + return } timer := time.NewTimer(remaining) + // Preserve the bounded post-commit wait after successful admission, + // even if the caller canceled while committing. Already-admitted + // publication can still make this durable receipt fresh. The slow-lane + // admission above remains cancellation-aware and can leave it pending. select { case <-receipt.done: + waitLap.finish() if !timer.Stop() { select { case <-timer.C: @@ -1306,6 +1383,7 @@ func (s *Server) waitBatchGraphReceipts(ctx context.Context, files []batchTransa } } case <-timer.C: + waitLap.finish() return } } @@ -1361,7 +1439,9 @@ func (s *Server) refreshBatchGraph(ctx context.Context, state *batchTransactionS for _, file := range receipt.Files { paths = append(paths, file.Path) } + lap := s.beginAcknowledgementLap(ctx, editAckTicketAdmission) tickets, err := scheduler.EnqueueFileMutations(context.Background(), paths) + lap.finish() if err != nil { for _, path := range paths { outcomes[path] = mutationReindexOutcome{Err: err} @@ -1402,7 +1482,7 @@ func (s *Server) refreshBatchGraph(ctx context.Context, state *batchTransactionS file.ReindexReceipt = "" file.ReindexGeneration = 0 } - outcome := s.beginBatchGraphRefresh(file.Path) + outcome := s.beginBatchGraphRefresh(ctx, file.Path, receipt) outcomes[file.Path] = outcome file.ReindexReceipt = outcome.Receipt file.ReindexGeneration = outcome.Generation @@ -1491,6 +1571,7 @@ func (s *Server) finishBatchTransaction(state *batchTransactionState, receipt ba } func (s *Server) handleAtomicBatchEdit(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + s.noteAcknowledgementParser(ctx, "not_applicable_text_splice") args := req.GetArguments() transactionID, _ := args["transaction_id"].(string) statusOnly, _ := args["status_only"].(bool) @@ -1500,10 +1581,11 @@ func (s *Server) handleAtomicBatchEdit(ctx context.Context, req mcp.CallToolRequ if err != nil { return mcp.NewToolResultError(err.Error()), nil } + response := s.batchAcknowledgementResponse(receipt) if isCompact(req) { - return mcp.NewToolResultText(fmt.Sprintf("%s %s disk=%s graph=%s\n", receipt.TransactionID, receipt.Status, receipt.DiskStatus, receipt.GraphStatus)), nil + return mcp.NewToolResultText(fmt.Sprintf("%s %s disk=%s graph=%s\n", receipt.TransactionID, receipt.Status, receipt.DiskStatus, receipt.GraphStatus) + response.referenceLine()), nil } - return s.respondJSONOrTOON(ctx, req, receipt) + return s.respondJSONOrTOON(ctx, req, response) } edits, err := parseBatchEdits(rawEdits) @@ -1565,6 +1647,7 @@ func (s *Server) handleAtomicBatchEdit(ctx context.Context, req mcp.CallToolRequ if err != nil { return mcp.NewToolResultError(err.Error()), nil } + response := s.batchAcknowledgementResponse(receipt) if isCompact(req) { var out strings.Builder fmt.Fprintf(&out, "%s %s disk=%s graph=%s\n", receipt.TransactionID, receipt.Status, receipt.DiskStatus, receipt.GraphStatus) @@ -1575,7 +1658,8 @@ func (s *Server) handleAtomicBatchEdit(ctx context.Context, req mcp.CallToolRequ } fmt.Fprintf(&out, "%s %s %s\n", result.Op, target, result.Status) } + out.WriteString(response.referenceLine()) return mcp.NewToolResultText(out.String()), nil } - return s.respondJSONOrTOON(ctx, req, receipt) + return s.respondJSONOrTOON(ctx, req, response) } diff --git a/internal/mcp/callers_chain_scoped_golden_test.go b/internal/mcp/callers_chain_scoped_golden_test.go index 61cee207e..96ed5be85 100644 --- a/internal/mcp/callers_chain_scoped_golden_test.go +++ b/internal/mcp/callers_chain_scoped_golden_test.go @@ -69,6 +69,7 @@ func TestGetCallersAndCallChainScopedGoldenMultiRepo(t *testing.T) { production := payloads(server(store, true)) require.Equal(t, payloads(server(struct{ graph.Store }{store}, true)), production, "optional capabilities changed a scoped payload") unwrapped := payloads(server(store, false)) + require.Equal(t, payloads(server(struct{ graph.Store }{store}, false)), unwrapped, "optional capabilities changed a scoped payload containing cross-repository matches") var callers struct { Nodes []struct { @@ -91,4 +92,9 @@ func TestGetCallersAndCallChainScopedGoldenMultiRepo(t *testing.T) { } require.Contains(t, unwrapped[0], `"kind":"matches"`, "the unwrapped caller walk crosses the match") require.Contains(t, unwrapped[3], `"kind":"matches"`, "the unwrapped call chain crosses the match") + var unwrappedCallers struct { + Boundaries []json.RawMessage `json:"boundaries"` + } + require.NoError(t, json.Unmarshal([]byte(unwrapped[0]), &unwrappedCallers)) + require.NotEmpty(t, unwrappedCallers.Boundaries, "the unwrapped oracle comparison includes the interface dispatch boundary") } diff --git a/internal/mcp/change_contract_edit_paths_test.go b/internal/mcp/change_contract_edit_paths_test.go index 00a76746b..9605aeda4 100644 --- a/internal/mcp/change_contract_edit_paths_test.go +++ b/internal/mcp/change_contract_edit_paths_test.go @@ -107,8 +107,7 @@ func setupContractMultiRepoServer(t *testing.T, names ...string) (*Server, *grap g := graph.New() multi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), manager, zap.NewNop()) - _, err = multi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, multi, manager) srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil) srv.multiIndexer = multi diff --git a/internal/mcp/checkout_binding.go b/internal/mcp/checkout_binding.go index 1e2391137..baa0fcfd9 100644 --- a/internal/mcp/checkout_binding.go +++ b/internal/mcp/checkout_binding.go @@ -771,7 +771,10 @@ func (s *Server) checkoutForRequestPath(ctx context.Context, path string) (store return checkout, true, nil } } - if s.lifecycle == nil { + if s.lifecycle == nil || (s.multiIndexer != nil && s.multiIndexer.RepoIdentityPending()) { + // Registered catalog positives above remain usable. An unresolved configured + // identity is not a reason to allocate a checkout-observation job on a miss; + // startup admission will report identity-pending without guessing ownership. return store_sqlite.Checkout{}, false, parentErr } // CWD observation establishes a session's scope. A different explicit diff --git a/internal/mcp/contract_core_capability_table_test.go b/internal/mcp/contract_core_capability_table_test.go index f132beeed..cdd3d5f34 100644 --- a/internal/mcp/contract_core_capability_table_test.go +++ b/internal/mcp/contract_core_capability_table_test.go @@ -217,6 +217,7 @@ func contractCoreCapabilityTable() []contractCoreCapRow { {"LightEdgeSequencer", capOf[graph.LightEdgeSequencer], capDropped, capReasonEdges + " (graph.EdgesLightSeq falls back to the filtered EdgesByKind)"}, {"NodeLightSequencer", capOf[graph.NodeLightSequencer], capDropped, "node-only streaming; graph.NodesLightSeq falls back to the AllNodesLight forwarder, the same rows materialised"}, {"Store", capOf[graph.Store], capDropped, "the whole backend contract; a request reader is never the store"}, + {"local:WALReclaimStats", capOf[walReclaimSnapshotReader], capDropped, capReasonBackend + " (physical store counters, never selected graph state)"}, {"AnalysisGenerationStore", capOf[graph.AnalysisGenerationStore], capDropped, capReasonBackend}, {"AnalysisQueryStore", capOf[graph.AnalysisQueryStore], capDropped, capReasonBackend}, {"TryAnalysisQueryStore", capOf[graph.TryAnalysisQueryStore], capDropped, capReasonBackend}, diff --git a/internal/mcp/deadcode_processes_test.go b/internal/mcp/deadcode_processes_test.go index abfbd57be..7aae376a5 100644 --- a/internal/mcp/deadcode_processes_test.go +++ b/internal/mcp/deadcode_processes_test.go @@ -103,3 +103,98 @@ func TestDeadCodeGenerationProcessesReadOnlyAdmittedSteps(t *testing.T) { _, err = s.readDeadCodeProcesses(cancelled, spy, query, graph.AnalysisGenerationHeader{GenerationID: 7}) require.ErrorIs(t, err, context.Canceled) } + +// deadCodeFallbackEntrySpy separates entry-point lookups made through the +// request-bound reader from lookups on the unbound request reader. +type deadCodeFallbackEntrySpy struct { + graph.Reader + unbound, bound int + boundCtx context.Context + cancel context.CancelFunc +} + +func (r *deadCodeFallbackEntrySpy) GetNodesByIDs(ids []string) map[string]*graph.Node { + r.unbound++ + return r.Reader.GetNodesByIDs(ids) +} +func (r *deadCodeFallbackEntrySpy) BindReadContext(ctx context.Context) graph.Reader { + return &deadCodeFallbackBoundEntries{Reader: r.Reader, spy: r, ctx: ctx} +} + +type deadCodeFallbackBoundEntries struct { + graph.Reader + spy *deadCodeFallbackEntrySpy + ctx context.Context +} + +// GetNodesByIDs behaves like a bound store whose lookup stops with the +// request: once ctx ends it returns no nodes and no error. +func (r *deadCodeFallbackBoundEntries) GetNodesByIDs(ids []string) map[string]*graph.Node { + r.spy.bound++ + r.spy.boundCtx = r.ctx + if r.spy.cancel != nil { + r.spy.cancel() + } + if r.ctx.Err() != nil { + return nil + } + return r.Reader.GetNodesByIDs(ids) +} + +func TestDeadCodeFallbackProcessAdmissionStopsWithTheRequest(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: "entry-a", RepoPrefix: "a"}) + g.AddNode(&graph.Node{ID: "entry-b", RepoPrefix: "b"}) + s := &Server{graph: g, session: newSessionState()} + installProcessesForTest(s, &analysis.ProcessResult{Processes: []analysis.Process{ + {ID: "process-a", Name: "a flow", EntryPoint: "entry-a", StepCount: 1}, + {ID: "process-b", Name: "b flow", EntryPoint: "entry-b", StepCount: 1}, + }}) + ctx := withRepoAllow(t.Context(), map[string]bool{"a": true}) + require.NotNil(t, s.getProcesses(), "fixture: the process snapshot must be current") + _, _, current := s.activeAnalysisQueryForAnswer() + require.False(t, current, "fixture: without a current analysis generation, admission takes the fallback") + + spy := &deadCodeFallbackEntrySpy{Reader: g} + result, err := s.deadCodeProcesses(ctx, spy) + require.NoError(t, err) + require.Len(t, result.Processes, 1) + require.Equal(t, "process-a", result.Processes[0].ID) + require.Zero(t, spy.unbound, "entry points must not be read through the unbound request reader") + require.Equal(t, 1, spy.bound) + require.Same(t, ctx, spy.boundCtx, "the lookup must be bound to the request context") + + ended, cancel := context.WithCancel(ctx) + spy = &deadCodeFallbackEntrySpy{Reader: g, cancel: cancel} + result, err = s.deadCodeProcesses(ended, spy) + require.ErrorIs(t, err, context.Canceled, "a request that ends during admission must not admit an empty process set") + require.Nil(t, result) + require.Zero(t, spy.unbound) +} + +// deadCodeFailingEntries is a reader whose cancellable entry-point lookup +// fails for a reason other than the request ending. +type deadCodeFailingEntries struct { + graph.Reader + err error +} + +func (r deadCodeFailingEntries) GetNodesByIDsContext(context.Context, []string) (map[string]*graph.Node, error) { + return nil, r.err +} + +func TestDeadCodeFallbackProcessAdmissionReportsALookupFailure(t *testing.T) { + g := graph.New() + g.AddNode(&graph.Node{ID: "entry-a", RepoPrefix: "a"}) + s := &Server{graph: g, session: newSessionState()} + installProcessesForTest(s, &analysis.ProcessResult{Processes: []analysis.Process{ + {ID: "process-a", Name: "a flow", EntryPoint: "entry-a", StepCount: 1}, + }}) + _, _, current := s.activeAnalysisQueryForAnswer() + require.False(t, current, "fixture: without a current analysis generation, admission takes the fallback") + + failure := errors.New("entry lookup failed") + result, err := s.deadCodeProcesses(withRepoAllow(t.Context(), map[string]bool{"a": true}), deadCodeFailingEntries{Reader: g, err: failure}) + require.ErrorIs(t, err, failure, "a failed scope lookup must not admit a silently empty process set") + require.Nil(t, result) +} diff --git a/internal/mcp/deadcode_scope.go b/internal/mcp/deadcode_scope.go index 859dd7d7d..098b4888d 100644 --- a/internal/mcp/deadcode_scope.go +++ b/internal/mcp/deadcode_scope.go @@ -73,7 +73,10 @@ func (s *Server) deadCodeProcesses(ctx context.Context, reader graph.Reader) (*a for _, process := range processes.Processes { ids = append(ids, process.EntryPoint) } - nodes := reader.GetNodesByIDs(ids) + nodes, err := deadCodeEntryNodes(ctx, reader, ids) + if err != nil { + return nil, err + } filtered := *processes filtered.Processes = nil for _, process := range processes.Processes { @@ -105,18 +108,8 @@ func (s *Server) readDeadCodeProcesses(ctx context.Context, reader graph.Reader, for _, row := range rows { ids = append(ids, row.EntryPoint) } - var nodes map[string]*graph.Node - if checked, ok := reader.(interface { - GetNodesByIDsContext(context.Context, []string) (map[string]*graph.Node, error) - }); ok { - nodes, err = checked.GetNodesByIDsContext(ctx, ids) - if err != nil { - return nil, err - } - } else { - nodes = graph.BindReadContext(reader, ctx).GetNodesByIDs(ids) - } - if err := ctx.Err(); err != nil { + nodes, err := deadCodeEntryNodes(ctx, reader, ids) + if err != nil { return nil, err } for _, row := range rows { @@ -150,3 +143,25 @@ func (s *Server) readDeadCodeProcesses(ctx context.Context, reader graph.Reader, } return result, ctx.Err() } + +// deadCodeEntryNodes reads process entry points for scope admission under the +// request: a cancellable batch lookup when the reader has one, the reader +// bound to ctx otherwise. A bound lookup that stops with the request returns +// no nodes, so an ended request reports ctx's error instead of admitting none. +func deadCodeEntryNodes(ctx context.Context, reader graph.Reader, ids []string) (map[string]*graph.Node, error) { + var nodes map[string]*graph.Node + if checked, ok := reader.(interface { + GetNodesByIDsContext(context.Context, []string) (map[string]*graph.Node, error) + }); ok { + var err error + if nodes, err = checked.GetNodesByIDsContext(ctx, ids); err != nil { + return nil, err + } + } else { + nodes = graph.BindReadContext(reader, ctx).GetNodesByIDs(ids) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return nodes, nil +} diff --git a/internal/mcp/diff_repo_scope_test.go b/internal/mcp/diff_repo_scope_test.go index 213da63c0..4036d9199 100644 --- a/internal/mcp/diff_repo_scope_test.go +++ b/internal/mcp/diff_repo_scope_test.go @@ -42,8 +42,10 @@ func TestDiffRepoScope(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/dispatch_acknowledgement_integration_test.go b/internal/mcp/dispatch_acknowledgement_integration_test.go new file mode 100644 index 000000000..650c80410 --- /dev/null +++ b/internal/mcp/dispatch_acknowledgement_integration_test.go @@ -0,0 +1,60 @@ +package mcp + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/indexer" +) + +func TestEditAcknowledgementIncludesSlowWaitAdmission(t *testing.T) { + for _, batch := range []bool{false, true} { + t.Run(map[bool]string{false: "single", true: "batch"}[batch], func(t *testing.T) { + s := newMutationServer() + done := make(chan indexer.MutationResult) + s.watcher = mutationTestWatcher{done: done} + ctx, ack, _ := s.withEditAcknowledgement(canonicalTimingContext()) + path := filepath.Join(t.TempDir(), "pending.go") + t.Cleanup(func() { + close(done) + s.mutationReceipts.Range(func(_, value any) bool { + if receipt, ok := value.(*mutationReceipt); ok { + select { + case <-receipt.done: + case <-time.After(5 * time.Second): + t.Error("owned ticket consumer did not finish") + } + } + return true + }) + }) + called := false + s.releaseDispatchSlot = func(context.Context, <-chan struct{}) bool { + called = true + lap := ack.snapshot().ReindexWait + require.EqualValues(t, 1, lap.Started, "slow-lane admission is part of the actual passive wait") + require.Zero(t, lap.Completed) + return false // capacity refusal must leave the committed receipt pending + } + if batch { + receipt := s.trackMutationTicket(&indexer.MutationTicket{Path: path, Generation: 1, Done: done}) + s.waitBatchGraphReceipts(ctx, []batchTransactionFile{{Path: path, ReindexReceipt: receipt.id}}) + require.True(t, receipt.outcome(true).Pending) + } else { + commitTimingFixture(t, s, ctx, path, "package p\n") + outcome := s.mutationReindexState(ctx, path) + require.NoError(t, outcome.Err) + require.True(t, outcome.Pending) + require.NotEmpty(t, outcome.Receipt) + } + require.True(t, called) + lap := ack.snapshot().ReindexWait + require.EqualValues(t, 1, lap.Started) + require.EqualValues(t, 1, lap.Completed, "refused passive-wait admission must close the lap") + }) + } +} diff --git a/internal/mcp/dispatch_classification.go b/internal/mcp/dispatch_classification.go new file mode 100644 index 000000000..c6df19c86 --- /dev/null +++ b/internal/mcp/dispatch_classification.go @@ -0,0 +1,46 @@ +package mcp + +import ( + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/zzet/gortex/internal/daemon" +) + +// DispatchClass classifies a tool before admission using the same facade +// operation selection and registry as execution. It does not resolve symbols, +// acquire views, promote tools, change session state or run a handler. +func (s *Server) DispatchClass(name string, arguments map[string]any) daemon.MCPDispatchClass { + // Native kinds and facade aliases of analyze all use the slow lane. + if name == "analyze" { + return daemon.MCPDispatchSlow + } + class, _ := daemon.ToolDispatchClass(name) + if !isFacadeToolName(name) || s == nil || s.facades == nil { + return class + } + req := mcpgo.CallToolRequest{} + req.Params.Arguments = arguments + operation := facadeRequestOperation(name, req) + if name == "session" && (operation == "subscribe" || operation == "unsubscribe") { + channel := normalizeFacadeOperation(req.GetString("channel", "")) + if !validFacadeSessionChannel(channel) { + return daemon.MCPDispatchFast + } + operation += "_" + channel + } + spec, known := s.facades.operation(name, operation) + if !known { + return daemon.MCPDispatchFast + } + switch spec.Effect { + case facadeEffectLocalWrite, facadeEffectControlWrite, facadeEffectExternalWrite: + return daemon.MCPDispatchMutating + } + class, _ = daemon.ToolDispatchClass(spec.Legacy) + if class == daemon.MCPDispatchSlow { + return class + } + // Read/session facades fix the conditional legacy write arguments, e.g. + // change.contract ack=false and analyze.co_change refresh=false. Never + // inherit a legacy mutation reservation for that effect-safe operation. + return daemon.MCPDispatchFast +} diff --git a/internal/mcp/dispatch_classification_test.go b/internal/mcp/dispatch_classification_test.go new file mode 100644 index 000000000..422a0995f --- /dev/null +++ b/internal/mcp/dispatch_classification_test.go @@ -0,0 +1,145 @@ +package mcp + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "strconv" + "strings" + "testing" + + "github.com/zzet/gortex/internal/daemon" +) + +// Some handlers are registered only when an optional service is installed. +// Inspect their real declarations too, so adding an optional tool cannot +// silently evade the runtime full/deferred catalogue census below. +func TestDispatchClassCoversOptionalToolDeclarations(t *testing.T) { + fset := token.NewFileSet() + entries, err := os.ReadDir(".") + if err != nil { + t.Fatal(err) + } + files := make(map[string]*ast.File) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + file, err := parser.ParseFile(fset, entry.Name(), nil, 0) + if err != nil { + t.Fatal(err) + } + if file.Name.Name == "mcp" { + files[entry.Name()] = file + } + } + constants := map[string]string{} + for _, file := range files { + for _, declaration := range file.Decls { + group, ok := declaration.(*ast.GenDecl) + if !ok || group.Tok != token.CONST { + continue + } + for _, item := range group.Specs { + value := item.(*ast.ValueSpec) + for index, expression := range value.Values { + literal, ok := expression.(*ast.BasicLit) + if ok && literal.Kind == token.STRING && index < len(value.Names) { + constants[value.Names[index].Name], _ = strconv.Unquote(literal.Value) + } + } + } + } + } + for _, file := range files { + for _, declaration := range file.Decls { + function, ok := declaration.(*ast.FuncDecl) + if !ok || function.Body == nil { + continue + } + ast.Inspect(function.Body, func(node ast.Node) bool { + call, ok := node.(*ast.CallExpr) + if !ok || len(call.Args) == 0 { + return true + } + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok || selector.Sel.Name != "NewTool" { + return true + } + name := "" + switch argument := call.Args[0].(type) { + case *ast.BasicLit: + name, _ = strconv.Unquote(argument.Value) + case *ast.Ident: + name = constants[argument.Name] + if argument.Name == "name" && function.Name.Name == "facadeToolDefinitionWithOperations" { + for _, facade := range facadeToolNames() { + if _, known := daemon.ToolDispatchClass(facade); !known { + t.Errorf("facade %q has no class", facade) + } + } + return true + } + } + if _, known := daemon.ToolDispatchClass(name); !known { + t.Errorf("tool declaration at %s has no class (resolved name %q)", fset.Position(call.Pos()), name) + } + return true + }) + } + } +} + +func TestDispatchClassCoversRegisteredAndFacadeTools(t *testing.T) { + for _, config := range []ToolPolicyConfig{{Preset: "full"}, {Preset: "core", Mode: "defer"}} { + srv := setupPresetServer(t, config) + for _, name := range srv.registeredToolNames() { + if _, known := daemon.ToolDispatchClass(name); !known { + t.Errorf("registered tool %q has no explicit dispatch class", name) + } + } + for _, spec := range facadeOperationSpecs() { + if _, known := daemon.ToolDispatchClass(spec.Legacy); !known { + t.Errorf("facade %s.%s implementation %q has no dispatch class", spec.Facade, spec.Operation, spec.Legacy) + } + } + } +} + +func TestDispatchClassUsesEffectiveFacadeOperation(t *testing.T) { + srv := setupPresetServer(t, ToolPolicyConfig{Preset: "full"}) + cases := []struct { + name string + args map[string]any + want daemon.MCPDispatchClass + }{ + {"edit_file", nil, daemon.MCPDispatchMutating}, + {"edit", map[string]any{"operation": "file"}, daemon.MCPDispatchMutating}, + {"refactor", map[string]any{"operation": "rename"}, daemon.MCPDispatchMutating}, + {"read", map[string]any{"operation": "file", "target": map[string]any{"file": "just-edited.go"}}, daemon.MCPDispatchFast}, + {"explore", map[string]any{"operation": "outline"}, daemon.MCPDispatchSlow}, + {"analyze", map[string]any{"kind": "dead_code"}, daemon.MCPDispatchSlow}, + {"analyze", map[string]any{"kind": "processes"}, daemon.MCPDispatchSlow}, + {"analyze", map[string]any{"kind": "help"}, daemon.MCPDispatchSlow}, + {"workspace_admin", map[string]any{"operation": "enrich_churn"}, daemon.MCPDispatchMutating}, + {"change_contract", map[string]any{"ack": true}, daemon.MCPDispatchMutating}, + {"change", map[string]any{"operation": "contract", "options": map[string]any{"ack": true}}, daemon.MCPDispatchFast}, + {"find_co_changing_symbols", nil, daemon.MCPDispatchMutating}, + {"analyze", map[string]any{"kind": "co_change", "options": map[string]any{"refresh": true}}, daemon.MCPDispatchSlow}, + {"recall", map[string]any{"operation": "surface"}, daemon.MCPDispatchFast}, + {"remember", map[string]any{"operation": "risk_ack"}, daemon.MCPDispatchMutating}, + {"overlay", map[string]any{"operation": "merge", "arguments": map[string]any{"to_disk": true}}, daemon.MCPDispatchFast}, + {"read", map[string]any{"operation": "symbol", "target": map[string]any{"symbol": "demo::f"}}, daemon.MCPDispatchFast}, + {"session", map[string]any{"operation": "subscribe", "channel": "diagnostics"}, daemon.MCPDispatchFast}, + {"edit", map[string]any{"operation": "not_registered"}, daemon.MCPDispatchFast}, + {"not_registered", nil, daemon.MCPDispatchFast}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + if got := srv.DispatchClass(test.name, test.args); got != test.want { + t.Errorf("class=%s want=%s", got, test.want) + } + }) + } +} diff --git a/internal/mcp/dispatch_slot_release_test.go b/internal/mcp/dispatch_slot_release_test.go index 4d18dad6e..3dd812323 100644 --- a/internal/mcp/dispatch_slot_release_test.go +++ b/internal/mcp/dispatch_slot_release_test.go @@ -29,7 +29,7 @@ type slotReleaseRecorder struct { // recordSlotReleases stands in for the daemon's slot release on s. func recordSlotReleases(s *Server, receipt func() *mutationReceipt) *slotReleaseRecorder { r := &slotReleaseRecorder{called: make(chan struct{}, 8)} - s.releaseDispatchSlot = func(context.Context) bool { + s.releaseDispatchSlot = func(context.Context, <-chan struct{}) bool { var seen slotRelease if tracked := receipt(); tracked != nil { seen.tracked = true @@ -217,6 +217,87 @@ func TestWaitBatchGraphReceiptsReturnsTheDispatchSlotOnlyWhileAReceiptIsPending( }) require.Empty(t, recorder.calls()) }) + + t.Run("published_while_queued", func(t *testing.T) { + s := &Server{mutationReindexWait: time.Minute} + first, publishFirst := track(t, s, caller, 1) + second, publishSecond := track(t, s, target, 2) + queued := make(chan struct{}) + // Stands in for a slow-lane admission that stays queued for the whole + // budget unless the wait it is for ends first. + s.releaseDispatchSlot = func(_ context.Context, ended <-chan struct{}) bool { + close(queued) + select { + case <-ended: + return true + case <-time.After(5 * time.Second): + return false + } + } + done := make(chan struct{}) + go func() { + defer close(done) + s.waitBatchGraphReceipts(context.Background(), []batchTransactionFile{ + {Path: caller, ReindexReceipt: first.id}, + {Path: target, ReindexReceipt: second.id}, + }) + }() + select { + case <-queued: + case <-time.After(2 * time.Second): + t.Fatal("the batch wait never asked for slow-lane capacity") + } + publishFirst() + published(t, first) + select { + case <-done: + t.Fatal("the batch wait ended while a receipt was still pending") + case <-time.After(20 * time.Millisecond): + } + publishSecond() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("the batch wait stayed queued after every receipt published") + } + }) +} + +func TestMutationReindexStateStopsQueueingOncePublished(t *testing.T) { + path := filepath.Join(t.TempDir(), "edited.go") + ctx, cancel, intent, w, _, complete := primaryPublicationIntentFixture(t, nil, path) + defer cancel() + defer func() { complete(); intent.close() }() + s := &Server{watcher: w, mutationReindexWait: time.Minute} + queued := make(chan struct{}) + // Stands in for a slow-lane admission that stays queued for the whole + // budget unless the wait it is for ends first. + s.releaseDispatchSlot = func(_ context.Context, published <-chan struct{}) bool { + close(queued) + select { + case <-published: + return true + case <-time.After(5 * time.Second): + return false + } + } + outcomes := make(chan mutationReindexOutcome, 1) + go func() { outcomes <- s.mutationReindexState(ctx, path) }() + select { + case <-queued: + case <-time.After(2 * time.Second): + t.Fatal("the edit wait never asked for slow-lane capacity") + } + complete() + select { + case outcome := <-outcomes: + require.NoError(t, outcome.Err) + require.True(t, outcome.Reindexed) + require.False(t, outcome.Pending) + require.Empty(t, outcome.Receipt) + case <-time.After(time.Second): + t.Fatal("a published edit stayed queued for slow-lane capacity") + } } func TestRenameReturnsTheDispatchSlotOnlyAtItsLastFile(t *testing.T) { @@ -232,7 +313,7 @@ func TestRenameReturnsTheDispatchSlotOnlyAtItsLastFile(t *testing.T) { s := newMutationServer() s.watcher = mutationTestWatcher{} var lastWritten []bool - s.releaseDispatchSlot = func(context.Context) bool { + s.releaseDispatchSlot = func(context.Context, <-chan struct{}) bool { content, err := os.ReadFile(writes[len(writes)-1].AbsPath) lastWritten = append(lastWritten, err == nil && string(content) == "package p // new\n") return true @@ -245,3 +326,21 @@ func TestRenameReturnsTheDispatchSlotOnlyAtItsLastFile(t *testing.T) { } require.Equal(t, []bool{true}, lastWritten, "one release, once the last file is written") } + +func TestMutationReindexStateKeepsReceiptWhenSlowWaitAdmissionUnavailable(t *testing.T) { + path := filepath.Join(t.TempDir(), "edited.go") + ctx, cancel, intent, w, _, complete := primaryPublicationIntentFixture(t, nil, path) + defer cancel() + defer func() { complete(); intent.close() }() + committed := []byte("package fixture\nfunc Committed(){}\n") + require.NoError(t, os.WriteFile(path, committed, 0600)) + s := &Server{watcher: w, mutationReindexWait: time.Minute} + s.releaseDispatchSlot = func(context.Context, <-chan struct{}) bool { return false } + outcome := s.mutationReindexState(ctx, path) + require.True(t, outcome.Pending) + require.NotEmpty(t, outcome.Receipt) + require.NoError(t, outcome.Err, "slow lane admission cannot turn committed bytes into a retryable busy error") + actual, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, committed, actual) +} diff --git a/internal/mcp/edit_acknowledgement.go b/internal/mcp/edit_acknowledgement.go new file mode 100644 index 000000000..c0197166b --- /dev/null +++ b/internal/mcp/edit_acknowledgement.go @@ -0,0 +1,200 @@ +package mcp + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/mark3labs/mcp-go/mcp" + + "github.com/zzet/gortex/internal/daemon" +) + +// This is a call-scoped private observation, not retained receipt history. In +// particular it must not be stored in the unbounded batchTransactions map. +// Its origin precedes boundHandler, not the dispatcher/transport or the legacy +// publication origin at middleware entry. No graph work is inferred from it. +type editAcknowledgement struct { + owner *Server + id uint64 + entered time.Time + mu sync.Mutex + actual time.Time + reply time.Time + outcome string + laps editAcknowledgementLaps + envelope editAcknowledgementEnvelope + dispatch daemon.MCPDispatchTiming + hasDispatch bool + parserScope string +} + +type editAcknowledgementKey struct{} + +var editAcknowledgementSequence atomic.Uint64 + +// A status call may borrow this pointer long enough to copy a snapshot. It +// never installs one: eviction clears the record's ownership even when other +// pending-source evidence still retains the physical commit record. +func retainedMutationAcknowledgement(record *mutationCommitRecord) *editAcknowledgement { + if !record.isPhysicalCommit() { + return nil + } + record.mu.RLock() + defer record.mu.RUnlock() + return record.acknowledgement +} + +// The two walls deliberately overlap: they are diagnostic alternatives, never +// additive laps. A detached handler can finish after the reply wall froze. +// Snapshot values own their scalar pointers; no encoder samples a live clock. +type editAcknowledgementSnapshot struct { + RPCID uint64 + Scope string + ReplyWallMS *float64 + ReplyState string + ReplyOutcome string + ActualWallMS *float64 + ActualState string + ActualFinishedLate bool + DispatcherQueueMS *float64 + DispatcherQueueState string + DispatcherPreambleMS *float64 + DispatcherState string + DispatcherReplyMS *float64 + DispatcherReplyState string + ParseGate editAcknowledgementLapSnapshot + DiskWrite editAcknowledgementLapSnapshot + TicketAdmission editAcknowledgementLapSnapshot + ReindexWait editAcknowledgementLapSnapshot + Middleware editAcknowledgementLapSnapshot + ReplyPrepare editAcknowledgementLapSnapshot + EditControl editAcknowledgementLapSnapshot + CallerResult editAcknowledgementLapSnapshot + ParserScope string + LapsDisjoint bool + replyNS int64 + hasReplyNS bool +} + +func editAcknowledgementFrom(ctx context.Context, owner *Server) *editAcknowledgement { + if ctx == nil { + return nil + } + ack, _ := ctx.Value(editAcknowledgementKey{}).(*editAcknowledgement) + if ack != nil && ack.owner == owner { + return ack + } + return nil +} + +// withEditAcknowledgement is called only after the existing measured-source +// classifier succeeded. A same-Server nested raw/facade call borrows the RPC +// identity; another Server cannot complete or freeze that owner's record. +func (s *Server) withEditAcknowledgement(ctx context.Context) (context.Context, *editAcknowledgement, bool) { + if existing := editAcknowledgementFrom(ctx, s); existing != nil { + return ctx, existing, false + } + dispatch, hasDispatch := daemon.MCPDispatchTimingForContext(ctx) + ack := &editAcknowledgement{owner: s, id: editAcknowledgementSequence.Add(1), entered: time.Now(), dispatch: dispatch, hasDispatch: hasDispatch} + return context.WithValue(ctx, editAcknowledgementKey{}, ack), ack, true +} + +func (a *editAcknowledgement) finishActual() { + a.finishActualAt(time.Now()) +} + +func (a *editAcknowledgement) finishActualAt(at time.Time) { + a.mu.Lock() + if a.actual.IsZero() { + a.actual = at + } + a.mu.Unlock() +} + +func (a *editAcknowledgement) finishReply(result *mcp.CallToolResult, err error) { + outcome := "result" + switch { + case err != nil: + outcome = "go_error" + case result == nil: + outcome = "no_result" + case result.IsError: + outcome = "tool_error" + } + a.finishReplyAt(time.Now(), outcome) +} + +func (a *editAcknowledgement) finishReplyAt(at time.Time, outcome string) { + a.mu.Lock() + if a.reply.IsZero() { + a.reply, a.outcome = at, outcome + } + a.mu.Unlock() +} + +func (a *editAcknowledgement) snapshot() editAcknowledgementSnapshot { + a.mu.Lock() + actual, reply, outcome, laps, envelope, parserScope := a.actual, a.reply, a.outcome, a.laps, a.envelope, a.parserScope + a.mu.Unlock() + result := editAcknowledgementSnapshot{ + RPCID: a.id, Scope: "mcp_bound_call", ReplyOutcome: outcome, + DispatcherQueueState: "unavailable_dispatcher_interface", + DispatcherState: "unavailable_dispatcher_interface", + DispatcherReplyState: "unavailable_dispatcher_interface", + } + result.ReplyWallMS, result.ReplyState = editAcknowledgementWall(a.entered, reply) + if result.ReplyState == "measured" { + result.replyNS, result.hasReplyNS = reply.Sub(a.entered).Nanoseconds(), true + } + result.ActualWallMS, result.ActualState = editAcknowledgementWall(a.entered, actual) + result.ActualFinishedLate = result.ReplyState == "measured" && result.ActualState == "measured" && actual.After(reply) + result.ParseGate = laps.snapshot(editAckParseGate, a.entered, reply, actual) + result.ParserScope = parserScope + if result.ParseGate.Started == 0 && (parserScope == "not_applicable_text_splice" || parserScope == "unsupported_non_go_move") { + result.ParseGate.State = "not_applicable" + } + result.DiskWrite = laps.snapshot(editAckDiskWrite, a.entered, reply, actual) + result.TicketAdmission = laps.snapshot(editAckTicketAdmission, a.entered, reply, actual) + result.ReindexWait = laps.snapshot(editAckReindexWait, a.entered, reply, actual) + result.EditControl = laps.snapshot(editAckLeafControl, a.entered, reply, actual) + result.CallerResult = acknowledgementCallerResult(a.entered, actual, reply) + result.LapsDisjoint = !laps.overlap + if a.hasDispatch { + if a.dispatch.Granted.Before(a.dispatch.Entered) || a.entered.Before(a.dispatch.Granted) { + result.DispatcherQueueState, result.DispatcherState = "invalid_interval", "invalid_interval" + result.DispatcherReplyState = "invalid_interval" + } else { + result.DispatcherQueueMS, result.DispatcherQueueState = editAcknowledgementWall(a.dispatch.Entered, a.dispatch.Granted) + result.DispatcherPreambleMS, result.DispatcherState = editAcknowledgementWall(a.dispatch.Granted, a.entered) + result.DispatcherReplyMS, result.DispatcherReplyState = editAcknowledgementWall(a.dispatch.Entered, reply) + } + } + result.Middleware, result.ReplyPrepare = envelope.snapshot(a.entered, reply, actual) + // These outer intervals surround, rather than contain, the leaf's fixed + // operation laps. A misplaced operation must not produce a plausible sum. + if envelope.observed { + for _, lap := range laps.data { + if lap.started > 0 && (envelope.leafStarted.IsZero() || lap.first.Before(envelope.leafStarted) || + (!envelope.leafFinished.IsZero() && lap.last.After(envelope.leafFinished))) { + result.LapsDisjoint = false + } + } + } + return result +} + +func editAcknowledgementWall(start, end time.Time) (*float64, string) { + if start.IsZero() { + return nil, "unavailable_origin" + } + if end.IsZero() { + return nil, "pending" + } + if end.Before(start) { + return nil, "invalid_order" + } + ms := float64(end.Sub(start)) / float64(time.Millisecond) + return &ms, "measured" +} diff --git a/internal/mcp/edit_acknowledgement_baseline_test.go b/internal/mcp/edit_acknowledgement_baseline_test.go new file mode 100644 index 000000000..d7afcb01d --- /dev/null +++ b/internal/mcp/edit_acknowledgement_baseline_test.go @@ -0,0 +1,51 @@ +package mcp + +import ( + "context" + "fmt" + "time" + + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// Frozen pre-acknowledgement allocation control from c3cc3e8184e1c29575ed0e25241c0e9cdbf8ec6e: +// tool_deadline.go boundToolHandler and edit_operation_observations.go +// observeEditOperationHandler, including the original callbacks and witness +// wrapper. Only these helper names and their reference are changed. A direct +// boundHandler with nil callbacks omits existing per-call work and is not the +// no-additional-allocation baseline. This is not an old full-dispatch benchmark. +func (s *Server) preAcknowledgementBoundToolHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHandlerFunc { + // This wrapper runs INSIDE boundHandler's actual execution, so answering + // an abandoned caller cannot count an unfinished source operation. + h = s.preAcknowledgementObserveEditOperationHandler(h) + return func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + name := req.Params.Name + return boundHandler(s, "tool", name, (func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error))(h), + func(stuck int64, timeout time.Duration) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(busyMessage("tool calls", stuck, timeout)), nil + }, + func(timeout time.Duration, note *mutationCommitNote, retained *requestViewPin) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(abandonedToolMessage(name, timeout, note.verdict(), retained)), nil + }, + func(r any) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf("tool %q internal error: %v", name, r)), nil + }, + )(ctx, req) + } +} + +func (s *Server) preAcknowledgementObserveEditOperationHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, req mcp.CallToolRequest) (result *mcp.CallToolResult, err error) { + if s == nil || !measuredSourceEdit(req) { + return h(ctx, req) + } + if existing := editOperationFrom(ctx); existing != nil && existing.owner == &s.editOperations { + return h(ctx, req) // a raw/facade nested handler borrows the owner + } + w := &editOperationWitness{owner: &s.editOperations} + ctx = context.WithValue(ctx, editOperationContextKey{}, w) + defer func() { w.finish(result, err) }() + return h(ctx, req) + } +} diff --git a/internal/mcp/edit_acknowledgement_control.go b/internal/mcp/edit_acknowledgement_control.go new file mode 100644 index 000000000..261646aa8 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_control.go @@ -0,0 +1,83 @@ +package mcp + +import "time" + +// Actual leaf preparation/validation/formatting/receipt/response control work +// between the fixed named operations. These are online monotonic intervals, +// never a residual or retained history. All helpers require a.mu. +func (a *editAcknowledgement) openEditControlLocked(at time.Time) { + laps := &a.laps + if !laps.controlObserved { + return + } + data := &laps.data[editAckLeafControl] + if at.IsZero() || at.Before(a.entered) || laps.active != 0 || !laps.lastEnd.IsZero() && at.Before(laps.lastEnd) { + data.invalid = true + laps.overlap = true + return + } + laps.sequence++ + laps.active, laps.activeKind = laps.sequence, editAckLeafControl + laps.controlStarted = at + data.started++ + if data.first.IsZero() { + data.first = at + } +} + +func (a *editAcknowledgement) closeEditControlLocked(at time.Time) { + laps := &a.laps + if !laps.controlObserved { + return + } + data := &laps.data[editAckLeafControl] + if laps.active == 0 || laps.activeKind != editAckLeafControl || laps.controlStarted.IsZero() { + data.invalid = true + laps.overlap = true + return + } + data.completed++ + if at.IsZero() || at.Before(laps.controlStarted) { + data.invalid = true + } else { + data.elapsed += at.Sub(laps.controlStarted) + } + data.last = at + laps.lastEnd = at + laps.controlStarted = time.Time{} + laps.active = 0 +} + +func (a *editAcknowledgement) leafControlOpenLocked() bool { + return a.laps.controlObserved && !a.envelope.leafStarted.IsZero() && a.envelope.leafFinished.IsZero() +} + +// The bound handler's actual finish and caller-facing reply finish already +// have genuine monotonic readings. This separate delivery/finalization pair +// starts where reply_prepare ends and excludes transport serialization. A +// detached physical handler finishing later is not clipped into the ACK. +func acknowledgementCallerResult(entered, actual, reply time.Time) editAcknowledgementLapSnapshot { + out := editAcknowledgementLapSnapshot{State: "unobserved"} + if actual.IsZero() { + return out + } + out.Started = 1 + if entered.IsZero() || actual.Before(entered) { + out.State = "invalid_interval" + return out + } + if reply.IsZero() { + out.State = "pending_reply" + return out + } + out.Completed = 1 + if actual.After(reply) { + out.State = "outside_reply_window" + return out + } + ms := float64(reply.Sub(actual)) / float64(time.Millisecond) + ackMS := ms + out.ObservedMS, out.AckMS, out.State = &ms, &ackMS, "measured" + out.ackNS, out.hasAckNS = reply.Sub(actual).Nanoseconds(), true + return out +} diff --git a/internal/mcp/edit_acknowledgement_control_test.go b/internal/mcp/edit_acknowledgement_control_test.go new file mode 100644 index 000000000..072842ae4 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_control_test.go @@ -0,0 +1,169 @@ +package mcp + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func beginControlledLeaf(t *testing.T, origin time.Time) *editAcknowledgement { + t.Helper() + a := &editAcknowledgement{entered: origin, envelope: editAcknowledgementEnvelope{observed: true, leafStarted: origin.Add(time.Millisecond)}} + a.mu.Lock() + a.laps.controlObserved = true + a.openEditControlLocked(a.envelope.leafStarted) + a.mu.Unlock() + return a +} + +func endControlledLeaf(a *editAcknowledgement, at time.Time) { + a.mu.Lock() + defer a.mu.Unlock() + a.envelope.leafFinished = at + a.closeEditControlLocked(at) +} + +func TestEditControlAccumulatesActualOnlineABALeafIntervals(t *testing.T) { + origin := time.Now() + a := beginControlledLeaf(t, origin) + for _, chunk := range []struct { + kind editAcknowledgementLapKind + start, end time.Duration + }{ + {editAckParseGate, 2, 4}, {editAckDiskWrite, 6, 7}, {editAckParseGate, 9, 10}, + } { + lap := a.beginLapAt(chunk.kind, origin.Add(chunk.start*time.Millisecond)) + lap.finishAt(origin.Add(chunk.end * time.Millisecond)) + } + endControlledLeaf(a, origin.Add(12*time.Millisecond)) + a.finishActualAt(origin.Add(14 * time.Millisecond)) + a.finishReplyAt(origin.Add(15*time.Millisecond), "result") + snapshot := a.snapshot() + require.True(t, snapshot.LapsDisjoint) + require.Equal(t, 3.0, *snapshot.ParseGate.AckMS) + require.Equal(t, 1.0, *snapshot.DiskWrite.AckMS) + require.Equal(t, 7.0, *snapshot.EditControl.AckMS, "actual1→2,4→6,7→9,10→12 intervals") + require.EqualValues(t, 4, snapshot.EditControl.Started) + require.EqualValues(t, 4, snapshot.EditControl.Completed) + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Equal(t, 15.0, *ack.SubtotalMS) + require.Equal(t, 0.0, *ack.UnattributedMS, "actual caller-result pair closes this controlled ACK; no residual becomes work") + require.Equal(t, "partial", ack.SumState) + require.Equal(t, 7.0, *ack.PhasesMS["edit_control"]) + *snapshot.EditControl.AckMS = 999 + require.Equal(t, 7.0, *a.snapshot().EditControl.AckMS, "snapshots own scalar values") +} + +func TestEditControlRecordsRealNotApplicableGuardWithoutInventingParse(t *testing.T) { + origin := time.Now() + a := beginControlledLeaf(t, origin) + gate := a.beginLapAt(editAckParseGate, origin.Add(2*time.Millisecond)) + gate.applicable = false + gate.finishAt(origin.Add(4 * time.Millisecond)) + endControlledLeaf(a, origin.Add(5*time.Millisecond)) + a.finishActualAt(origin.Add(6 * time.Millisecond)) + a.finishReplyAt(origin.Add(7*time.Millisecond), "result") + snapshot := a.snapshot() + require.True(t, snapshot.LapsDisjoint) + require.Equal(t, "not_applicable", snapshot.ParseGate.State) + require.Nil(t, snapshot.ParseGate.AckMS) + require.Equal(t, 4.0, *snapshot.EditControl.AckMS, "guard dispatch is an actually observed control interval") + require.Equal(t, 7.0, *buildAcknowledgementPayload(&snapshot, "observed").SubtotalMS) +} + +func TestEditControlMissingNamedEndOrReversedTransitionCannotProduceCoverage(t *testing.T) { + for _, name := range []string{"missing_end", "reversed", "overlap"} { + t.Run(name, func(t *testing.T) { + origin := time.Now() + a := beginControlledLeaf(t, origin) + lap := a.beginLapAt(editAckDiskWrite, origin.Add(2*time.Millisecond)) + switch name { + case "reversed": + lap.finishAt(origin.Add(time.Millisecond)) + case "overlap": + nested := a.beginLapAt(editAckParseGate, origin.Add(3*time.Millisecond)) + nested.finishAt(origin.Add(4 * time.Millisecond)) + lap.finishAt(origin.Add(5 * time.Millisecond)) + } + endControlledLeaf(a, origin.Add(6*time.Millisecond)) + a.finishActualAt(origin.Add(7 * time.Millisecond)) + a.finishReplyAt(origin.Add(8*time.Millisecond), "go_error") + snapshot := a.snapshot() + payload := buildAcknowledgementPayload(&snapshot, "observed") + require.Nil(t, payload.SubtotalMS) + require.Nil(t, payload.UnattributedMS) + require.Contains(t, []string{"invalid_overlap", "invalid_interval"}, payload.SumState) + }) + } +} + +func TestAcknowledgementCallerResultUsesActualEndpointsWithoutDetachedClipping(t *testing.T) { + origin := time.Now() + for _, tc := range []struct { + name string + actual, reply time.Duration + want string + }{ + {"delivery", 4, 7, "measured"}, {"zero", 4, 4, "measured"}, + {"detached", 7, 4, "outside_reply_window"}, {"missing_actual", 0, 4, "unobserved"}, + {"pending_reply", 4, 0, "pending_reply"}, {"before_origin", -1, 4, "invalid_interval"}, + } { + t.Run(tc.name, func(t *testing.T) { + var actual, reply time.Time + if tc.actual != 0 { + actual = origin.Add(tc.actual * time.Millisecond) + } + if tc.reply != 0 { + reply = origin.Add(tc.reply * time.Millisecond) + } + got := acknowledgementCallerResult(origin, actual, reply) + require.Equal(t, tc.want, got.State) + if tc.want == "measured" { + require.Equal(t, float64(tc.reply-tc.actual), *got.AckMS) + require.Equal(t, *got.AckMS, *got.ObservedMS) + } else { + require.Nil(t, got.AckMS) + require.Nil(t, got.ObservedMS) + } + }) + } +} + +func TestEditControlFractionalMillisecondsUseObservedNanoseconds(t *testing.T) { + origin := time.Now() + a := &editAcknowledgement{entered: origin, envelope: editAcknowledgementEnvelope{observed: true, leafStarted: origin.Add(100 * time.Microsecond)}} + a.mu.Lock() + a.laps.controlObserved = true + a.openEditControlLocked(a.envelope.leafStarted) + a.mu.Unlock() + endControlledLeaf(a, origin.Add(300*time.Microsecond)) + a.finishActualAt(origin.Add(300 * time.Microsecond)) + a.finishReplyAt(origin.Add(300*time.Microsecond), "result") + snapshot := a.snapshot() + roundedSum := *snapshot.Middleware.AckMS + *snapshot.EditControl.AckMS + require.Greater(t, roundedSum, *snapshot.ReplyWallMS, "per-lap float0.1+0.2 must not reject actual disjoint300000ns") + payload := buildAcknowledgementPayload(&snapshot, "observed") + require.Equal(t, "partial", payload.SumState) + require.Equal(t, 0.3, *payload.SubtotalMS) + require.Equal(t, 0.0, *payload.UnattributedMS) + + overflow := a.snapshot() + overflow.EditControl.ackNS++ + ms := float64(overflow.EditControl.ackNS) / 1e6 + overflow.EditControl.AckMS, overflow.EditControl.ObservedMS = &ms, &ms + rejected := buildAcknowledgementPayload(&overflow, "observed") + require.Equal(t, "invalid_total", rejected.SumState, "even1actualns beyond ACK must be refused without tolerance") + require.Nil(t, rejected.SubtotalMS) + + inconsistent := a.snapshot() + modified := *inconsistent.EditControl.AckMS + 0.01 + inconsistent.EditControl.AckMS, inconsistent.EditControl.ObservedMS = &modified, &modified + require.Equal(t, "invalid_interval", buildAcknowledgementPayload(&inconsistent, "observed").SumState, "public scalars must match actual private readings") + + synthetic := snapshot + synthetic.hasReplyNS = false + synthetic.Middleware.hasAckNS = false + synthetic.EditControl.hasAckNS = false + require.Equal(t, "invalid_total", buildAcknowledgementPayload(&synthetic, "observed").SumState, "missing actualNS evidence keeps the conservative float fallback") +} diff --git a/internal/mcp/edit_acknowledgement_dispatch_test.go b/internal/mcp/edit_acknowledgement_dispatch_test.go new file mode 100644 index 000000000..82aa7240a --- /dev/null +++ b/internal/mcp/edit_acknowledgement_dispatch_test.go @@ -0,0 +1,61 @@ +package mcp + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/daemon" +) + +func TestAcknowledgementDispatchSnapshotKeepsSeparateOriginAndInvalidStates(t *testing.T) { + start := time.Now() + a := &editAcknowledgement{entered: start.Add(10 * time.Millisecond), hasDispatch: true, + dispatch: daemon.MCPDispatchTiming{Entered: start, Granted: start.Add(4 * time.Millisecond)}} + a.finishActualAt(start.Add(25 * time.Millisecond)) + a.finishReplyAt(start.Add(30*time.Millisecond), "result") + snapshot := a.snapshot() + require.Equal(t, 4.0, *snapshot.DispatcherQueueMS) + require.Equal(t, 6.0, *snapshot.DispatcherPreambleMS) + require.Equal(t, 20.0, *snapshot.ReplyWallMS, "admission timestamps do not silently move the existing bound-call origin") + require.Equal(t, 30.0, *snapshot.DispatcherReplyMS) + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Equal(t, "queue_and_preamble_precede_bound_call_subtotal", ack.DispatcherScope) + require.Equal(t, 4.0, *ack.PhasesMS["dispatcher_queue"]) + require.Equal(t, 6.0, *ack.PhasesMS["dispatcher_preamble"]) + require.Equal(t, 30.0, *ack.DispatcherReplyMS) + require.Equal(t, 20.0, *ack.ReplyWallMS) + require.Equal(t, 5.0, *ack.PhasesMS["caller_result"], "actual25→reply30 belongs to A") + require.Equal(t, 5.0, *ack.SubtotalMS, "outside-bound queue4+preamble6 must not enter the bound-call subtotal") + require.Equal(t, 15.0, *ack.UnattributedMS) + a.mu.Lock() + savedActual := a.actual + a.actual = time.Time{} + a.mu.Unlock() + withoutActual := a.snapshot() + require.Nil(t, buildAcknowledgementPayload(&withoutActual, "observed").SubtotalMS, + "outside-bound admission alone cannot become a bound-call subtotal") + a.mu.Lock() + a.actual = savedActual + a.mu.Unlock() + *snapshot.DispatcherQueueMS = 999 + require.Equal(t, 4.0, *a.snapshot().DispatcherQueueMS) + for _, tc := range []struct{ entered, granted time.Time }{ + {start.Add(5 * time.Millisecond), start.Add(4 * time.Millisecond)}, + {start, start.Add(11 * time.Millisecond)}, + } { + a.dispatch = daemon.MCPDispatchTiming{Entered: tc.entered, Granted: tc.granted} + invalid := a.snapshot() + require.Equal(t, "invalid_interval", invalid.DispatcherQueueState) + require.Equal(t, "invalid_interval", invalid.DispatcherState) + require.Nil(t, invalid.DispatcherQueueMS) + require.Nil(t, invalid.DispatcherPreambleMS) + require.Nil(t, buildAcknowledgementPayload(&invalid, "observed").DispatcherReplyMS) + } + a.hasDispatch = false + missing := a.snapshot() + require.Nil(t, missing.DispatcherQueueMS) + require.Equal(t, "unavailable_dispatcher_interface", missing.DispatcherQueueState) + require.Nil(t, buildAcknowledgementPayload(&missing, "observed").PhasesMS["dispatcher_queue"]) +} diff --git a/internal/mcp/edit_acknowledgement_envelope.go b/internal/mcp/edit_acknowledgement_envelope.go new file mode 100644 index 000000000..c36f5611b --- /dev/null +++ b/internal/mcp/edit_acknowledgement_envelope.go @@ -0,0 +1,109 @@ +package mcp + +import ( + "context" + "time" + + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// The first actual middleware invocation owns these two outer boundaries. +// They do not include the leaf, whose parse/disk/admission/wait laps can repeat. +// Deferred view/repository/activity cleanup ends at the existing actual-handler +// endpoint, not at a newly registered defer which would run before that cleanup. +type editAcknowledgementEnvelope struct { + observed bool + leafStarted, leafFinished time.Time +} + +func (s *Server) noteAcknowledgementMiddleware(ctx context.Context) { + if a := editAcknowledgementFrom(ctx, s); a != nil { + a.mu.Lock() + a.envelope.observed = true + a.mu.Unlock() + } +} + +func (s *Server) invokeAcknowledgedLeaf(ctx context.Context, req mcp.CallToolRequest, h mcpserver.ToolHandlerFunc) (*mcp.CallToolResult, error) { + a := editAcknowledgementFrom(ctx, s) + if a == nil || !a.beginLeaf() { + return h(ctx, req) // reads/native/nested calls add no envelope clock or defer + } + return runAcknowledgedLeaf(a, ctx, req, h) +} + +func runAcknowledgedLeaf(a *editAcknowledgement, ctx context.Context, req mcp.CallToolRequest, h mcpserver.ToolHandlerFunc) (*mcp.CallToolResult, error) { + defer func() { + a.mu.Lock() + a.envelope.leafFinished = time.Now() + a.closeEditControlLocked(a.envelope.leafFinished) + a.mu.Unlock() + }() + return h(ctx, req) +} + +func (a *editAcknowledgement) beginLeaf() bool { + a.mu.Lock() + defer a.mu.Unlock() + if !a.envelope.observed || !a.envelope.leafStarted.IsZero() { + return false + } + a.envelope.leafStarted = time.Now() + a.laps.controlObserved = true + a.openEditControlLocked(a.envelope.leafStarted) + return true +} + +func (e editAcknowledgementEnvelope) snapshot(entered, reply, actual time.Time) (editAcknowledgementLapSnapshot, editAcknowledgementLapSnapshot) { + if !e.observed { + return editAcknowledgementLapSnapshot{State: "unobserved"}, editAcknowledgementLapSnapshot{State: "unobserved"} + } + if e.leafStarted.IsZero() { + // A refusal before the leaf is actual middleware work, not a fabricated + // zero-duration leaf/reply phase. Pending work remains pending. + middleware := acknowledgementEnvelopeInterval(entered, actual, entered, reply, actual) + cleanup := editAcknowledgementLapSnapshot{State: "pending", Started: 1} + if !actual.IsZero() { + cleanup.State, cleanup.Completed, cleanup.NotApplicable = "not_applicable", 1, 1 + } + return middleware, cleanup + } + middleware := acknowledgementEnvelopeInterval(entered, e.leafStarted, entered, reply, actual) + cleanup := acknowledgementEnvelopeInterval(e.leafFinished, actual, entered, reply, actual) + if e.leafStarted.Before(entered) || (!e.leafFinished.IsZero() && e.leafFinished.Before(e.leafStarted)) { + middleware = editAcknowledgementLapSnapshot{State: "invalid_interval", Started: 1, Completed: 1} + cleanup = middleware + } + return middleware, cleanup +} + +func acknowledgementEnvelopeInterval(start, end, entered, reply, actual time.Time) editAcknowledgementLapSnapshot { + out := editAcknowledgementLapSnapshot{Started: 1, State: "incomplete"} + if start.IsZero() || end.IsZero() { + return out + } + out.Completed = 1 + if entered.IsZero() || start.Before(entered) || end.Before(start) { + out.State = "invalid_interval" + return out + } + if !actual.IsZero() && end.After(actual) { + out.State = "outside_execution_window" + return out + } + ms := float64(end.Sub(start)) / float64(time.Millisecond) + out.ObservedMS = &ms + if reply.IsZero() { + out.State = "pending_reply" + return out + } + if reply.Before(entered) || end.After(reply) { + out.State = "outside_reply_window" + return out + } + ackMS := ms + out.AckMS, out.State = &ackMS, "measured" + out.ackNS, out.hasAckNS = end.Sub(start).Nanoseconds(), true + return out +} diff --git a/internal/mcp/edit_acknowledgement_envelope_test.go b/internal/mcp/edit_acknowledgement_envelope_test.go new file mode 100644 index 000000000..977dbd1c9 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_envelope_test.go @@ -0,0 +1,225 @@ +package mcp + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func TestAcknowledgementEnvelopeDisjointLeafChunksAndCleanup(t *testing.T) { + start := time.Now() + a := &editAcknowledgement{entered: start, envelope: editAcknowledgementEnvelope{ + observed: true, leafStarted: start.Add(3 * time.Millisecond), leafFinished: start.Add(14 * time.Millisecond), + }} + for _, chunk := range []struct{ begin, end time.Duration }{{4 * time.Millisecond, 6 * time.Millisecond}, {9 * time.Millisecond, 12 * time.Millisecond}} { + lap := a.beginLapAt(editAckDiskWrite, start.Add(chunk.begin)) + lap.finishAt(start.Add(chunk.end)) + } + a.finishActualAt(start.Add(18 * time.Millisecond)) + a.finishReplyAt(start.Add(20*time.Millisecond), "result") + snapshot := a.snapshot() + require.True(t, snapshot.LapsDisjoint) + require.Equal(t, 3.0, *snapshot.Middleware.AckMS) + require.Equal(t, 4.0, *snapshot.ReplyPrepare.AckMS) + require.Equal(t, 5.0, *snapshot.DiskWrite.AckMS) + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Equal(t, 14.0, *ack.SubtotalMS) + require.Equal(t, 6.0, *ack.UnattributedMS, "only actual18→20 caller delivery is added; synthetic unobserved leaf control stays missing") + require.Equal(t, "partial", ack.SumState) + *snapshot.Middleware.AckMS = 999 + require.Equal(t, 3.0, *a.snapshot().Middleware.AckMS) +} + +func TestAcknowledgementEnvelopeRejectsMisplacedAndMissingBoundaries(t *testing.T) { + start := time.Now() + for _, tc := range []struct { + name string + leafBegin, leafEnd, lapBegin, lapEnd time.Duration + }{ + {"operation in middleware", 3, 8, 1, 2}, + {"operation in cleanup", 3, 8, 8, 9}, + {"no leaf with operation", 0, 0, 4, 5}, + {"reversed leaf", 8, 3, 4, 5}, + } { + t.Run(tc.name, func(t *testing.T) { + a := &editAcknowledgement{entered: start, envelope: editAcknowledgementEnvelope{observed: true}} + if tc.leafBegin != 0 { + a.envelope.leafStarted = start.Add(tc.leafBegin * time.Millisecond) + a.envelope.leafFinished = start.Add(tc.leafEnd * time.Millisecond) + } + lap := a.beginLapAt(editAckDiskWrite, start.Add(tc.lapBegin*time.Millisecond)) + lap.finishAt(start.Add(tc.lapEnd * time.Millisecond)) + a.finishActualAt(start.Add(10 * time.Millisecond)) + a.finishReplyAt(start.Add(11*time.Millisecond), "result") + snapshot := a.snapshot() + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Contains(t, []string{"invalid_overlap", "invalid_interval"}, ack.SumState) + require.Nil(t, ack.SubtotalMS) + require.Nil(t, ack.UnattributedMS) + }) + } + a := &editAcknowledgement{entered: start, envelope: editAcknowledgementEnvelope{observed: true, leafStarted: start.Add(time.Millisecond)}} + a.finishReplyAt(start.Add(2*time.Millisecond), "go_error") + require.Equal(t, "incomplete", a.snapshot().ReplyPrepare.State) + require.Nil(t, a.snapshot().ReplyPrepare.ObservedMS) +} + +func TestAcknowledgementEnvelopeActualMiddlewareAndRecoveredPanic(t *testing.T) { + for _, panicLeaf := range []bool{false, true} { + t.Run(map[bool]string{false: "normal", true: "panic"}[panicLeaf], func(t *testing.T) { + s := fastPathTestServer(t) + s.ToolCallTimeout = -1 + var a *editAcknowledgement + leafCalls := 0 + wrapped := s.wrapToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + leafCalls++ + require.Same(t, a, editAcknowledgementFrom(ctx, s)) + if panicLeaf { + panic("owned envelope oracle") + } + // A nested facade/raw invocation must not replace the first leaf + // boundaries or include its duration a second time. + before := a.envelope.leafStarted + _, err := s.invokeAcknowledgedLeaf(ctx, req, func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText("nested"), nil + }) + require.NoError(t, err) + require.Equal(t, before, a.envelope.leafStarted) + return mcp.NewToolResultText(`{"ok":true}`), nil + }) + capture := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return wrapped(ctx, req) + }) + result, err := capture(t.Context(), writeFileRequest(nil)) + require.NoError(t, err) + require.Equal(t, panicLeaf, result.IsError) + require.Equal(t, 1, leafCalls) + snapshot := a.snapshot() + require.Equal(t, "measured", snapshot.Middleware.State) + require.Equal(t, "measured", snapshot.ReplyPrepare.State) + require.Equal(t, "measured", snapshot.EditControl.State) + require.NotNil(t, snapshot.EditControl.AckMS) + leafMS := float64(a.envelope.leafFinished.Sub(a.envelope.leafStarted)) / float64(time.Millisecond) + require.InDelta(t, leafMS, *snapshot.EditControl.AckMS, 1e-9, "the real leaf, including nested calls/panic unwinding, owns one control interval") + require.Equal(t, "measured", snapshot.CallerResult.State) + payload := buildAcknowledgementPayload(&snapshot, "observed") + require.InDelta(t, *snapshot.ReplyWallMS, *payload.SubtotalMS, 1e-6) + require.Equal(t, "partial", payload.SumState, "source endpoints are not actual100-edit acceptance") + require.True(t, snapshot.LapsDisjoint) + require.False(t, a.envelope.leafFinished.After(a.actual), "actual finish includes all middleware defers") + require.Equal(t, "measured", snapshot.ActualState) + require.LessOrEqual(t, *snapshot.Middleware.AckMS+*snapshot.ReplyPrepare.AckMS, *snapshot.ReplyWallMS) + }) + } +} + +func TestAcknowledgementEnvelopeActualPreLeafRefusal(t *testing.T) { + s := fastPathTestServer(t) + s.ToolCallTimeout = -1 + var a *editAcknowledgement + wrapped := s.wrapToolHandler(func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { + t.Error("invalid freshness input must refuse before the leaf") + return mcp.NewToolResultText("unexpected"), nil + }) + capture := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return wrapped(ctx, req) + }) + result, err := capture(t.Context(), writeFileRequest(map[string]any{"wait_deadline": "invalid-duration"})) + require.NoError(t, err) + require.True(t, result.IsError) + snapshot := a.snapshot() + require.Equal(t, "measured", snapshot.Middleware.State) + require.Equal(t, "not_applicable", snapshot.ReplyPrepare.State) + require.Equal(t, "unobserved", snapshot.EditControl.State) + require.Nil(t, snapshot.EditControl.AckMS) + require.Nil(t, snapshot.ReplyPrepare.ObservedMS) + require.True(t, a.envelope.leafStarted.IsZero()) + require.Equal(t, *snapshot.ActualWallMS, *snapshot.Middleware.AckMS) +} + +func TestAcknowledgementEnvelopeLateLeafDoesNotClipCleanupIntoReply(t *testing.T) { + s := fastPathTestServer(t) + s.ToolCallTimeout = time.Minute + entered := make(chan *editAcknowledgement, 1) + release, done := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + var a *editAcknowledgement + t.Cleanup(func() { + unblock() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("owned late leaf did not join") + } + if a != nil { + require.Eventually(t, func() bool { return a.snapshot().ActualState == "measured" }, 5*time.Second, time.Millisecond, + "cleanup joins the actual middleware after the caller goroutine returned") + } + }) + wrapped := s.wrapToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + entered <- editAcknowledgementFrom(ctx, s) + <-release + return mcp.NewToolResultText(`{"ok":true}`), nil + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + returned := make(chan error, 1) + go func() { + defer close(done) + _, err := wrapped(ctx, writeFileRequest(nil)) + returned <- err + }() + select { + case a = <-entered: + case <-time.After(5 * time.Second): + t.Fatal("leaf did not enter") + } + cancel() + select { + case err := <-returned: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("caller did not receive cancellation") + } + frozen := a.snapshot() + require.Nil(t, frozen.ReplyPrepare.ObservedMS) + require.Equal(t, "incomplete", frozen.EditControl.State) + require.Nil(t, frozen.EditControl.AckMS) + waitPastAcknowledgementReply(t, a) + unblock() + require.Eventually(t, func() bool { return a.snapshot().ActualState == "measured" }, 5*time.Second, time.Millisecond) + after := a.snapshot() + require.Equal(t, frozen.ReplyWallMS, after.ReplyWallMS) + require.True(t, after.ActualFinishedLate) + require.NotNil(t, after.ReplyPrepare.ObservedMS) + require.Nil(t, after.ReplyPrepare.AckMS) + require.Equal(t, "outside_reply_window", after.ReplyPrepare.State) + require.NotNil(t, after.EditControl.ObservedMS) + require.Nil(t, after.EditControl.AckMS, "late physical leaf control must not be clipped into ACK") + require.Equal(t, "outside_reply_window", after.EditControl.State) + require.Equal(t, "outside_reply_window", after.CallerResult.State) + require.Nil(t, after.CallerResult.AckMS) + require.Equal(t, "incomplete", frozen.EditControl.State, "earlier snapshots remain immutable") +} + +func TestAcknowledgementEnvelopeUnobservedReadsHaveNoIdentityOrAllocation(t *testing.T) { + s := &Server{} + req := editOperationRequest("read", "file") + result := mcp.NewToolResultText("read") + h := func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error) { return result, nil } + ctx := t.Context() + before := editAcknowledgementSequence.Load() + allocations := testing.AllocsPerRun(100, func() { + s.noteAcknowledgementMiddleware(ctx) + _, _ = s.invokeAcknowledgedLeaf(ctx, req, h) + }) + require.Zero(t, allocations, "nil observer must not allocate an envelope/token/closure") + require.Equal(t, before, editAcknowledgementSequence.Load()) +} diff --git a/internal/mcp/edit_acknowledgement_io.go b/internal/mcp/edit_acknowledgement_io.go new file mode 100644 index 000000000..2dfc61619 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_io.go @@ -0,0 +1,27 @@ +package mcp + +import ( + "context" + "os" +) + +// These fixed caller boundaries surround only forward source durability. +// Journal/backup/rollback work remains separate and unclassified; no parent +// batch interval encloses these chunks or adds them a second time. +func (s *Server) writeAcknowledgedBatchFile(ctx context.Context, writer func(string, []byte, os.FileMode) error, path string, content []byte, mode os.FileMode) error { + lap := s.beginAcknowledgementLap(ctx, editAckDiskWrite) + defer lap.finish() + return writer(path, content, mode) +} + +func (s *Server) removeAcknowledgedBatchFile(ctx context.Context, remover func(string) error, path string) error { + lap := s.beginAcknowledgementLap(ctx, editAckDiskWrite) + defer lap.finish() + return remover(path) +} + +func (s *Server) syncAcknowledgedBatchDirectories(ctx context.Context, dirs ...string) error { + lap := s.beginAcknowledgementLap(ctx, editAckDiskWrite) + defer lap.finish() + return s.syncBatchDirectories(dirs...) +} diff --git a/internal/mcp/edit_acknowledgement_laps.go b/internal/mcp/edit_acknowledgement_laps.go new file mode 100644 index 000000000..436dccb9d --- /dev/null +++ b/internal/mcp/edit_acknowledgement_laps.go @@ -0,0 +1,205 @@ +package mcp + +import ( + "context" + "time" +) + +// Fixed acknowledgement operations, not a general span/event collection. The +// slots accumulate repeated chunks (for example more than one file) without +// retaining per-call arrays, contexts, source strings or parser results. +type editAcknowledgementLapKind uint8 + +const ( + editAckParseGate editAcknowledgementLapKind = iota + editAckDiskWrite + editAckTicketAdmission + editAckReindexWait + editAckLeafControl + editAckLapCount +) + +type editAcknowledgementLapData struct { + started, completed uint64 + notApplicable uint64 + elapsed time.Duration + first, last time.Time + invalid bool +} + +type editAcknowledgementLaps struct { + data [editAckLapCount]editAcknowledgementLapData + sequence uint64 + active uint64 + activeKind editAcknowledgementLapKind + lastEnd time.Time + controlStarted time.Time + controlObserved bool + overlap bool +} + +// ObservedMS is actual completed execution, even if it outlived the caller. +// AckMS is present only for complete valid chunks contained in the frozen +// reply wall. They are alternatives, not two durations to add together. +type editAcknowledgementLapSnapshot struct { + ObservedMS *float64 + AckMS *float64 + State string + Started uint64 + Completed uint64 + NotApplicable uint64 + ackNS int64 // actual scalar retained until final projection, never interval history + hasAckNS bool +} + +type editAcknowledgementLap struct { + owner *editAcknowledgement + kind editAcknowledgementLapKind + sequence uint64 + started time.Time + applicable bool +} + +func (s *Server) beginAcknowledgementLap(ctx context.Context, kind editAcknowledgementLapKind) editAcknowledgementLap { + return beginEditAcknowledgementLap(editAcknowledgementFrom(ctx, s), kind) +} + +func beginEditAcknowledgementLap(a *editAcknowledgement, kind editAcknowledgementLapKind) editAcknowledgementLap { + if a == nil { + return editAcknowledgementLap{} // raw/read/native callers do not sample a clock + } + return a.beginLapAt(kind, time.Now()) +} + +// Keep the real old/new parse-gate work inside one interval. Callers retain +// their existing enable/refusal/override policy; unsupported languages remain +// the gate's existing Checked=false verdict, not a new parse implementation. +func (s *Server) checkAcknowledgedParseGate(ctx context.Context, path string, before, after []byte) parseGateResult { + s.noteAcknowledgementParser(ctx, "old_new_parse_gate") + lap := s.beginAcknowledgementLap(ctx, editAckParseGate) + defer lap.finish() + result := checkParseGate(path, before, after) + lap.applicable = result.Checked + return result +} + +func (a *editAcknowledgement) beginLapAt(kind editAcknowledgementLapKind, at time.Time) editAcknowledgementLap { + a.mu.Lock() + defer a.mu.Unlock() + laps := &a.laps + if laps.active != 0 && laps.activeKind == editAckLeafControl { + a.closeEditControlLocked(at) + } + laps.sequence++ + data := &laps.data[kind] + data.started++ + if data.first.IsZero() { + data.first = at + } + if at.IsZero() || at.Before(a.entered) { + data.invalid = true + } + if laps.active != 0 { + laps.overlap = true + data.invalid = true + laps.data[laps.activeKind].invalid = true + } else { + if !laps.lastEnd.IsZero() && at.Before(laps.lastEnd) { + laps.overlap = true + data.invalid = true + } + laps.active, laps.activeKind = laps.sequence, kind + } + return editAcknowledgementLap{owner: a, kind: kind, sequence: laps.sequence, started: at, applicable: true} +} + +func (lap *editAcknowledgementLap) finish() { + if lap.owner == nil { + return // no clock on an unobserved or already-finished token + } + lap.finishAt(time.Now()) +} + +func (lap *editAcknowledgementLap) finishAt(at time.Time) { + a := lap.owner + if a == nil { + return + } + lap.owner = nil // this token belongs to one synchronous caller + a.mu.Lock() + defer a.mu.Unlock() + laps := &a.laps + data := &laps.data[lap.kind] + data.completed++ + if !lap.applicable { + data.notApplicable++ + } + if at.IsZero() || at.Before(lap.started) || lap.started.IsZero() { + data.invalid = true + } else if lap.applicable { + data.elapsed += at.Sub(lap.started) + } + if data.last.IsZero() || at.After(data.last) { + data.last = at + } + matched := laps.active == lap.sequence + if matched { + laps.active = 0 + } else { + laps.overlap = true + data.invalid = true + } + // A checked-but-not-applicable parse gate still performed actual guard + // dispatch. Its measured call belongs to leaf control, not a fake parse. + if matched && !lap.applicable && !data.invalid && a.leafControlOpenLocked() { + a.openEditControlLocked(lap.started) + a.closeEditControlLocked(at) + } + if at.After(laps.lastEnd) { + laps.lastEnd = at + } + if matched && laps.active == 0 && a.leafControlOpenLocked() { + a.openEditControlLocked(at) + } +} + +func (laps editAcknowledgementLaps) snapshot(kind editAcknowledgementLapKind, entered, reply, actual time.Time) editAcknowledgementLapSnapshot { + data := laps.data[kind] + out := editAcknowledgementLapSnapshot{Started: data.started, Completed: data.completed, NotApplicable: data.notApplicable, State: "unobserved"} + if data.started == 0 { + return out + } + if data.invalid || entered.IsZero() || data.first.Before(entered) { + out.State = "invalid_interval" + return out + } + if data.completed != data.started { + out.State = "incomplete" + return out + } + if data.notApplicable == data.started { + out.State = "not_applicable" + return out + } + if !actual.IsZero() && data.last.After(actual) { + out.State = "outside_execution_window" + return out + } + ms := float64(data.elapsed) / float64(time.Millisecond) + out.ObservedMS = &ms + if reply.IsZero() { + out.State = "pending_reply" + return out + } + if reply.Before(entered) || data.last.After(reply) { + out.State = "outside_reply_window" + return out + } + ackMS := ms + out.AckMS, out.State = &ackMS, "measured" + out.ackNS, out.hasAckNS = data.elapsed.Nanoseconds(), true + if data.notApplicable > 0 { + out.State = "measured_with_not_applicable_chunks" + } + return out +} diff --git a/internal/mcp/edit_acknowledgement_laps_test.go b/internal/mcp/edit_acknowledgement_laps_test.go new file mode 100644 index 000000000..21e4bd2ee --- /dev/null +++ b/internal/mcp/edit_acknowledgement_laps_test.go @@ -0,0 +1,267 @@ +package mcp + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/indexer" +) + +func TestEditAcknowledgementRepeatedDisjointChunksAndUnclassifiedGap(t *testing.T) { + start := time.Now() + a := &editAcknowledgement{entered: start} + for _, chunk := range []struct { + kind editAcknowledgementLapKind + begin, end time.Duration + }{ + {editAckParseGate, time.Millisecond, 3 * time.Millisecond}, + {editAckDiskWrite, 4 * time.Millisecond, 7 * time.Millisecond}, + {editAckParseGate, 8 * time.Millisecond, 12 * time.Millisecond}, + {editAckTicketAdmission, 13 * time.Millisecond, 18 * time.Millisecond}, + } { + lap := a.beginLapAt(chunk.kind, start.Add(chunk.begin)) + lap.finishAt(start.Add(chunk.end)) + lap.finishAt(start.Add(50 * time.Millisecond)) // finished token is a no-op + } + a.finishActualAt(start.Add(20 * time.Millisecond)) + a.finishReplyAt(start.Add(22*time.Millisecond), "result") + snapshot := a.snapshot() + require.True(t, snapshot.LapsDisjoint) + require.EqualValues(t, 2, snapshot.ParseGate.Started) + require.EqualValues(t, 2, snapshot.ParseGate.Completed) + require.Equal(t, 6.0, *snapshot.ParseGate.AckMS) + require.Equal(t, 3.0, *snapshot.DiskWrite.AckMS) + require.Equal(t, 5.0, *snapshot.TicketAdmission.AckMS) + measured := *snapshot.ParseGate.AckMS + *snapshot.DiskWrite.AckMS + *snapshot.TicketAdmission.AckMS + require.Equal(t, 14.0, measured) + require.Equal(t, 22.0, *snapshot.ReplyWallMS) + require.Greater(t, *snapshot.ReplyWallMS-measured, 0.05*(*snapshot.ReplyWallMS), + "missing middleware/wait/reply work remains unclassified, not assigned to a lap to force the sum") + *snapshot.ParseGate.AckMS = 1000 + require.Equal(t, 6.0, *a.snapshot().ParseGate.AckMS) +} + +func TestEditAcknowledgementLapsRejectOverlapMissingAndLateIntervals(t *testing.T) { + start := time.Now() + t.Run("overlap", func(t *testing.T) { + a := &editAcknowledgement{entered: start} + parse := a.beginLapAt(editAckParseGate, start.Add(time.Millisecond)) + disk := a.beginLapAt(editAckDiskWrite, start.Add(2*time.Millisecond)) + disk.finishAt(start.Add(3 * time.Millisecond)) + parse.finishAt(start.Add(4 * time.Millisecond)) + a.finishReplyAt(start.Add(5*time.Millisecond), "result") + got := a.snapshot() + require.False(t, got.LapsDisjoint) + require.Equal(t, "invalid_interval", got.ParseGate.State) + require.Nil(t, got.ParseGate.AckMS) + require.Nil(t, got.DiskWrite.AckMS) + }) + t.Run("missing end", func(t *testing.T) { + a := &editAcknowledgement{entered: start} + a.beginLapAt(editAckParseGate, start.Add(time.Millisecond)) + a.finishReplyAt(start.Add(5*time.Millisecond), "go_error") + require.Equal(t, "incomplete", a.snapshot().ParseGate.State) + require.Nil(t, a.snapshot().ParseGate.ObservedMS) + require.Nil(t, a.snapshot().ParseGate.AckMS) + require.Equal(t, "unobserved", a.snapshot().DiskWrite.State) + }) + t.Run("negative interval", func(t *testing.T) { + a := &editAcknowledgement{entered: start} + lap := a.beginLapAt(editAckParseGate, start.Add(3*time.Millisecond)) + lap.finishAt(start.Add(2 * time.Millisecond)) + a.finishReplyAt(start.Add(5*time.Millisecond), "result") + require.Equal(t, "invalid_interval", a.snapshot().ParseGate.State) + require.Nil(t, a.snapshot().ParseGate.ObservedMS) + }) + t.Run("actual execution containment", func(t *testing.T) { + a := &editAcknowledgement{entered: start} + lap := a.beginLapAt(editAckDiskWrite, start.Add(time.Millisecond)) + lap.finishAt(start.Add(4 * time.Millisecond)) + a.finishActualAt(start.Add(3 * time.Millisecond)) + a.finishReplyAt(start.Add(5*time.Millisecond), "result") + require.Equal(t, "outside_execution_window", a.snapshot().DiskWrite.State) + require.Nil(t, a.snapshot().DiskWrite.ObservedMS) + }) + t.Run("late actual execution is not clipped into ack", func(t *testing.T) { + a := &editAcknowledgement{entered: start} + lap := a.beginLapAt(editAckDiskWrite, start.Add(time.Millisecond)) + a.finishReplyAt(start.Add(2*time.Millisecond), "go_error") + lap.finishAt(start.Add(4 * time.Millisecond)) + a.finishActualAt(start.Add(5 * time.Millisecond)) + require.Equal(t, "outside_reply_window", a.snapshot().DiskWrite.State) + require.Equal(t, 3.0, *a.snapshot().DiskWrite.ObservedMS) + require.Nil(t, a.snapshot().DiskWrite.AckMS, "do not turn three actual milliseconds into one clipped millisecond") + }) +} + +func TestEditAcknowledgementRealParseGatePreservesVerdictsAndApplicability(t *testing.T) { + s := newMutationServer() + before := []byte("package p\nfunc A() {}\n") + for _, tt := range []struct { + name, path, after string + checked, blocked bool + }{ + {"allow", "a.go", "package p\nfunc A() { println(1) }\n", true, false}, + {"refuse", "a.go", "package p\nfunc A( {\n", true, true}, + {"unsupported", "a.unknown-language", "anything", false, false}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx, ack, _ := s.withEditAcknowledgement(t.Context()) + got := s.checkAcknowledgedParseGate(ctx, tt.path, before, []byte(tt.after)) + require.Equal(t, checkParseGate(tt.path, before, []byte(tt.after)), got, "only observe the existing parser verdict") + require.Equal(t, tt.checked, got.Checked) + require.Equal(t, tt.blocked, got.Blocked) + ack.finishActual() + ack.finishReply(mcp.NewToolResultText("fixture gate returned"), nil) + snapshot := ack.snapshot().ParseGate + require.EqualValues(t, 1, snapshot.Started) + require.EqualValues(t, 1, snapshot.Completed) + if tt.checked { + require.Equal(t, "measured", snapshot.State) + require.NotNil(t, snapshot.AckMS) + } else { + require.Equal(t, "not_applicable", snapshot.State) + require.Nil(t, snapshot.ObservedMS) + require.Nil(t, snapshot.AckMS) + } + }) + } +} + +func TestEditAcknowledgementRealDiskAndCanonicalTicketBoundaries(t *testing.T) { + for _, scheduleErr := range []error{nil, errors.New("fixture refused enqueue")} { + s := newMutationServer() + s.watcher = &canonicalPublicationWatcher{mutationTestWatcher: mutationTestWatcher{scheduleErr: scheduleErr}} + path := filepath.Join(t.TempDir(), "written.go") + var record *mutationCommitRecord + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + var err error + record, err = s.commitFileMutation(ctx, "write_file", "", "", "written.go", path, []byte("package p\n"), 0o600) + if err != nil { + return nil, err + } + outcome := s.mutationReindexState(ctx, path) + record.recordGraph(outcome) + if outcome.Err != nil { + return mcp.NewToolResultError(outcome.Err.Error()), nil + } + return mcp.NewToolResultText("source and ticket returned"), nil + }) + result, err := h(canonicalTimingContext(), editOperationRequest("write_file", "")) + require.NoError(t, err) + require.Equal(t, scheduleErr != nil, result.IsError) + bytes, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "package p\n", string(bytes)) + ack := retainedMutationAcknowledgement(record).snapshot() + require.EqualValues(t, 1, ack.DiskWrite.Completed) + require.EqualValues(t, 1, ack.TicketAdmission.Completed) + require.Equal(t, "measured", ack.DiskWrite.State) + require.Equal(t, "measured", ack.TicketAdmission.State, "failed admission is an observed call, not fake missing work") + if scheduleErr == nil { + require.EqualValues(t, 1, ack.ReindexWait.Completed) + require.Equal(t, "measured", ack.ReindexWait.State, "immediate ticket completion still reaches the actual wait") + } else { + require.Equal(t, "unobserved", ack.ReindexWait.State) + } + require.Equal(t, "unobserved", ack.ParseGate.State, "a raw commit did not call the gate") + require.True(t, ack.LapsDisjoint) + } +} + +type acknowledgementWaitWatcher struct { + mutationTestWatcher + afterAdmission func() +} + +func (w acknowledgementWaitWatcher) EnqueueFileMutation(ctx context.Context, path string) (*indexer.MutationTicket, error) { + ticket, err := w.mutationTestWatcher.EnqueueFileMutation(ctx, path) + if w.afterAdmission != nil { + w.afterAdmission() + } + return ticket, err +} + +func TestEditAcknowledgementActualWaitTimerAndCancellation(t *testing.T) { + for _, cancelled := range []bool{false, true} { + t.Run(map[bool]string{false: "timer", true: "caller cancellation"}[cancelled], func(t *testing.T) { + s := newMutationServer() + s.mutationReindexWait = 5 * time.Millisecond // exercise the existing timer, not a latency assertion + done := make(chan indexer.MutationResult, 1) + ctx, cancel := context.WithCancel(canonicalTimingContext()) + defer cancel() + watcher := acknowledgementWaitWatcher{mutationTestWatcher: mutationTestWatcher{done: done}} + if cancelled { + watcher.afterAdmission = cancel + } + s.watcher = watcher + ctx, ack, _ := s.withEditAcknowledgement(ctx) + path := filepath.Join(t.TempDir(), "wait.go") + commitTimingFixture(t, s, ctx, path, "package p\n") + // Release the sole existing ticket consumer on every assertion path. + t.Cleanup(func() { + close(done) + s.mutationReceipts.Range(func(_, value any) bool { + if receipt, ok := value.(*mutationReceipt); ok { + select { + case <-receipt.done: + case <-time.After(5 * time.Second): + t.Error("owned ticket consumer did not join") + } + } + return true + }) + }) + outcome := s.mutationReindexState(ctx, path) + require.True(t, outcome.Pending) + require.NotEmpty(t, outcome.Receipt) + ack.finishActual() + ack.finishReply(mcp.NewToolResultText("pending"), nil) + got := ack.snapshot() + require.Equal(t, "measured", got.ReindexWait.State) + require.EqualValues(t, 1, got.ReindexWait.Started) + require.EqualValues(t, 1, got.ReindexWait.Completed) + require.NotNil(t, got.ReindexWait.AckMS) + require.True(t, got.LapsDisjoint) + }) + } +} + +func TestEditAcknowledgementUnobservedLapAddsNoAllocationOrIdentity(t *testing.T) { + s, ctx := &Server{}, t.Context() + before := editAcknowledgementSequence.Load() + require.Zero(t, testing.AllocsPerRun(100, func() { + lap := s.beginAcknowledgementLap(ctx, editAckDiskWrite) + lap.finish() + })) + require.Equal(t, before, editAcknowledgementSequence.Load()) +} + +func TestEditAcknowledgementActualDiskFailureDoesNotInventAdmission(t *testing.T) { + s := newMutationServer() + path := t.TempDir() // an atomic file rename cannot replace this directory + var record *mutationCommitRecord + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + var err error + record, err = s.commitFileMutation(ctx, "write_file", "", "", "owned-directory", path, []byte("after"), 0o600) + return nil, err + }) + result, err := h(canonicalTimingContext(), editOperationRequest("write_file", "")) + require.Error(t, err) + require.Nil(t, result) + ack := retainedMutationAcknowledgement(record).snapshot() + require.EqualValues(t, 1, ack.DiskWrite.Completed) + require.Equal(t, "measured", ack.DiskWrite.State, "observe the real failed file call's elapsed work") + require.Equal(t, "unobserved", ack.TicketAdmission.State) + require.Equal(t, "go_error", ack.ReplyOutcome) + info, statErr := os.Stat(path) + require.NoError(t, statErr) + require.True(t, info.IsDir(), "failed source replacement must preserve its existing target") +} diff --git a/internal/mcp/edit_acknowledgement_parse.go b/internal/mcp/edit_acknowledgement_parse.go new file mode 100644 index 000000000..c37885082 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_parse.go @@ -0,0 +1,28 @@ +package mcp + +import ( + "context" + "go/ast" + "go/parser" + "go/token" +) + +func (s *Server) noteAcknowledgementParser(ctx context.Context, scope string) { + if a := editAcknowledgementFrom(ctx, s); a != nil { + a.mu.Lock() + if a.parserScope == "" { + a.parserScope = scope + } else if a.parserScope != scope { + a.parserScope = "mixed_observed_parser_paths" + } + a.mu.Unlock() + } +} + +// Observe only explicit parser calls. go/format's internal parse+format work +// remains unclassified, rather than reporting the combined call as pure parse. +func parseAcknowledgedGoFile(a *editAcknowledgement, fset *token.FileSet, filename string, source []byte, mode parser.Mode) (*ast.File, error) { + lap := beginEditAcknowledgementLap(a, editAckParseGate) + defer lap.finish() + return parser.ParseFile(fset, filename, source, mode) +} diff --git a/internal/mcp/edit_acknowledgement_retention_test.go b/internal/mcp/edit_acknowledgement_retention_test.go new file mode 100644 index 000000000..82b7da297 --- /dev/null +++ b/internal/mcp/edit_acknowledgement_retention_test.go @@ -0,0 +1,63 @@ +package mcp + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestEditAcknowledgementPhysicalFilesShareOneRPCButRetriesDoNot(t *testing.T) { + s := newMutationServer() + ctx, first, owner := s.withEditAcknowledgement(canonicalTimingContext()) + require.True(t, owner) + root := t.TempDir() + a := commitTimingFixture(t, s, ctx, filepath.Join(root, "a.go"), "first") + b := commitTimingFixture(t, s, ctx, filepath.Join(root, "b.go"), "second") + require.Same(t, first, retainedMutationAcknowledgement(a)) + require.Same(t, first, retainedMutationAcknowledgement(b), "two physical files must not multiply the RPC wall") + retryCtx, retry, retryOwner := s.withEditAcknowledgement(canonicalTimingContext()) + require.True(t, retryOwner) + c := commitTimingFixture(t, s, retryCtx, filepath.Join(root, "a.go"), "third") + require.Same(t, retry, retainedMutationAcknowledgement(c)) + require.NotEqual(t, first.id, retry.id) + // An unobserved/native/raw commit has no invented MCP acknowledgement. + raw := commitTimingFixture(t, s, canonicalTimingContext(), filepath.Join(root, "raw.go"), "raw") + require.Nil(t, retainedMutationAcknowledgement(raw)) +} + +func TestEditAcknowledgementHistoryEndsAtLedgerExpiryAndCap(t *testing.T) { + for _, expire := range []bool{true, false} { + t.Run(fmt.Sprintf("expire=%t", expire), func(t *testing.T) { + s := newMutationServer() + ctx, ack, _ := s.withEditAcknowledgement(canonicalTimingContext()) + r := commitTimingFixture(t, s, ctx, filepath.Join(t.TempDir(), "held.go"), "held") + require.Same(t, ack, retainedMutationAcknowledgement(r)) + // Keep a physical receipt alive independently, as failed checkout + // source evidence can. It must not retain acknowledgement history. + if expire { + r.mu.Lock() + r.startedAt = time.Now().Add(-2 * mutationCommitRetention) + r.mu.Unlock() + } + count := 1 + if !expire { + count = maxMutationCommits + } + for i := range count { + s.mutationCommits.put(&mutationCommitRecord{id: fmt.Sprintf("ack-cap-%d", i), startedAt: time.Now()}) + } + _, found := s.mutationCommits.byReceipt(r.id) + require.False(t, found) + require.Nil(t, retainedMutationAcknowledgement(r)) + ack.finishReplyAt(ack.entered.Add(time.Millisecond), "result") + ack.finishActualAt(ack.entered.Add(2 * time.Millisecond)) + require.Nil(t, retainedMutationAcknowledgement(r), "late completion cannot reinstall an evicted pointer") + // The still-running call can finish its own private observation; + // that is not a resurrected history owner or a new registry. + require.Equal(t, "measured", ack.snapshot().ReplyState) + }) + } +} diff --git a/internal/mcp/edit_acknowledgement_test.go b/internal/mcp/edit_acknowledgement_test.go new file mode 100644 index 000000000..8c74aa66c --- /dev/null +++ b/internal/mcp/edit_acknowledgement_test.go @@ -0,0 +1,194 @@ +package mcp + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func TestEditAcknowledgementSnapshotFirstWinsAndOwnedValues(t *testing.T) { + start := time.Now() + a := &editAcknowledgement{id: 9, entered: start} + pending := a.snapshot() + require.Equal(t, "pending", pending.ReplyState) + require.Nil(t, pending.ReplyWallMS) + require.Nil(t, pending.ActualWallMS) + require.Nil(t, pending.DispatcherQueueMS) + require.Nil(t, pending.DispatcherPreambleMS) + require.Equal(t, "unavailable_dispatcher_interface", pending.DispatcherQueueState) + require.Equal(t, "unavailable_dispatcher_interface", pending.DispatcherState) + a.finishReplyAt(start.Add(4*time.Millisecond), "go_error") + a.finishReplyAt(start.Add(10*time.Millisecond), "result") + frozen := a.snapshot() + require.Equal(t, "mcp_bound_call", frozen.Scope) + require.EqualValues(t, 9, frozen.RPCID) + require.Equal(t, 4.0, *frozen.ReplyWallMS) + require.Equal(t, "go_error", frozen.ReplyOutcome) + a.finishActualAt(start.Add(12 * time.Millisecond)) + a.finishActualAt(start.Add(20 * time.Millisecond)) + complete := a.snapshot() + require.Equal(t, frozen.ReplyWallMS, complete.ReplyWallMS, "physical completion cannot move caller ready") + require.True(t, complete.ActualFinishedLate) + require.Equal(t, 12.0, *complete.ActualWallMS) + require.Nil(t, frozen.ActualWallMS, "previous snapshots are immutable") + *complete.ReplyWallMS, *complete.ActualWallMS = 99, 99 + require.Equal(t, 4.0, *a.snapshot().ReplyWallMS) + require.Equal(t, 12.0, *a.snapshot().ActualWallMS) +} + +func TestEditAcknowledgementInvalidOrderIsNotClipped(t *testing.T) { + start := time.Now() + a := &editAcknowledgement{entered: start} + a.finishActualAt(start.Add(-time.Millisecond)) + a.finishReplyAt(start.Add(-2*time.Millisecond), "result") + snapshot := a.snapshot() + require.Equal(t, "invalid_order", snapshot.ReplyState) + require.Equal(t, "invalid_order", snapshot.ActualState) + require.Nil(t, snapshot.ReplyWallMS) + require.Nil(t, snapshot.ActualWallMS) + require.False(t, snapshot.ActualFinishedLate) + zero := &editAcknowledgement{entered: start} + zero.finishReplyAt(start, "result") + require.Equal(t, "measured", zero.snapshot().ReplyState) + require.Zero(t, *zero.snapshot().ReplyWallMS, "valid zero differs from missing or invalid") + missingOrigin := &editAcknowledgement{} + missingOrigin.finishReplyAt(start, "result") + require.Equal(t, "unavailable_origin", missingOrigin.snapshot().ReplyState) + require.Nil(t, missingOrigin.snapshot().ReplyWallMS) +} + +func TestEditAcknowledgementNestedOwnershipAndLegacyOrigin(t *testing.T) { + first, other := &Server{ToolCallTimeout: -1}, &Server{ToolCallTimeout: -1} + legacyOrigin := time.Now().Add(-time.Second) + ctx := withToolReceivedAt(t.Context(), legacyOrigin) + var outerRecord, nestedRecord, otherRecord *editAcknowledgement + inner := first.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + nestedRecord = editAcknowledgementFrom(ctx, first) + require.Equal(t, legacyOrigin, toolReceivedAt(ctx)) + return mcp.NewToolResultText("nested"), nil + }) + foreign := other.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + otherRecord = editAcknowledgementFrom(ctx, other) + require.Nil(t, editAcknowledgementFrom(ctx, first), "another Server cannot freeze the parent's record") + return mcp.NewToolResultText("foreign"), nil + }) + outer := first.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + outerRecord = editAcknowledgementFrom(ctx, first) + require.NotNil(t, outerRecord) + _, err := inner(ctx, req) + require.NoError(t, err) + require.Equal(t, "pending", outerRecord.snapshot().ActualState, "nested return is not outer actual completion") + require.Equal(t, "pending", outerRecord.snapshot().ReplyState, "nested reply cannot freeze the shared RPC wall") + _, err = foreign(ctx, req) + require.NoError(t, err) + observeEditSourceEffect(ctx, true, true) + observeEditSourceComplete(ctx) + return mcp.NewToolResultText("outer"), nil + }) + _, err := outer(ctx, editOperationRequest("edit", "file")) + require.NoError(t, err) + require.Same(t, outerRecord, nestedRecord) + require.NotSame(t, outerRecord, otherRecord) + require.NotEqual(t, outerRecord.id, otherRecord.id) + require.Equal(t, "measured", outerRecord.snapshot().ReplyState) + require.Equal(t, "measured", outerRecord.snapshot().ActualState) + require.EqualValues(t, 1, first.editOperations.completed.Load(), "actual source counting stays unchanged") + require.Zero(t, other.editOperations.completed.Load(), "a reply alone is not a completed edit") +} + +func TestEditAcknowledgementCallerReadyPrecedesLatePhysicalHandler(t *testing.T) { + s := &Server{ToolCallTimeout: time.Minute} + entered, release, finished := make(chan *editAcknowledgement, 1), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(func() { + unblock() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("owned actual handler did not join") + } + }) + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + defer close(finished) + entered <- editAcknowledgementFrom(ctx, s) + <-release // same non-cancellable completion contract as a held batch commit + observeEditSourceEffect(ctx, true, true) + observeEditSourceComplete(ctx) + return mcp.NewToolResultText("committed"), nil + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + type reply struct { + result *mcp.CallToolResult + err error + } + got := make(chan reply, 1) + go func() { + result, err := h(ctx, editOperationRequest("batch_edit", "")) + got <- reply{result, err} + }() + var record *editAcknowledgement + select { + case record = <-entered: + case <-time.After(5 * time.Second): + t.Fatal("actual handler did not enter") + } + cancel() + select { + case result := <-got: + require.ErrorIs(t, result.err, context.Canceled) + require.Nil(t, result.result) + case <-time.After(5 * time.Second): + t.Fatal("caller did not receive its cancellation reply") + } + frozen := record.snapshot() + require.Equal(t, "measured", frozen.ReplyState) + require.Equal(t, "go_error", frozen.ReplyOutcome) + require.Equal(t, "pending", frozen.ActualState) + require.Zero(t, s.editOperations.completed.Load()) + waitPastAcknowledgementReply(t, record) + unblock() + require.Eventually(t, func() bool { return record.snapshot().ActualState == "measured" }, 5*time.Second, time.Millisecond) + after := record.snapshot() + require.Equal(t, frozen.ReplyWallMS, after.ReplyWallMS) + require.True(t, after.ActualFinishedLate) + require.EqualValues(t, 1, s.editOperations.completed.Load()) +} + +// Channel ordering alone need not advance a coarse monotonic clock. Keep the +// actual handler held until a real reading is later than the frozen reply, so +// the strict late/outside-window assertions have measured separation. +func waitPastAcknowledgementReply(t *testing.T, record *editAcknowledgement) { + t.Helper() + record.mu.Lock() + reply := record.reply + record.mu.Unlock() + require.False(t, reply.IsZero(), "caller reply must already be frozen") + require.Eventually(t, func() bool { return time.Now().After(reply) }, 5*time.Second, time.Millisecond, + "monotonic clock did not advance past the frozen caller reply") +} + +func TestEditAcknowledgementReadsKeepBaselineAllocationsAndNoIdentity(t *testing.T) { + s := &Server{ToolCallTimeout: -1} + ctx, req := t.Context(), editOperationRequest("read_file", "") + result := mcp.NewToolResultText("read") + noop := func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if editAcknowledgementFrom(ctx, s) != nil { + t.Error("read acquired a mutation acknowledgement") + } + return result, nil + } + baseline := s.preAcknowledgementBoundToolHandler(noop) + observed := s.boundToolHandler(noop) + before := editAcknowledgementSequence.Load() + plain := testing.AllocsPerRun(100, func() { _, _ = baseline(ctx, req) }) + wrapped := testing.AllocsPerRun(100, func() { _, _ = observed(ctx, req) }) + t.Logf("allocations/read: pre-feature wrapper=%g, acknowledgement wrapper=%g", plain, wrapped) + require.LessOrEqual(t, wrapped, plain, "read wrapper must add no allocation versus the complete pre-feature wrapper") + require.Equal(t, before, editAcknowledgementSequence.Load(), "read never creates an acknowledgement identity") +} diff --git a/internal/mcp/edit_acknowledgement_tool_boundaries_test.go b/internal/mcp/edit_acknowledgement_tool_boundaries_test.go new file mode 100644 index 000000000..5042a260c --- /dev/null +++ b/internal/mcp/edit_acknowledgement_tool_boundaries_test.go @@ -0,0 +1,202 @@ +package mcp + +import ( + "context" + "errors" + "go/parser" + "go/token" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" +) + +func TestAcknowledgementBatchForwardDurabilityAndSetFallback(t *testing.T) { + for _, mode := range []string{"point", "set", "set-fallback"} { + t.Run(mode, func(t *testing.T) { + var watcher watcherHistory = mutationTestWatcher{} + if mode != "point" { + watcher = &batchSetSchedulerSpy{unsupported: mode == "set-fallback"} + } + s := newAtomicBatchTestServer(t, watcher) + s.ToolCallTimeout = -1 + dir := t.TempDir() + paths := []string{writeAtomicBatchFixture(t, dir, "a.txt", "before\n"), writeAtomicBatchFixture(t, dir, "b.txt", "before\n")} + var a *editAcknowledgement + h := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return s.handleAtomicBatchEdit(ctx, req) + }) + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "ack-durability", "edits": []any{ + map[string]any{"op": "edit_file", "path": paths[0], "old_string": "before", "new_string": "after"}, + map[string]any{"op": "edit_file", "path": paths[1], "old_string": "before", "new_string": "after"}, + }} + result, err := h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError, "%v", result.Content) + for _, path := range paths { + require.Equal(t, "after\n", readAtomicBatchFixture(t, path)) + } + snapshot := a.snapshot() + require.EqualValues(t, 3, snapshot.DiskWrite.Completed, "two forward writes plus one deduplicated directory-sync envelope; no journal/backup/cleanup laps") + wantAdmissions := map[string]uint64{"point": 2, "set": 1, "set-fallback": 3}[mode] + require.Equal(t, wantAdmissions, snapshot.TicketAdmission.Completed) + require.True(t, snapshot.LapsDisjoint) + require.NotNil(t, snapshot.DiskWrite.AckMS) + require.Equal(t, "not_applicable", snapshot.ParseGate.State) + require.Nil(t, snapshot.ParseGate.ObservedMS, "text replacement does not gain a fake parser duration") + state, _ := s.batchTransactions.Load("ack-durability") + original := s.batchAcknowledgement(state.(*batchTransactionState)) + result, err = h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + require.Equal(t, original, s.batchAcknowledgement(state.(*batchTransactionState)), "duplicate cannot add durability/admission laps to the original") + }) + } +} + +func TestAcknowledgementBatchFailedForwardWriteExcludesRollbackAndJournal(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = -1 + path := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + sentinel := errors.New("forward write refused") + s.batchWriteOverride = func(string, []byte, os.FileMode) error { return sentinel } + var a *editAcknowledgement + h := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return s.handleAtomicBatchEdit(ctx, req) + }) + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "ack-failure", "edits": []any{ + map[string]any{"op": "edit_file", "path": path, "old_string": "before", "new_string": "after"}, + }} + _, err := h(t.Context(), req) + require.NoError(t, err) + require.Equal(t, "before\n", readAtomicBatchFixture(t, path)) + snapshot := a.snapshot() + require.EqualValues(t, 1, snapshot.DiskWrite.Completed, "failed forward call is measured; rollback/journal durability is not silently relabelled") + require.NotNil(t, snapshot.DiskWrite.ObservedMS) + require.Zero(t, snapshot.TicketAdmission.Started) + require.Zero(t, s.editOperations.completed.Load()) +} + +func TestAcknowledgementBatchAdmissionKeepsOriginalSchedulerContext(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + parent, cancel := context.WithCancel(t.Context()) + cancel() + ctx, a, _ := s.withEditAcknowledgement(parent) + outcome := s.beginBatchGraphRefresh(ctx, filepath.Join(t.TempDir(), "target.go"), batchTransactionReceipt{}) + require.NoError(t, outcome.Err, "the original canonical batch scheduler uses Background even when caller observation is cancelled") + a.finishActualAt(time.Now()) + a.finishReplyAt(time.Now(), "go_error") + require.EqualValues(t, 1, a.snapshot().TicketAdmission.Completed) + require.NotNil(t, a.snapshot().TicketAdmission.ObservedMS) +} + +func TestAcknowledgementBatchMoveDeleteCountsForwardCallsOnly(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = -1 + dir := t.TempDir() + source := writeAtomicBatchFixture(t, dir, "source.txt", "move me\n") + deleted := writeAtomicBatchFixture(t, dir, "delete.txt", "delete me\n") + destination := filepath.Join(dir, "destination.txt") + var a *editAcknowledgement + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + receipt, err := s.runBatchTransaction(ctx, []batchEditItem{ + atomicFileMove(source, destination, testSHA256("move me\n")), + atomicFileDelete(deleted, testSHA256("delete me\n")), + }, "ack-move-delete") + if err != nil { + return nil, err + } + return mcp.NewToolResultJSON(receipt) + }) + result, err := h(t.Context(), editOperationRequest("batch_edit", "")) + require.NoError(t, err) + require.False(t, result.IsError) + require.NoFileExists(t, source) + require.NoFileExists(t, deleted) + require.Equal(t, "move me\n", readAtomicBatchFixture(t, destination)) + snapshot := a.snapshot() + require.EqualValues(t, 4, snapshot.DiskWrite.Completed, "one forward write, two forward removals and one deduplicated directory-sync envelope") + require.EqualValues(t, 3, snapshot.TicketAdmission.Completed) + require.True(t, snapshot.LapsDisjoint) +} + +func TestAcknowledgementMoveRealParserAndSourceDurability(t *testing.T) { + s, root := setupMoveInlineRepo(t, map[string]string{ + "pkg/a.go": "package pkg\n\nfunc Foo() int { return 42 }\n", + "pkg/b.go": "package pkg\n\nfunc Bar() int { return 1 }\n", + }) + s.ToolCallTimeout = -1 + var a *editAcknowledgement + h := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return s.handleMoveSymbol(ctx, req) + }) + req := editOperationRequest("move_symbol", "") + req.Params.Arguments = map[string]any{"id": "pkg/a.go::Foo", "target_file": "pkg/b.go"} + result, err := h(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError, "%v", result.Content) + require.NotContains(t, readAtomicBatchFixture(t, filepath.Join(root, "pkg/a.go")), "func Foo") + require.Contains(t, readAtomicBatchFixture(t, filepath.Join(root, "pkg/b.go")), "func Foo") + snapshot := a.snapshot() + require.EqualValues(t, 2, snapshot.DiskWrite.Completed) + require.GreaterOrEqual(t, snapshot.ParseGate.Completed, uint64(2), "real source and existing-target Go parser calls") + require.NotNil(t, snapshot.ParseGate.AckMS) + require.True(t, snapshot.LapsDisjoint) + require.Equal(t, "explicit_go_parser_calls_formatting_unattributed", snapshot.ParserScope) + require.Zero(t, snapshot.TicketAdmission.Started, "synchronous move reindex is not a queue admission") +} + +func TestAcknowledgementParserPreservesErrorAndRawAllocationBehaviour(t *testing.T) { + source := []byte("package p\nfunc F(\n") + _, plainErr := parser.ParseFile(token.NewFileSet(), "p.go", source, parser.ParseComments) + _, observedErr := parseAcknowledgedGoFile(nil, token.NewFileSet(), "p.go", source, parser.ParseComments) + require.EqualError(t, observedErr, plainErr.Error()) + plain := testing.AllocsPerRun(100, func() { _, _ = parser.ParseFile(token.NewFileSet(), "p.go", source, parser.ParseComments) }) + observed := testing.AllocsPerRun(100, func() { _, _ = parseAcknowledgedGoFile(nil, token.NewFileSet(), "p.go", source, parser.ParseComments) }) + require.Equal(t, plain, observed, "nil observer adds no parser timing/context allocation") + a := &editAcknowledgement{entered: time.Now()} + _, err := parseAcknowledgedGoFile(a, token.NewFileSet(), "p.go", source, parser.ParseComments) + require.EqualError(t, err, plainErr.Error()) + a.finishActualAt(time.Now()) + a.finishReplyAt(time.Now(), "tool_error") + require.EqualValues(t, 1, a.snapshot().ParseGate.Completed) + require.NotNil(t, a.snapshot().ParseGate.ObservedMS, "a real rejected parse still took measured work") +} + +func TestAcknowledgementMoveUnsupportedLanguagesKeepNullParserTime(t *testing.T) { + for _, extension := range []string{"py", "ts"} { + t.Run(extension, func(t *testing.T) { + s, _ := setupMoveInlineRepo(t, map[string]string{"pkg/a.go": "package pkg\nfunc F() {}\n"}) + s.ToolCallTimeout = -1 + path := "pkg/unsupported." + extension + id := path + "::F" + s.graph.AddNode(&graph.Node{ID: id, Name: "F", Kind: graph.KindFunction, FilePath: path, StartLine: 1, EndLine: 1}) + var a *editAcknowledgement + h := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + a = editAcknowledgementFrom(ctx, s) + return s.handleMoveSymbol(ctx, req) + }) + req := editOperationRequest("move_symbol", "") + req.Params.Arguments = map[string]any{"id": id, "target_file": "pkg/target." + extension} + result, err := h(t.Context(), req) + require.NoError(t, err) + require.True(t, result.IsError) + snapshot := a.snapshot() + require.Equal(t, "unsupported_non_go_move", snapshot.ParserScope) + require.Equal(t, "not_applicable", snapshot.ParseGate.State) + require.Nil(t, snapshot.ParseGate.ObservedMS) + require.Zero(t, snapshot.DiskWrite.Started) + }) + } +} diff --git a/internal/mcp/edit_acknowledgement_wire.go b/internal/mcp/edit_acknowledgement_wire.go new file mode 100644 index 000000000..d2e8f0e9c --- /dev/null +++ b/internal/mcp/edit_acknowledgement_wire.go @@ -0,0 +1,173 @@ +package mcp + +import ( + "strconv" + + "github.com/zzet/gortex/internal/indexer" +) + +type acknowledgementLapCount struct { + Started uint64 `json:"started"` + Completed uint64 `json:"completed"` + NotApplicable uint64 `json:"not_applicable"` +} + +// Acknowledgement and patch walls have different owners and can overlap. The +// shared RPC identity lets a consumer deduplicate several physical file +// receipts; this object never claims its subtotal is a complete critical path. +type acknowledgementPayload struct { + State string `json:"state"` + RPCID *string `json:"rpc_id"` + Scope string `json:"scope"` + Denominator string `json:"denominator"` + Clock string `json:"clock"` + ReplyWallMS *float64 `json:"caller_response_ready_ms"` + ReplyState string `json:"caller_response_state"` + ReplyOutcome string `json:"caller_response_outcome,omitempty"` + ActualWallMS *float64 `json:"actual_handler_ms"` + ActualState string `json:"actual_handler_state"` + ActualFinishedLate bool `json:"actual_finished_after_reply"` + PhasesMS map[string]*float64 `json:"phases_ms"` + PhaseStates map[string]string `json:"phase_states"` + Counts map[string]acknowledgementLapCount `json:"counts"` + ObservedExecutionMS map[string]*float64 `json:"observed_execution_ms"` + Disjoint bool `json:"disjoint_observed_intervals"` + SubtotalMS *float64 `json:"observed_ack_subtotal_ms"` + UnattributedMS *float64 `json:"ack_unattributed_ms"` + SumState string `json:"sum_state"` + CombinedAckPatchSumState string `json:"combined_ack_patch_sum_state"` + ExistingPhaseScope string `json:"existing_ack_phase_scope"` + PublicationResidualState string `json:"publication_residual_state"` + DispatcherScope string `json:"dispatcher_scope"` + DispatcherReplyMS *float64 `json:"dispatcher_to_caller_response_ready_ms"` + DispatcherReplyState string `json:"dispatcher_response_state"` + ParserScope string `json:"parse_scope"` +} + +func buildAcknowledgementPayload(snapshot *editAcknowledgementSnapshot, state string) acknowledgementPayload { + out := acknowledgementPayload{ + State: state, Scope: "shared_rpc_mcp_bound_call", Clock: "unavailable", + Denominator: "bound_tool_entry_to_outer_return_before_transport_serialization", + ReplyState: state, ActualState: state, SumState: "unavailable", + DispatcherScope: "queue_and_preamble_precede_bound_call_subtotal", DispatcherReplyState: state, ParserScope: "unobserved", + CombinedAckPatchSumState: "not_additive_overlapping_lifetimes", + ExistingPhaseScope: "legacy_per_receipt_publication", PublicationResidualState: "legacy_per_receipt_publication", + PhasesMS: make(map[string]*float64, 11), PhaseStates: make(map[string]string, 11), + Counts: make(map[string]acknowledgementLapCount, 8), ObservedExecutionMS: make(map[string]*float64, 8), + } + for _, name := range []string{"dispatcher_queue", "dispatcher_preamble", "middleware", "parse", "disk_write", "publish", "reindex_wait", "edit_control", "reply_prepare", "caller_result", "transport"} { + out.PhasesMS[name], out.PhaseStates[name] = nil, state + } + if snapshot == nil { + return out + } + out.State = "observed" + id := "rpc-" + strconv.FormatUint(snapshot.RPCID, 10) + out.RPCID, out.Clock = &id, "monotonic" + out.ReplyWallMS, out.ReplyState, out.ReplyOutcome = snapshot.ReplyWallMS, snapshot.ReplyState, snapshot.ReplyOutcome + out.ActualWallMS, out.ActualState = snapshot.ActualWallMS, snapshot.ActualState + out.ActualFinishedLate, out.Disjoint = snapshot.ActualFinishedLate, snapshot.LapsDisjoint + out.DispatcherReplyMS, out.DispatcherReplyState = snapshot.DispatcherReplyMS, snapshot.DispatcherReplyState + if out.DispatcherReplyState == "" { + out.DispatcherReplyState = "unavailable_dispatcher_interface" + } + if snapshot.ParserScope != "" { + out.ParserScope = snapshot.ParserScope + } + out.ExistingPhaseScope, out.PublicationResidualState = "shared_rpc", "unavailable_shared_rpc_scope" + for _, name := range []string{"dispatcher_queue", "dispatcher_preamble"} { + out.PhaseStates[name] = "unavailable_dispatcher_interface" + } + if snapshot.DispatcherQueueState != "" { + out.PhasesMS["dispatcher_queue"], out.PhaseStates["dispatcher_queue"] = snapshot.DispatcherQueueMS, snapshot.DispatcherQueueState + } + if snapshot.DispatcherState != "" { + out.PhasesMS["dispatcher_preamble"], out.PhaseStates["dispatcher_preamble"] = snapshot.DispatcherPreambleMS, snapshot.DispatcherState + } + out.PhaseStates["transport"] = "unavailable_boundary" + var subtotal float64 + var subtotalNS int64 + exact := snapshot.hasReplyNS + observed, invalid := 0, false + if exact && (snapshot.replyNS < 0 || snapshot.ReplyWallMS == nil || float64(snapshot.replyNS)/1e6 != *snapshot.ReplyWallMS) { + invalid = true + } + for name, lap := range map[string]editAcknowledgementLapSnapshot{ + "middleware": snapshot.Middleware, "reply_prepare": snapshot.ReplyPrepare, "caller_result": snapshot.CallerResult, + "parse": snapshot.ParseGate, "disk_write": snapshot.DiskWrite, + "publish": snapshot.TicketAdmission, "reindex_wait": snapshot.ReindexWait, "edit_control": snapshot.EditControl, + } { + if lap.State == "" { + lap.State = "unobserved" + } + out.PhasesMS[name], out.PhaseStates[name] = lap.AckMS, lap.State + out.Counts[name] = acknowledgementLapCount{lap.Started, lap.Completed, lap.NotApplicable} + out.ObservedExecutionMS[name] = lap.ObservedMS + if lap.Completed > lap.Started || lap.NotApplicable > lap.Completed || + ((lap.AckMS != nil || lap.ObservedMS != nil) && (lap.Started == 0 || lap.Completed != lap.Started)) || + (lap.AckMS != nil && (lap.ObservedMS == nil || *lap.AckMS != *lap.ObservedMS)) { + out.PhasesMS[name], out.ObservedExecutionMS[name] = nil, nil + out.PhaseStates[name], invalid = "invalid_count", true + continue + } + if lap.State == "invalid_interval" || lap.State == "outside_execution_window" { + invalid = true + } + if lap.AckMS != nil { + if !lap.hasAckNS { + exact = false // legacy synthetic snapshots have no actual nanosecond evidence + } else if lap.ackNS < 0 || float64(lap.ackNS)/1e6 != *lap.AckMS || subtotalNS > (1<<63-1)-lap.ackNS { + invalid = true + } else { + subtotalNS += lap.ackNS + } + subtotal += *lap.AckMS + observed++ + } + } + if exact { + // Compare/sum observed integers, not rounded per-lap milliseconds. + // No tolerance, clamp or unobserved remainder is introduced. + subtotal = float64(subtotalNS) / 1e6 + } + switch { + case !snapshot.LapsDisjoint: + out.SumState = "invalid_overlap" + case invalid: + out.SumState = "invalid_interval" + case snapshot.ReplyState != "measured" || snapshot.ReplyWallMS == nil: + out.SumState = "pending_or_unavailable_reply" + case observed == 0: + out.SumState = "no_ack_intervals_observed" + case exact && (subtotalNS < 0 || subtotalNS > snapshot.replyNS) || !exact && (subtotal < 0 || subtotal > *snapshot.ReplyWallMS): + out.SumState = "invalid_total" + default: + unattributed := *snapshot.ReplyWallMS - subtotal + if exact { + unattributed = float64(snapshot.replyNS-subtotalNS) / 1e6 + } + out.SubtotalMS, out.UnattributedMS, out.SumState = &subtotal, &unattributed, "partial" + } + return out +} + +func attachAcknowledgementPayload(payload map[string]any, snapshot *editAcknowledgementSnapshot, state string) { + ack := buildAcknowledgementPayload(snapshot, state) + payload["acknowledgement"] = ack + if snapshot == nil { + return // preserve independently valid legacy per-file evidence + } + phases, _ := payload["phases_ms"].(map[string]*float64) + states, _ := payload["phase_states"].(map[string]string) + for _, name := range []string{"dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait", "edit_control", "caller_result"} { + phases[name], states[name] = ack.PhasesMS[name], ack.PhaseStates[name] + } + window := payload["phase_timing"].(indexer.EditTimingWindow) + window.Clock, window.Coverage = "monotonic", "partial" + window.AcknowledgementWallMS = ack.ReplyWallMS + // The shared RPC origin and per-file publication origin are different. + // Keep the original receipt wall/patch evidence, but do not subtract the + // replacement shared-RPC laps from its per-file publication interval. + window.PublicationUnattributedWallMS = nil + payload["phase_timing"] = window +} diff --git a/internal/mcp/edit_acknowledgement_wire_test.go b/internal/mcp/edit_acknowledgement_wire_test.go new file mode 100644 index 000000000..b5ae8877b --- /dev/null +++ b/internal/mcp/edit_acknowledgement_wire_test.go @@ -0,0 +1,233 @@ +package mcp + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/indexer" +) + +func decodeAcknowledgementResult(t *testing.T, result *mcp.CallToolResult, err error) map[string]any { + t.Helper() + require.NoError(t, err) + require.NotNil(t, result) + require.False(t, result.IsError) + require.NotEmpty(t, result.Content) + text, ok := result.Content[0].(mcp.TextContent) + require.True(t, ok) + var payload map[string]any + require.NoError(t, json.Unmarshal([]byte(text.Text), &payload)) + return payload +} + +func ackWireMilliseconds(value float64) *float64 { return &value } + +func TestAcknowledgementWirePartialSubtotalHasSeparateDenominator(t *testing.T) { + timings := (indexer.PublicationPhaseSnapshot{}).EditTimings() + timings.Milliseconds["gate_wait"], timings.States["gate_wait"] = ackWireMilliseconds(2), "measured" + timings.Window.PatchWallMS, timings.Window.PublicationWallMS = ackWireMilliseconds(30), ackWireMilliseconds(100) + timings.Window.PublicationUnattributedWallMS = ackWireMilliseconds(50) + payload := map[string]any{"phases_ms": timings.Milliseconds, "phase_states": timings.States, "phase_timing": timings.Window} + snapshot := editAcknowledgementSnapshot{ + RPCID: 3, ReplyWallMS: ackWireMilliseconds(20), ReplyState: "measured", ActualState: "measured", + LapsDisjoint: true, ParseGate: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(1), AckMS: ackWireMilliseconds(1), State: "measured", Started: 1, Completed: 1}, + DiskWrite: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(2), AckMS: ackWireMilliseconds(2), State: "measured", Started: 1, Completed: 1}, + TicketAdmission: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(3), AckMS: ackWireMilliseconds(3), State: "measured", Started: 1, Completed: 1}, + ReindexWait: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(4), AckMS: ackWireMilliseconds(4), State: "measured", Started: 1, Completed: 1}, + } + attachAcknowledgementPayload(payload, &snapshot, "observed") + ack := payload["acknowledgement"].(acknowledgementPayload) + require.Equal(t, "rpc-3", *ack.RPCID) + require.Equal(t, "shared_rpc_mcp_bound_call", ack.Scope) + require.Equal(t, "partial", ack.SumState) + require.Equal(t, 10.0, *ack.SubtotalMS) + require.Equal(t, 10.0, *ack.UnattributedMS) + require.Greater(t, *ack.UnattributedMS, 0.05*(*ack.ReplyWallMS), "a large real gap is not assigned to a phase to force the 5% gate") + require.Equal(t, "not_additive_overlapping_lifetimes", ack.CombinedAckPatchSumState) + window := payload["phase_timing"].(indexer.EditTimingWindow) + require.Equal(t, 20.0, *window.AcknowledgementWallMS) + require.Equal(t, 30.0, *window.PatchWallMS) + require.Equal(t, 100.0, *window.PublicationWallMS) + require.Nil(t, window.PublicationUnattributedWallMS, "do not subtract shared-RPC values from a per-file interval") + require.Equal(t, 2.0, *payload["phases_ms"].(map[string]*float64)["gate_wait"]) + encoded, err := json.Marshal(payload) + require.NoError(t, err) + var wire map[string]any + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.Len(t, wire["phases_ms"], 19) // original required keys plus four explicit measured additions + wireAck := wire["acknowledgement"].(map[string]any) + phases := wireAck["phases_ms"].(map[string]any) + require.Len(t, phases, 11) + for _, name := range []string{"dispatcher_queue", "dispatcher_preamble", "middleware", "edit_control", "reply_prepare", "caller_result", "transport"} { + require.Contains(t, phases, name) + require.Nil(t, phases[name], name) + } +} + +func TestAcknowledgementWireRejectsInvalidSumsAndLateClipping(t *testing.T) { + for _, overlap := range []bool{false, true} { + snapshot := editAcknowledgementSnapshot{RPCID: 1, ReplyWallMS: ackWireMilliseconds(1), ReplyState: "measured", LapsDisjoint: !overlap, + DiskWrite: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(2), AckMS: ackWireMilliseconds(2), State: "measured", Started: 1, Completed: 1}} + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Nil(t, ack.SubtotalMS) + require.Nil(t, ack.UnattributedMS) + require.Contains(t, []string{"invalid_total", "invalid_overlap"}, ack.SumState) + } + start := time.Now() + record := &editAcknowledgement{entered: start} + lap := record.beginLapAt(editAckDiskWrite, start.Add(time.Millisecond)) + record.finishReplyAt(start.Add(2*time.Millisecond), "go_error") + lap.finishAt(start.Add(5 * time.Millisecond)) + record.finishActualAt(start.Add(6 * time.Millisecond)) + snapshot := record.snapshot() + ack := buildAcknowledgementPayload(&snapshot, "observed") + require.Nil(t, ack.PhasesMS["disk_write"]) + require.Equal(t, 4.0, *ack.ObservedExecutionMS["disk_write"]) + require.Equal(t, "outside_reply_window", ack.PhaseStates["disk_write"]) + require.Nil(t, ack.SubtotalMS) + invalid := editAcknowledgementSnapshot{ReplyWallMS: ackWireMilliseconds(10), ReplyState: "measured", LapsDisjoint: true, + DiskWrite: editAcknowledgementLapSnapshot{ObservedMS: ackWireMilliseconds(2), AckMS: ackWireMilliseconds(2), State: "measured", Started: 2, Completed: 1}} + bad := buildAcknowledgementPayload(&invalid, "observed") + require.Equal(t, "invalid_count", bad.PhaseStates["disk_write"]) + require.Nil(t, bad.PhasesMS["disk_write"]) + require.Nil(t, bad.SubtotalMS) +} + +func TestMutationStatusAcknowledgementSharesRPCAndPreservesPublication(t *testing.T) { + s := newMutationServer() + s.ToolCallTimeout = -1 + s.watcher = &canonicalPublicationWatcher{} + root := t.TempDir() + var records []*mutationCommitRecord + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + for _, name := range []string{"a.go", "b.go"} { + path := filepath.Join(root, name) + record, err := s.commitFileMutation(ctx, "write_file", "", "", name, path, []byte("package p\n"), 0o600) + if err != nil { + return nil, err + } + record.recordGraph(s.mutationReindexState(ctx, path)) + records = append(records, record) + } + return mcp.NewToolResultText("done"), nil + }) + _, err := h(canonicalTimingContext(), editOperationRequest("write_file", "")) + require.NoError(t, err) + require.Len(t, records, 2) + var firstRPC string + for _, record := range records { + timeline, _ := retainedMutationPublication(record) + before := timeline.Snapshot() + payload := s.mutationStatusPayload(record) + require.Equal(t, before, payload["publication_phases"], "legacy first-wins origin and offsets remain unchanged") + ack := payload["acknowledgement"].(acknowledgementPayload) + require.Equal(t, "observed", ack.State) + if firstRPC == "" { + firstRPC = *ack.RPCID + } + require.Equal(t, firstRPC, *ack.RPCID) + require.EqualValues(t, 2, ack.Counts["disk_write"].Completed) + require.EqualValues(t, 2, ack.Counts["publish"].Completed) + require.EqualValues(t, 2, ack.Counts["reindex_wait"].Completed) + require.Equal(t, "partial", ack.SumState) + phases := payload["phases_ms"].(map[string]*float64) + original := *phases["disk_write"] + *phases["disk_write"] = 999 + require.Equal(t, original, *s.mutationStatusPayload(record)["phases_ms"].(map[string]*float64)["disk_write"]) + } + ack := retainedMutationAcknowledgement(records[0]) + for i := range maxMutationCommits { + s.mutationCommits.put(&mutationCommitRecord{id: fmt.Sprintf("wire-cap-%d", i), startedAt: time.Now()}) + } + ack.finishActualAt(time.Now()) // first-wins completion cannot resurrect history + missing := s.mutationStatusPayload(records[0])["acknowledgement"].(acknowledgementPayload) + require.Equal(t, "unavailable_recording_or_evicted", missing.State) + require.Nil(t, missing.RPCID) + require.Nil(t, missing.ReplyWallMS) +} + +func TestAtomicBatchResponseAndStatusExposeOnlyBoundedTimingReference(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = -1 + file := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"format": "json", "transaction_id": "wire-batch", "edits": []any{ + map[string]any{"op": "edit_file", "path": file, "old_string": "before", "new_string": "after"}, + }} + h := s.boundToolHandler(s.handleAtomicBatchEdit) + result, err := h(t.Context(), req) + first := decodeAcknowledgementResult(t, result, err) + id := first["acknowledgement_receipt"].(string) + status := editOperationRequest("mutation_status", "") + status.Params.Arguments = map[string]any{"receipt": id, "format": "json"} + result, err = s.handleMutationStatus(t.Context(), status) + timing := decodeAcknowledgementResult(t, result, err) + require.Equal(t, "batch_acknowledgement", timing["receipt_kind"]) + require.Equal(t, true, timing["found"]) + require.NotContains(t, timing, "disk_status") + require.NotContains(t, timing, "graph_status") + result, err = h(t.Context(), req) + duplicate := decodeAcknowledgementResult(t, result, err) + require.Equal(t, id, duplicate["acknowledgement_receipt"]) + result, err = s.handleMutationStatus(t.Context(), status) + require.Equal(t, timing["acknowledgement"], decodeAcknowledgementResult(t, result, err)["acknowledgement"]) + manifest, err := os.ReadFile(batchManifestPath("wire-batch")) + require.NoError(t, err) + require.NotContains(t, string(manifest), id) + require.NotContains(t, string(manifest), "acknowledgement") + compact := editOperationRequest("batch_edit", "") + compact.Params.Arguments = map[string]any{"transaction_id": "wire-batch", "status_only": true, "compact": true} + result, err = h(t.Context(), compact) + require.NoError(t, err) + require.Contains(t, result.Content[0].(mcp.TextContent).Text, "acknowledgement_receipt="+id) + status.Params.Arguments = map[string]any{"receipt": id, "format": "toon"} + result, err = s.handleMutationStatus(t.Context(), status) + require.NoError(t, err) + require.Contains(t, result.Content[0].(mcp.TextContent).Text, "shared_rpc_mcp_bound_call") + for i := range maxMutationCommits { + s.mutationCommits.put(&mutationCommitRecord{id: fmt.Sprintf("wire-batch-cap-%d", i), startedAt: time.Now()}) + } + status.Params.Arguments = map[string]any{"receipt": id, "format": "json"} + result, err = s.handleMutationStatus(t.Context(), status) + unavailable := decodeAcknowledgementResult(t, result, err) + require.Equal(t, false, unavailable["found"]) + require.Equal(t, "unavailable_ledger_entry_or_scope", unavailable["acknowledgement"].(map[string]any)["state"]) + require.NotContains(t, unavailable, "disk_status") +} + +func TestBatchAcknowledgementStatusHonoursCheckoutIncarnationAndRestart(t *testing.T) { + s := newMutationServer() + ctx, _, _ := s.withEditAcknowledgement(t.Context()) + state := &batchTransactionState{receipt: batchTransactionReceipt{TransactionID: "scoped", CheckoutID: "checkout-a", Incarnation: "inc-a"}} + s.bindBatchAcknowledgement(ctx, state) + id := state.acknowledgementReceipt + for _, tt := range []struct { + checkout, incarnation string + found bool + }{ + {"checkout-a", "inc-a", true}, {"checkout-a", "inc-b", false}, {"checkout-b", "inc-a", false}, {"", "", false}, + } { + scope := context.WithValue(t.Context(), checkoutMutationContextKey{}, &checkoutMutationState{checkoutID: tt.checkout, incarnation: tt.incarnation}) + payload, handled := s.batchAcknowledgementReceiptPayload(scope, id) + require.True(t, handled) + require.Equal(t, tt.found, payload["found"]) + if !tt.found { + require.Nil(t, payload["acknowledgement"].(acknowledgementPayload).RPCID) + } + require.NotContains(t, payload, "disk_status") + require.NotContains(t, payload, "graph_status") + } + restarted := &Server{} + payload, handled := restarted.batchAcknowledgementReceiptPayload(t.Context(), id) + require.True(t, handled) + require.Equal(t, false, payload["found"]) + require.Equal(t, "unavailable_ledger_entry_or_scope", payload["acknowledgement"].(acknowledgementPayload).State) +} diff --git a/internal/mcp/edit_cpu_observations.go b/internal/mcp/edit_cpu_observations.go new file mode 100644 index 000000000..889540b39 --- /dev/null +++ b/internal/mcp/edit_cpu_observations.go @@ -0,0 +1,177 @@ +package mcp + +import ( + "fmt" + "os" + "sync/atomic" + "time" + + "github.com/zzet/gortex/internal/platform" +) + +type editOperationCountSnapshot struct { + Completed uint64 `json:"completed"` + CommittedWithError uint64 `json:"committed_with_error"` + FailedAfterSourceEffect uint64 `json:"failed_after_source_effect"` + RolledBack uint64 `json:"rolled_back"` + UnknownSourceComparison uint64 `json:"unknown_source_comparison"` +} + +func (c *editOperationCounters) snapshot() editOperationCountSnapshot { + return editOperationCountSnapshot{ + Completed: c.completed.Load(), CommittedWithError: c.committedWithError.Load(), + FailedAfterSourceEffect: c.failedAfterSourceEffect.Load(), RolledBack: c.rolledBack.Load(), + UnknownSourceComparison: c.unknownSourceComparison.Load(), + } +} + +func (c editOperationCountSnapshot) follows(before editOperationCountSnapshot) bool { + return c.Completed >= before.Completed && c.CommittedWithError >= before.CommittedWithError && + c.FailedAfterSourceEffect >= before.FailedAfterSourceEffect && c.RolledBack >= before.RolledBack && + c.UnknownSourceComparison >= before.UnknownSourceComparison +} + +type editCPUSample struct { + started, finished time.Duration + boundsKnown bool + cpu time.Duration + available bool + reason string + before, after editOperationCountSnapshot +} + +type editCPUObservation struct { + origin time.Time + epoch string + baseline editCPUSample // immutable: no reset or initialization during stats reads +} + +type editCPUSampleWire struct { + StartedSeconds *float64 `json:"sample_started_seconds"` + FinishedSeconds *float64 `json:"sample_finished_seconds"` + CPUSeconds *float64 `json:"cpu_seconds"` + CPUAvailable bool `json:"cpu_available"` + CPUReason string `json:"cpu_reason,omitempty"` + CountsBefore editOperationCountSnapshot `json:"counts_before"` + CountsAfter editOperationCountSnapshot `json:"counts_after"` +} + +type editProcessWindow struct { + Scope string `json:"scope"` + CountScope string `json:"count_scope"` + CPUScope string `json:"cpu_scope"` + Epoch string `json:"epoch"` + Baseline *editCPUSampleWire `json:"baseline"` + Sample editCPUSampleWire `json:"sample"` + CPUSecondsPerEdit *float64 `json:"cpu_seconds_per_edit"` + AverageState string `json:"average_state"` + AverageReason string `json:"average_reason,omitempty"` +} + +var editCPUEpochSequence atomic.Uint64 + +// initializeEditCPUObservation runs only at the end of NewServer, before any +// caller can execute its registered tools. A literal Server has no baseline. +func (s *Server) initializeEditCPUObservation(sample func() (time.Duration, error)) { + origin := time.Now() + s.editCPU = &editCPUObservation{ + origin: origin, + epoch: fmt.Sprintf("%d-%d-%d", os.Getpid(), origin.UnixNano(), editCPUEpochSequence.Add(1)), + baseline: captureEditCPU(&s.editOperations, origin, sample), + } +} + +// captureEditCPU is called only for constructor/stats observations. All five +// outcome counters bracket ONE process sample. An unstable bracket is retained, +// never retried into a silently different window or presented as an exact count. +func captureEditCPU(c *editOperationCounters, origin time.Time, sample func() (time.Duration, error)) editCPUSample { + result := editCPUSample{before: c.snapshot(), boundsKnown: !origin.IsZero()} + if result.boundsKnown { + result.started = time.Since(origin) + } + cpu, err := sample() + if result.boundsKnown { + result.finished = time.Since(origin) + } + result.after = c.snapshot() + if err != nil { + reason := []rune(err.Error()) + if len(reason) > 256 { + reason = reason[:256] + } + result.reason = string(reason) + } else if cpu < 0 { + result.reason = "invalid negative process CPU sample" + } else { + result.cpu, result.available = cpu, true + } + return result +} + +func (sample editCPUSample) wire() editCPUSampleWire { + result := editCPUSampleWire{ + CPUAvailable: sample.available, CPUReason: sample.reason, + CountsBefore: sample.before, CountsAfter: sample.after, + } + if sample.boundsKnown { + started, finished := sample.started.Seconds(), sample.finished.Seconds() + result.StartedSeconds, result.FinishedSeconds = &started, &finished + } + if sample.available { + cpu := sample.cpu.Seconds() + result.CPUSeconds = &cpu + } + return result +} + +func (s *Server) editProcessWindowSnapshot(sample func() (time.Duration, error)) editProcessWindow { + var origin time.Time + if s.editCPU != nil { + origin = s.editCPU.origin + } + return projectEditProcessWindow(s.editCPU, captureEditCPU(&s.editOperations, origin, sample)) +} + +func projectEditProcessWindow(observation *editCPUObservation, current editCPUSample) editProcessWindow { + result := editProcessWindow{ + Scope: "process_window", CountScope: "server_all_sessions_views", + CPUScope: "whole_process_including_background", Sample: current.wire(), AverageState: "unavailable", + } + if observation == nil { + result.AverageReason = "baseline_uninitialized" + return result + } + result.Epoch = observation.epoch + baseline := observation.baseline + wire := baseline.wire() // caller-owned scalar pointers; never expose the stored baseline + result.Baseline = &wire + switch { + case !baseline.available: + result.AverageReason = "baseline_cpu_unavailable" + case baseline.before != (editOperationCountSnapshot{}) || baseline.after != (editOperationCountSnapshot{}): + result.AverageReason = "baseline_counts_not_zero" + case !current.available: + result.AverageReason = "cpu_unavailable" + case !baseline.boundsKnown || !current.boundsKnown || baseline.started < 0 || baseline.finished < baseline.started || + current.started < baseline.finished || current.finished < current.started || current.cpu < baseline.cpu: + result.AverageReason = "invalid_order" + case !current.before.follows(baseline.after) || !current.after.follows(current.before): + result.AverageReason = "invalid_count_order" + case current.before != current.after: + result.AverageReason = "unstable_count_bracket" + case current.after.Completed == 0: + result.AverageReason = "no_completed_edits" + case current.after.CommittedWithError != 0 || current.after.FailedAfterSourceEffect != 0 || current.after.UnknownSourceComparison != 0 || current.after.RolledBack != 0: + result.AverageReason = "non_clean_window" + default: + average := (current.cpu - baseline.cpu).Seconds() / float64(current.after.Completed) + result.CPUSecondsPerEdit, result.AverageState = &average, "measured_process_window" + } + return result +} + +// processEditWindow is shared by graph_stats and gortex://stats. The graph part +// of those payloads is view-scoped; this explicit process-window block is not. +func (s *Server) processEditWindow() editProcessWindow { + return s.editProcessWindowSnapshot(platform.ProcessCPUTime) +} diff --git a/internal/mcp/edit_cpu_observations_test.go b/internal/mcp/edit_cpu_observations_test.go new file mode 100644 index 000000000..61160cf58 --- /dev/null +++ b/internal/mcp/edit_cpu_observations_test.go @@ -0,0 +1,194 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func editCPUProjectionFixture() (*editCPUObservation, editCPUSample) { + return &editCPUObservation{epoch: "test-server-epoch", baseline: editCPUSample{ + boundsKnown: true, finished: time.Millisecond, cpu: time.Second, available: true, + }}, editCPUSample{ + boundsKnown: true, started: 2 * time.Millisecond, finished: 3 * time.Millisecond, + cpu: 3 * time.Second, available: true, + before: editOperationCountSnapshot{Completed: 2}, after: editOperationCountSnapshot{Completed: 2}, + } +} + +func TestEditCPUProjectionStatesAndRawEvidence(t *testing.T) { + for _, tt := range []struct { + name string + change func(*editCPUObservation, *editCPUSample) + reason string + zero bool + }{ + {name: "clean process window"}, + {name: "known zero CPU", zero: true, change: func(b *editCPUObservation, c *editCPUSample) { b.baseline.cpu, c.cpu = 0, 0 }}, + {name: "no completed edits", reason: "no_completed_edits", change: func(_ *editCPUObservation, c *editCPUSample) { + c.before, c.after = editOperationCountSnapshot{}, editOperationCountSnapshot{} + }}, + {name: "baseline unsupported", reason: "baseline_cpu_unavailable", change: func(b *editCPUObservation, _ *editCPUSample) { + b.baseline.available, b.baseline.reason = false, "unsupported" + }}, + {name: "sample unavailable", reason: "cpu_unavailable", change: func(_ *editCPUObservation, c *editCPUSample) { c.available, c.reason = false, "sampling failed" }}, + {name: "unstable count", reason: "unstable_count_bracket", change: func(_ *editCPUObservation, c *editCPUSample) { c.before.Completed = 1 }}, + {name: "reversed sample clock", reason: "invalid_order", change: func(_ *editCPUObservation, c *editCPUSample) { c.finished = c.started - 1 }}, + {name: "CPU regression", reason: "invalid_order", change: func(_ *editCPUObservation, c *editCPUSample) { c.cpu = time.Millisecond }}, + {name: "count regression", reason: "invalid_count_order", change: func(_ *editCPUObservation, c *editCPUSample) { c.before.Completed = 3 }}, + {name: "baseline already counted", reason: "baseline_counts_not_zero", change: func(b *editCPUObservation, _ *editCPUSample) { b.baseline.after.Completed = 1 }}, + {name: "committed warning", reason: "non_clean_window", change: func(_ *editCPUObservation, c *editCPUSample) { + c.before.CommittedWithError, c.after.CommittedWithError = 1, 1 + }}, + {name: "rollback is a subset", reason: "non_clean_window", change: func(_ *editCPUObservation, c *editCPUSample) { + c.before.FailedAfterSourceEffect, c.after.FailedAfterSourceEffect = 1, 1 + c.before.RolledBack, c.after.RolledBack = 1, 1 + }}, + {name: "unknown comparison", reason: "non_clean_window", change: func(_ *editCPUObservation, c *editCPUSample) { + c.before.UnknownSourceComparison, c.after.UnknownSourceComparison = 1, 1 + }}, + } { + t.Run(tt.name, func(t *testing.T) { + baseline, current := editCPUProjectionFixture() + if tt.change != nil { + tt.change(baseline, ¤t) + } + got := projectEditProcessWindow(baseline, current) + require.Equal(t, baseline.epoch, got.Epoch) + require.Equal(t, current.before, got.Sample.CountsBefore) + require.Equal(t, current.after, got.Sample.CountsAfter) + require.Equal(t, baseline.baseline.after, got.Baseline.CountsAfter) + encoded, err := json.Marshal(got) + require.NoError(t, err) + var wire map[string]any + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.Contains(t, wire, "cpu_seconds_per_edit", "missing must be explicit null, never an omitted/zero average") + if tt.reason != "" { + require.Nil(t, got.CPUSecondsPerEdit) + require.Nil(t, wire["cpu_seconds_per_edit"]) + require.Equal(t, "unavailable", got.AverageState) + require.Equal(t, tt.reason, got.AverageReason) + } else { + require.NotNil(t, got.CPUSecondsPerEdit) + want := 1.0 // (3s process CPU - 1s baseline) / 2 clean completed edits + if tt.zero { + want = 0 + } + require.Equal(t, want, *got.CPUSecondsPerEdit) + require.Equal(t, "measured_process_window", got.AverageState) + } + if !current.available { + require.Nil(t, got.Sample.CPUSeconds) + require.Equal(t, "sampling failed", got.Sample.CPUReason) + } + }) + } +} + +func TestEditCPUCollectorBracketsExactlyOneSampleWithoutReset(t *testing.T) { + s := &Server{} + s.initializeEditCPUObservation(func() (time.Duration, error) { return time.Second, nil }) + epoch := s.editCPU.epoch + calls := 0 + first := s.editProcessWindowSnapshot(func() (time.Duration, error) { + calls++ + s.editOperations.completed.Add(1) // actual counter change during the CPU endpoint + return 3 * time.Second, nil + }) + require.Equal(t, 1, calls) + require.Zero(t, first.Sample.CountsBefore.Completed) + require.EqualValues(t, 1, first.Sample.CountsAfter.Completed) + require.Equal(t, "unstable_count_bracket", first.AverageReason) + require.Nil(t, first.CPUSecondsPerEdit) + second := s.editProcessWindowSnapshot(func() (time.Duration, error) { calls++; return 3 * time.Second, nil }) + require.Equal(t, 2, calls) + require.Equal(t, epoch, second.Epoch) + require.EqualValues(t, 1, second.Sample.CountsBefore.Completed) + require.Equal(t, second.Sample.CountsBefore, second.Sample.CountsAfter) + require.Equal(t, 2.0, *second.CPUSecondsPerEdit) + require.Zero(t, second.Baseline.CountsAfter.Completed) + require.EqualValues(t, 1, s.editOperations.completed.Load()) +} + +func TestEditCPUWireCannotMutateStoredBaseline(t *testing.T) { + s := &Server{} + s.initializeEditCPUObservation(func() (time.Duration, error) { return time.Second, nil }) + s.editOperations.completed.Add(1) + sample := func() (time.Duration, error) { return 2 * time.Second, nil } + first := s.editProcessWindowSnapshot(sample) + *first.Baseline.CPUSeconds = 1000 + *first.Baseline.StartedSeconds = -1000 + first.Baseline.CountsAfter.Completed = 1000 + *first.CPUSecondsPerEdit = 1000 + second := s.editProcessWindowSnapshot(sample) + require.Equal(t, 1.0, *second.Baseline.CPUSeconds) + require.GreaterOrEqual(t, *second.Baseline.StartedSeconds, 0.0) + require.Zero(t, second.Baseline.CountsAfter.Completed) + require.Equal(t, 1.0, *second.CPUSecondsPerEdit) +} + +func TestEditCPUConstructorBaselineAndEpoch(t *testing.T) { + first, _ := setupTestServer(t) + second, _ := setupTestServer(t) + require.NotNil(t, first.editCPU) + require.NotNil(t, second.editCPU) + require.NotEmpty(t, first.editCPU.epoch) + require.NotEqual(t, first.editCPU.epoch, second.editCPU.epoch) + require.Zero(t, first.editCPU.baseline.before) + require.Zero(t, first.editCPU.baseline.after) + require.True(t, first.editCPU.baseline.boundsKnown) + if first.editCPU.baseline.available { + require.GreaterOrEqual(t, first.editCPU.baseline.cpu, time.Duration(0)) + } else { + require.NotEmpty(t, first.editCPU.baseline.reason, "unsupported/error must remain explicit") + } +} + +func TestEditCPULiteralAndUnavailableBaselineStayUnavailable(t *testing.T) { + s := &Server{} + got := s.editProcessWindowSnapshot(func() (time.Duration, error) { return time.Second, nil }) + require.Empty(t, got.Epoch) + require.Nil(t, got.Baseline) + require.Nil(t, got.Sample.StartedSeconds) + require.Equal(t, "baseline_uninitialized", got.AverageReason) + s.initializeEditCPUObservation(func() (time.Duration, error) { return 0, errors.New("test CPU unavailable") }) + s.editOperations.completed.Add(1) + epoch := s.editCPU.epoch + got = s.editProcessWindowSnapshot(func() (time.Duration, error) { return time.Second, nil }) + require.Equal(t, epoch, got.Epoch) + require.Nil(t, got.Baseline.CPUSeconds) + require.Equal(t, "test CPU unavailable", got.Baseline.CPUReason) + require.NotNil(t, got.Sample.CPUSeconds, "raw later endpoint remains available") + require.Nil(t, got.CPUSecondsPerEdit) + require.Equal(t, "baseline_cpu_unavailable", got.AverageReason, "stats reads never replace a failed baseline") +} + +func TestEditCPUStatsProcessScopeSurvivesResourceAndToolViews(t *testing.T) { + stack := newViewStack(t) + stack.srv.editOperations.completed.Add(3) // projection scope fixture, not a real edit oracle + var resource, tool editProcessWindow + _, err := stack.readBoundResource(t, stack.worktreeRoot, "gortex://stats", func(ctx context.Context) ([]mcp.ResourceContents, error) { + resource = stack.srv.buildGraphStatsPayload(ctx)["process_edit_window"].(editProcessWindow) + return nil, nil + }) + require.NoError(t, err) + _, err = stack.callWithView(t, "", "graph_stats", nil, func(ctx context.Context) (*mcp.CallToolResult, error) { + tool = stack.srv.buildGraphStatsPayload(ctx)["process_edit_window"].(editProcessWindow) + return mcp.NewToolResultText(`{"ok":true}`), nil + }) + require.NoError(t, err) + require.Equal(t, "process_window", tool.Scope) + require.Equal(t, "server_all_sessions_views", tool.CountScope) + require.Equal(t, "whole_process_including_background", tool.CPUScope) + require.Equal(t, resource.Epoch, tool.Epoch) + require.Equal(t, resource.Baseline, tool.Baseline) + require.EqualValues(t, 3, resource.Sample.CountsAfter.Completed) + require.EqualValues(t, 3, tool.Sample.CountsAfter.Completed) + // Live CPU/timestamps differ between sequential reads; graph-view equality + // remains covered separately by TestResourceStatsEqualsTheGraphStatsTool. +} diff --git a/internal/mcp/edit_operation_observations.go b/internal/mcp/edit_operation_observations.go new file mode 100644 index 000000000..84c022ad8 --- /dev/null +++ b/internal/mcp/edit_operation_observations.go @@ -0,0 +1,168 @@ +package mcp + +import ( + "bytes" + "context" + "sync/atomic" + + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// These are Server-wide actual-handler outcomes, not delivered acknowledgements, +// per-file commits, selected-view counts, or native editor operations. A daemon's +// SharedServer shares them across its clients. No process sampling occurs here. +// All fields are monotonic for this Server's lifetime; reading never resets them. +type editOperationCounters struct { + completed atomic.Uint64 + committedWithError atomic.Uint64 + failedAfterSourceEffect atomic.Uint64 + rolledBack atomic.Uint64 // subset of failedAfterSourceEffect + unknownSourceComparison atomic.Uint64 +} + +type editOperationContextKey struct{} + +type editOperationWitness struct { + owner *editOperationCounters + flags atomic.Uint32 +} + +const ( + editSourceEffect uint32 = 1 << iota + editSourceComplete + editSourceWarning + editSourceRolledBack + editSourceUnknownComparison +) + +// measuredSourceEdit deliberately follows the five requested edit surfaces, +// including their facade spellings. Other tools and unresolved operations never +// allocate a witness. Physical watcher patches are not logical editor stimuli. +func measuredSourceEdit(req mcp.CallToolRequest) bool { + switch req.Params.Name { + case "write_file", "edit_file", "edit_symbol", "batch_edit", "move_symbol": + return true + case "edit", "refactor": + // Match the facade's normalization/inference/defaults too: an + // omitted operation with a file/symbol/changes selector is callable. + operation := resolveFacadeOperationAlias(req.Params.Name, normalizeFacadeOperation(req.GetString("operation", ""))) + if operation == "" { + operation = inferFacadeOperation(req.Params.Name, req.GetArguments()) + } + if operation == "" { + operation = defaultFacadeOperation(req.Params.Name) + } + if req.Params.Name == "refactor" { + return operation == "move" + } + switch operation { + case "write", "file", "symbol", "batch": + return true + } + } + return false +} + +func editOperationFrom(ctx context.Context) *editOperationWitness { + w, _ := ctx.Value(editOperationContextKey{}).(*editOperationWitness) + return w +} + +// observeEditSourceEffect records a successful forward source write/removal. +// Rollback/journal writes must not call it. Unchanged rewrites are excluded; +// unavailable prior bytes remain unknown instead of being guessed equal/changed. +func observeEditSourceEffect(ctx context.Context, changed, comparisonKnown bool) { + editOperationFrom(ctx).observeSourceEffect(changed, comparisonKnown) +} + +func observeEditSourceReplacement(ctx context.Context, before, after []byte, existed, comparisonKnown bool) { + w := editOperationFrom(ctx) + if w == nil { + return // no extra byte comparison for an unobserved/raw caller + } + w.observeSourceEffect(!existed || !bytes.Equal(before, after), comparisonKnown) +} + +func (w *editOperationWitness) observeSourceEffect(changed, comparisonKnown bool) { + if w == nil || (comparisonKnown && !changed) { + return + } + flags := editSourceEffect + if !comparisonKnown { + flags |= editSourceUnknownComparison + } + w.flags.Or(flags) +} + +func observeEditSourceComplete(ctx context.Context) { + if w := editOperationFrom(ctx); w != nil { + w.flags.Or(editSourceComplete) + } +} + +// The batch's typed terminal receipt supplies these facts; text formatting and +// err==nil cannot distinguish committed bytes with a journal warning from an +// aborted/rolled-back/conflicted transaction. +func observeEditSourceTerminal(ctx context.Context, committed, warning, rolledBack bool) { + w := editOperationFrom(ctx) + if w == nil { + return + } + var flags uint32 + if committed { + flags |= editSourceComplete + } + if warning { + flags |= editSourceWarning + } + if rolledBack { + flags |= editSourceRolledBack + } + w.flags.Or(flags) +} + +func (w *editOperationWitness) finish(result *mcp.CallToolResult, err error) { + flags := w.flags.Load() + if flags&editSourceEffect == 0 { + return // replay/status/dry-run/no-op/refusal before any source effect + } + cleanResult := err == nil && result != nil && !result.IsError + if flags&editSourceComplete != 0 { + switch { + case !cleanResult || flags&editSourceWarning != 0: + w.owner.committedWithError.Add(1) + case flags&editSourceUnknownComparison != 0: + w.owner.unknownSourceComparison.Add(1) + default: + w.owner.completed.Add(1) + } + return + } + w.owner.failedAfterSourceEffect.Add(1) + if flags&editSourceRolledBack != 0 { + w.owner.rolledBack.Add(1) + } +} + +func (s *Server) observeEditOperationHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, req mcp.CallToolRequest) (result *mcp.CallToolResult, err error) { + if s == nil || !measuredSourceEdit(req) { + return h(ctx, req) + } + return s.runMeasuredEditOperation(ctx, req, h) + } +} + +// runMeasuredEditOperation is called only after measuredSourceEdit succeeded. +// The bound-call observer shares that classification; completion still runs +// inside the actual handler, including after an abandoned caller has returned. +func (s *Server) runMeasuredEditOperation(ctx context.Context, req mcp.CallToolRequest, h mcpserver.ToolHandlerFunc) (result *mcp.CallToolResult, err error) { + if existing := editOperationFrom(ctx); existing != nil && existing.owner == &s.editOperations { + return h(ctx, req) // a raw/facade nested handler borrows the owner + } + w := &editOperationWitness{owner: &s.editOperations} + ctx = context.WithValue(ctx, editOperationContextKey{}, w) + defer func() { w.finish(result, err) }() + return h(ctx, req) +} diff --git a/internal/mcp/edit_operation_observations_test.go b/internal/mcp/edit_operation_observations_test.go new file mode 100644 index 000000000..9b44fb7de --- /dev/null +++ b/internal/mcp/edit_operation_observations_test.go @@ -0,0 +1,178 @@ +package mcp + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func editOperationRequest(name, operation string) mcp.CallToolRequest { + var req mcp.CallToolRequest + req.Params.Name = name + if operation != "" { + req.Params.Arguments = map[string]any{"operation": operation} + } + return req +} + +func TestEditOperationMeasuredSurfaces(t *testing.T) { + for _, pair := range [][2]string{ + {"write_file", ""}, {"edit_file", ""}, {"edit_symbol", ""}, {"batch_edit", ""}, {"move_symbol", ""}, + {"edit", "write"}, {"edit", "file"}, {"edit", "symbol"}, {"edit", "batch"}, {"refactor", "move"}, {"edit", " FILE "}, {"edit", ""}, + } { + require.True(t, measuredSourceEdit(editOperationRequest(pair[0], pair[1])), "%v", pair) + } + for _, args := range []map[string]any{ + {"target": map[string]any{"file": "a.go"}, "content": "new"}, + {"target": map[string]any{"file": "a.go"}, "match": "old", "replacement": "new"}, + {"target": map[string]any{"symbol": "a.go::A"}}, + {"changes": []any{map[string]any{"op": "edit_file"}}}, + } { + req := editOperationRequest("edit", "") + req.Params.Arguments = args + require.True(t, measuredSourceEdit(req), "facade inference must retain operation ownership") + } + for _, pair := range [][2]string{{"read", "source"}, {"change", "receipt"}, {"mutation_status", ""}, {"edit", "docs"}, {"refactor", "rename"}} { + require.False(t, measuredSourceEdit(editOperationRequest(pair[0], pair[1])), "%v", pair) + } +} + +// These are witness/actual-execution contract controls. Real tool write seams +// are covered separately; synthetic marks here do not prove their integration. +func TestEditOperationActualResultAndSourceOutcome(t *testing.T) { + good := mcp.NewToolResultText("applied") + bad := mcp.NewToolResultError("failed after write") + for _, tt := range []struct { + name string + effect, complete, warning bool + rolledBack, unknown bool + result *mcp.CallToolResult + err error + completed, committedError uint64 + partial, rolled, unclassified uint64 + }{ + {name: "clean", effect: true, complete: true, result: good, completed: 1}, + {name: "nil result is not success", effect: true, complete: true, committedError: 1}, + {name: "tool error nil Go error", effect: true, complete: true, result: bad, committedError: 1}, + {name: "Go error", effect: true, complete: true, result: good, err: errors.New("format failed"), committedError: 1}, + {name: "committed journal warning", effect: true, complete: true, warning: true, result: good, committedError: 1}, + {name: "partial source effects", effect: true, result: bad, partial: 1}, + {name: "rolled back", effect: true, rolledBack: true, result: good, partial: 1, rolled: 1}, + {name: "prior bytes unknown", effect: true, complete: true, unknown: true, result: good, unclassified: 1}, + {name: "no writes", result: good}, + {name: "unchanged rewrite", complete: true, result: good}, + {name: "refused before disk", result: bad}, + } { + t.Run(tt.name, func(t *testing.T) { + s := &Server{ToolCallTimeout: -1} + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if tt.effect { + observeEditSourceEffect(ctx, true, !tt.unknown) + } else { + observeEditSourceEffect(ctx, false, true) + } + observeEditSourceTerminal(ctx, tt.complete, tt.warning, tt.rolledBack) + return tt.result, tt.err + }) + result, err := h(t.Context(), editOperationRequest("edit", "file")) + if tt.result == nil { + require.Nil(t, result) + } else { + require.Same(t, tt.result, result) + } + require.Equal(t, tt.err, err) + require.Equal(t, tt.completed, s.editOperations.completed.Load()) + require.Equal(t, tt.committedError, s.editOperations.committedWithError.Load()) + require.Equal(t, tt.partial, s.editOperations.failedAfterSourceEffect.Load()) + require.Equal(t, tt.rolled, s.editOperations.rolledBack.Load()) + require.Equal(t, tt.unclassified, s.editOperations.unknownSourceComparison.Load()) + }) + } +} + +func TestEditOperationNestedOwnerAndServerIsolation(t *testing.T) { + first, second := &Server{ToolCallTimeout: -1}, &Server{ToolCallTimeout: -1} + leaf := func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + // Multiple forward file writes are one logical handler completion. + observeEditSourceEffect(ctx, true, true) + observeEditSourceEffect(ctx, true, true) + observeEditSourceComplete(ctx) + return mcp.NewToolResultText("applied"), nil + } + inner := first.boundToolHandler(leaf) + other := second.boundToolHandler(leaf) + outer := first.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if _, err := other(ctx, req); err != nil { + return nil, err + } + return inner(ctx, req) + }) + _, err := outer(t.Context(), editOperationRequest("batch_edit", "")) + require.NoError(t, err) + require.EqualValues(t, 1, first.editOperations.completed.Load(), "nested handler borrows its owning Server witness") + require.EqualValues(t, 1, second.editOperations.completed.Load(), "another Server must not borrow or update the first's counter") +} + +func TestEditOperationLateHandlerCompletionIsNotCallerReply(t *testing.T) { + s := &Server{ToolCallTimeout: 20 * time.Millisecond} + entered, release, joined := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + stop := func() { once.Do(func() { close(release) }) } + t.Cleanup(func() { + stop() + select { + case <-joined: + case <-time.After(time.Second): + t.Error("owned actual handler did not join") + } + }) + h := s.boundToolHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + defer close(joined) + observeEditSourceEffect(ctx, true, true) + observeEditSourceComplete(ctx) + close(entered) + <-release // model the non-cancellable physical/journal completion + return mcp.NewToolResultText("applied"), nil + }) + replied := make(chan *mcp.CallToolResult, 1) + go func() { + result, _ := h(t.Context(), editOperationRequest("batch_edit", "")) + replied <- result + }() + select { + case <-entered: + case <-time.After(time.Second): + t.Fatal("actual handler never started") + } + select { + case result := <-replied: + require.True(t, result.IsError) + case <-time.After(time.Second): + t.Fatal("caller did not receive its deadline result") + } + require.Zero(t, s.editOperations.completed.Load(), "timeout reply is not completion of the actual handler") + stop() + require.Eventually(t, func() bool { return s.editOperations.completed.Load() == 1 }, time.Second, time.Millisecond) + require.Eventually(t, func() bool { return AbandonedToolCalls() == 0 }, time.Second, time.Millisecond) + require.Zero(t, s.editOperations.committedWithError.Load(), "the actual handler result, not its caller's timeout, defines this outcome") +} + +func TestEditOperationReadWitnessPathAddsNoAllocation(t *testing.T) { + s := &Server{} + result := mcp.NewToolResultText("read") + h := s.observeEditOperationHandler(func(ctx context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if editOperationFrom(ctx) != nil { + panic("read received an edit witness") + } + return result, nil + }) + req := editOperationRequest("read", "source") + ctx := t.Context() + allocs := testing.AllocsPerRun(100, func() { _, _ = h(ctx, req) }) + require.Zero(t, allocs, "only this narrow observer's read branch, not total middleware overhead") +} diff --git a/internal/mcp/edit_operation_tools_test.go b/internal/mcp/edit_operation_tools_test.go new file mode 100644 index 000000000..0cb4d2164 --- /dev/null +++ b/internal/mcp/edit_operation_tools_test.go @@ -0,0 +1,306 @@ +package mcp + +import ( + "context" + "encoding/json" + "errors" + "maps" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +// Exercise registered middleware/facade lowering through the same HandleMessage +// entry as daemon dispatch. This is protocol integration, not a real client run. +func callEditOperationFrame(t *testing.T, s *Server, ctx context.Context, name string, args map[string]any) *mcp.CallToolResult { + t.Helper() + s.EnsureToolPromotedForSession(ctx, name) + ctx = WithAuthorizedToolCall(ctx, name) + out := callToolFrame(t, s, ctx, name, args) + var reply struct { + Error any `json:"error"` + Result *mcp.CallToolResult `json:"result"` + } + require.NoError(t, json.Unmarshal([]byte(out), &reply)) + require.Nil(t, reply.Error, "%s", out) + require.NotNil(t, reply.Result, "%s", out) + return reply.Result +} + +func TestEditOperationFiveRealToolSeamsLegacyAndFacade(t *testing.T) { + for _, facade := range []bool{false, true} { + for _, operation := range []string{"write", "file", "symbol", "batch", "move"} { + t.Run(map[bool]string{false: "legacy", true: "facade"}[facade]+"/"+operation, func(t *testing.T) { + t.Setenv(batchTransactionDirEnv, filepath.Join(t.TempDir(), "transactions")) + s, root := setupMoveInlineRepo(t, map[string]string{ + "pkg/a.go": "package pkg\n\nfunc Foo() int { return 42 }\n", + "pkg/b.go": "package pkg\n\nfunc Bar() int { return 1 }\n", + "notes.txt": "before\n", "other.txt": "before\n", + }) + s.ToolCallTimeout = -1 + const session = "operation-counter" + preset := "full" + if facade { + preset = "facade-v1" + } + s.NoteSessionToolPolicy(session, preset, "hide") + ctx := WithSessionID(t.Context(), session) + legacy := map[string]string{"write": "write_file", "file": "edit_file", "symbol": "edit_symbol", "batch": "batch_edit", "move": "move_symbol"}[operation] + name := legacy + var args map[string]any + switch operation { + case "write": + args = map[string]any{"path": "notes.txt", "content": "after\n", "mutation_id": "count-write"} + case "file": + args = map[string]any{"path": "notes.txt", "old_string": "before", "new_string": "after", "mutation_id": "count-file"} + case "symbol": + args = map[string]any{"id": "pkg/a.go::Foo", "old_source": "return 42", "new_source": "return 43", "mutation_id": "count-symbol"} + case "batch": + args = map[string]any{"transaction_id": "count-batch", "edits": []any{ + map[string]any{"op": "edit_file", "path": filepath.Join(root, "notes.txt"), "old_string": "before", "new_string": "after"}, + map[string]any{"op": "edit_file", "path": filepath.Join(root, "other.txt"), "old_string": "before", "new_string": "after"}, + }} + case "move": + args = map[string]any{"id": "pkg/a.go::Foo", "target_file": "pkg/b.go"} + } + if facade { + name = "edit" + args = map[string]any{"operation": operation, "output": map[string]any{"format": "json", "max_bytes": 0}} + switch operation { + case "write": + args["target"], args["content"] = map[string]any{"file": "notes.txt"}, "after\n" + args["options"] = map[string]any{"mutation_id": "count-write"} + case "file": + args["target"], args["match"], args["replacement"] = map[string]any{"file": "notes.txt"}, "before", "after" + args["options"] = map[string]any{"mutation_id": "count-file"} + case "symbol": + args["target"], args["match"], args["replacement"] = map[string]any{"symbol": "pkg/a.go::Foo"}, "return 42", "return 43" + args["options"] = map[string]any{"mutation_id": "count-symbol"} + case "batch": + args["changes"] = []any{ + map[string]any{"op": "edit_file", "path": filepath.Join(root, "notes.txt"), "old_string": "before", "new_string": "after"}, + map[string]any{"op": "edit_file", "path": filepath.Join(root, "other.txt"), "old_string": "before", "new_string": "after"}, + } + args["options"] = map[string]any{"transaction_id": "count-batch"} + case "move": + name = "refactor" + args["target"], args["destination"] = map[string]any{"symbol": "pkg/a.go::Foo"}, "pkg/b.go" + } + } + preview := maps.Clone(args) + preview["dry_run"] = true + result := callEditOperationFrame(t, s, ctx, name, preview) + require.False(t, result.IsError, "%v", result.Content) + require.Zero(t, s.editOperations.completed.Load(), "preview is not a completed edit") + result = callEditOperationFrame(t, s, ctx, name, args) + require.False(t, result.IsError, "%v", result.Content) + if operation == "write" { + written := decodeFileOpsResult(t, result) + receipt, ok := written["mutation_receipt"].(string) + require.True(t, ok) + require.NotEmpty(t, receipt) + require.Equal(t, "count-write", written["mutation_id"], "facade options keep the caller mutation identity") + statusTool := "mutation_status" + statusArgs := map[string]any{"receipt": receipt, "format": "json"} + if facade { + statusTool = "change" + statusArgs = map[string]any{"operation": "receipt", "options": map[string]any{"receipt": receipt}, "output": map[string]any{"format": "json", "max_bytes": 0}} + } + statusResult := callEditOperationFrame(t, s, ctx, statusTool, statusArgs) + require.False(t, statusResult.IsError, "%v", statusResult.Content) + status := decodeFileOpsResult(t, statusResult) + require.Equal(t, receipt, status["receipt"], "status is bound to the write ACK, not its logical id") + require.NotEqual(t, "count-write", receipt) + } + require.EqualValues(t, 1, s.editOperations.completed.Load(), "one logical operation, not per-file commits") + require.Zero(t, s.editOperations.committedWithError.Load()) + require.Zero(t, s.editOperations.failedAfterSourceEffect.Load()) + if operation == "symbol" || operation == "move" { + a, err := os.ReadFile(filepath.Join(root, "pkg/a.go")) + require.NoError(t, err) + if operation == "symbol" { + require.Contains(t, string(a), "return 43") + } else { + require.NotContains(t, string(a), "func Foo") + b, err := os.ReadFile(filepath.Join(root, "pkg/b.go")) + require.NoError(t, err) + require.Contains(t, string(b), "func Foo") + } + } else { + require.Equal(t, "after\n", readAtomicBatchFixture(t, filepath.Join(root, "notes.txt"))) + if operation == "batch" { + require.Equal(t, "after\n", readAtomicBatchFixture(t, filepath.Join(root, "other.txt"))) + } + } + if operation != "move" { + // The real idempotency/transaction replay writes nothing. + result = callEditOperationFrame(t, s, ctx, name, args) + require.False(t, result.IsError, "%v", result.Content) + require.EqualValues(t, 1, s.editOperations.completed.Load()) + } + if operation == "batch" && !facade { + result = callEditOperationFrame(t, s, ctx, name, map[string]any{"transaction_id": "count-batch", "status_only": true}) + require.False(t, result.IsError, "%v", result.Content) + require.EqualValues(t, 1, s.editOperations.completed.Load(), "status is not another source operation") + } + }) + } + } +} + +func TestEditOperationWriteNoopAndRefusalDoNotCount(t *testing.T) { + s, root := setupTestServer(t) + s.ToolCallTimeout = -1 + ctx := WithSessionID(t.Context(), "operation-noop") + s.NoteSessionToolPolicy("operation-noop", "full", "hide") + path := filepath.Join(root, "unchanged.txt") + require.NoError(t, os.WriteFile(path, []byte("same\n"), 0o644)) + result := callEditOperationFrame(t, s, ctx, "write_file", map[string]any{"path": path, "content": "same\n"}) + require.False(t, result.IsError, "%v", result.Content) + result = callEditOperationFrame(t, s, ctx, "edit_file", map[string]any{"path": path, "old_string": "missing", "new_string": "after"}) + require.True(t, result.IsError) + require.Zero(t, s.editOperations.completed.Load()) + require.Zero(t, s.editOperations.failedAfterSourceEffect.Load()) + require.Equal(t, "same\n", readAtomicBatchFixture(t, path)) +} + +func TestEditOperationBatchRollbackAndCommittedJournalWarning(t *testing.T) { + for _, warning := range []bool{false, true} { + t.Run(map[bool]string{false: "rollback", true: "committed-warning"}[warning], func(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = -1 + dir := t.TempDir() + a := writeAtomicBatchFixture(t, dir, "a.txt", "before\n") + b := writeAtomicBatchFixture(t, dir, "b.txt", "before\n") + if warning { + s.batchDurabilityOverride = &batchDurabilityOps{writeFile: func(path string, data []byte, mode os.FileMode) error { + if filepath.Base(path) == "manifest.json" { + var receipt batchTransactionReceipt + if err := json.Unmarshal(data, &receipt); err != nil { + return err + } + if receipt.Status == "committed" { + return errors.New("test terminal journal unavailable") + } + } + return durableAtomicWriteFile(path, data, mode) + }} + } else { + s.batchWriteOverride = func(path string, data []byte, mode os.FileMode) error { + if path == b { + return errors.New("test second target refused") + } + return durableAtomicWriteFile(path, data, mode) + } + } + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "typed-outcome", "edits": []any{ + map[string]any{"op": "edit_file", "path": a, "old_string": "before", "new_string": "after"}, + map[string]any{"op": "edit_file", "path": b, "old_string": "before", "new_string": "after"}, + }} + result, err := s.boundToolHandler(s.handleAtomicBatchEdit)(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError, "typed receipt is carried by a non-tool-error result") + require.Zero(t, s.editOperations.completed.Load(), "errnil/non-IsError alone is insufficient") + if warning { + require.EqualValues(t, 1, s.editOperations.committedWithError.Load()) + require.Zero(t, s.editOperations.failedAfterSourceEffect.Load()) + require.Equal(t, "after\n", readAtomicBatchFixture(t, a)) + } else { + require.EqualValues(t, 1, s.editOperations.failedAfterSourceEffect.Load()) + require.EqualValues(t, 1, s.editOperations.rolledBack.Load()) + require.Zero(t, s.editOperations.committedWithError.Load()) + require.Equal(t, "before\n", readAtomicBatchFixture(t, a)) + } + }) + } +} + +func TestEditOperationRealBatchDetachedOriginalAndDuplicate(t *testing.T) { + s := newAtomicBatchTestServer(t, mutationTestWatcher{}) + s.ToolCallTimeout = time.Minute + path := writeAtomicBatchFixture(t, t.TempDir(), "target.txt", "before\n") + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + s.batchWriteOverride = func(target string, data []byte, mode os.FileMode) error { + close(entered) + <-release // actual first transaction already passed its cancellation gate + return durableAtomicWriteFile(target, data, mode) + } + originalJoined := make(chan struct{}) + t.Cleanup(func() { + unblock() + select { + case <-originalJoined: + case <-time.After(5 * time.Second): + t.Error("owned physical batch handler did not join before fixture cleanup") + } + require.Eventually(t, func() bool { return AbandonedToolCalls() == 0 }, 5*time.Second, time.Millisecond) + }) + req := editOperationRequest("batch_edit", "") + req.Params.Arguments = map[string]any{"transaction_id": "detached-original", "edits": []any{ + map[string]any{"op": "edit_file", "path": path, "old_string": "before", "new_string": "after"}, + }} + h := s.boundToolHandler(func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + defer close(originalJoined) + return s.handleAtomicBatchEdit(ctx, req) + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + type callerReply struct { + result *mcp.CallToolResult + err error + } + replied := make(chan callerReply, 1) + go func() { + result, err := h(ctx, req) + replied <- callerReply{result: result, err: err} + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("original did not reach actual physical commit boundary") + } + cancel() + select { + case reply := <-replied: + require.ErrorIs(t, reply.err, context.Canceled, "caller cancellation is a Go error, not an in-band tool result") + require.Nil(t, reply.result, "actual disk work remains held after the caller stops waiting") + case <-time.After(5 * time.Second): + t.Fatal("original caller did not receive its cancellation reply") + } + require.Zero(t, s.editOperations.completed.Load()) + stateValue, found := s.batchTransactions.Load("detached-original") + require.True(t, found) + state := stateValue.(*batchTransactionState) + timingID := state.acknowledgementReceipt + statusReq := editOperationRequest("mutation_status", "") + statusReq.Params.Arguments = map[string]any{"receipt": timingID, "format": "json"} + statusResult, statusErr := s.handleMutationStatus(t.Context(), statusReq) + beforeTiming := decodeAcknowledgementResult(t, statusResult, statusErr)["acknowledgement"].(map[string]any) + require.Equal(t, "measured", beforeTiming["caller_response_state"]) + require.Nil(t, beforeTiming["actual_handler_ms"]) + duplicateCtx, stop := context.WithCancel(t.Context()) + stop() + _, _ = s.boundToolHandler(s.handleAtomicBatchEdit)(duplicateCtx, req) // existing reply, no physical write + require.Zero(t, s.editOperations.completed.Load()) + unblock() + require.Eventually(t, func() bool { return s.editOperations.completed.Load() == 1 }, 5*time.Second, time.Millisecond) + require.Eventually(t, func() bool { + payload, _ := s.batchAcknowledgementReceiptPayload(t.Context(), timingID) + return payload["acknowledgement"].(acknowledgementPayload).ActualState == "measured" + }, 5*time.Second, time.Millisecond) + statusResult, statusErr = s.handleMutationStatus(t.Context(), statusReq) + afterTiming := decodeAcknowledgementResult(t, statusResult, statusErr)["acknowledgement"].(map[string]any) + require.Equal(t, beforeTiming["caller_response_ready_ms"], afterTiming["caller_response_ready_ms"]) + require.Equal(t, true, afterTiming["actual_finished_after_reply"]) + require.Eventually(t, func() bool { return AbandonedToolCalls() == 0 }, 5*time.Second, time.Millisecond) + require.Zero(t, s.editOperations.committedWithError.Load()) + require.Equal(t, "after\n", readAtomicBatchFixture(t, path)) +} diff --git a/internal/mcp/edit_probe_output_contract_test.go b/internal/mcp/edit_probe_output_contract_test.go new file mode 100644 index 000000000..1ffc7366d --- /dev/null +++ b/internal/mcp/edit_probe_output_contract_test.go @@ -0,0 +1,334 @@ +package mcp + +import ( + "encoding/json" + "regexp" + "strings" + "testing" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + toon "github.com/toon-format/toon-go" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" +) + +// Exercise the actual public edit and seven probe producers. TOON is the text +// contract, not permission to consume a structured JSON sibling or invent the +// metadata that its compact node rows deliberately omit. +func TestEditProbeActualJSONAndTOONWireContracts(t *testing.T) { + for _, format := range []string{"json", "toon"} { + t.Run(format, func(t *testing.T) { + original := "package main\n\nfunc Example() int { return 1 }\nfunc Caller() int { return Example() }\n" + s, _ := setupMoveInlineRepo(t, map[string]string{"main.go": original}) + t.Cleanup(s.DrainBackground) + s.ToolCallTimeout = -1 + logCore, captured := observer.New(zap.DebugLevel) + s.logger = zap.New(logCore) + const session = "edit-output-contract" + s.NoteSessionToolPolicy(session, "facade-v1", "hide") + // The in-memory fixture serves its base corpus; it has no checkout catalog. + ctx := WithSessionID(t.Context(), session) + output := map[string]any{"format": format, "max_bytes": 0} + ack := callEditOperationFrame(t, s, ctx, "edit", map[string]any{"operation": "file", "target": map[string]any{"file": "main.go"}, "match": "func Caller() int { return Example() }", "replacement": "func Caller() int { return Example() }\nfunc Added() int { return Example() }", "options": map[string]any{"mutation_id": "output-" + format}, "output": output}) + decode := func(result any) map[string]any { + raw, err := json.Marshal(result) + require.NoError(t, err) + var envelope struct { + IsError bool `json:"isError"` + Structured any `json:"structuredContent"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + require.NoError(t, json.Unmarshal(raw, &envelope)) + if envelope.IsError { + for _, entry := range captured.All() { + if entry.Message != "mcp: handler panic recovered" && entry.Message != "tool handler panic recovered" { + continue + } + t.Logf("actual handler panic stack: %v", entry.ContextMap()["stack"]) + } + } + require.False(t, envelope.IsError, "%s", raw) + require.NotEmpty(t, envelope.Content) + for _, auxiliary := range envelope.Content[1:] { + require.Equal(t, "text", auxiliary.Type) + require.Contains(t, []string{momentumNote(6), momentumEscalationNote(10, false), momentumEscalationNote(10, true)}, auxiliary.Text) + } + require.Equal(t, "text", envelope.Content[0].Type) + text := envelope.Content[0].Text + var body map[string]any + if format == "json" { + require.NoError(t, json.Unmarshal([]byte(text), &body)) + } else { + require.Nil(t, envelope.Structured, "TOON text must be the evaluated wire") + require.False(t, json.Valid([]byte(text)), "requested TOON silently fell back to JSON") + decoded, err := toon.Decode([]byte(text), toon.WithStrictMode(true)) + require.NoError(t, err) + var ok bool + body, ok = decoded.(map[string]any) + require.True(t, ok) + } + return body + } + require.NotEmpty(t, decode(ack)["mutation_receipt"], "source ACK must survive the selected output format") + calls := []struct { + name, operation string + target map[string]any + query string + }{ + {"read", "source", map[string]any{"symbol": "main.go::Added"}, ""}, + {"read", "summary", map[string]any{"file": "main.go"}, ""}, + {"read", "editing_context", map[string]any{"file": "main.go"}, ""}, + {"search", "symbols", nil, "Added"}, + {"search", "text", nil, "Added"}, + {"relations", "callers", map[string]any{"symbol": "main.go::Example"}, ""}, + {"trace", "call_chain", map[string]any{"symbol": "main.go::Added"}, ""}, + } + for _, call := range calls { + args := map[string]any{"operation": call.operation, "output": output, "options": map[string]any{"limit": 1000, "max_per_file": 0, "expand": "off", "depth": 1, "context_lines": 0}} + if call.target != nil { + args["target"] = call.target + } + if call.query != "" { + args["query"] = call.query + } + body := decode(callEditOperationFrame(t, s, ctx, call.name, args)) + switch call.operation { + case "source": + require.Equal(t, "main.go::Added", body["id"]) + require.Contains(t, body["source"], "return Example()") + case "summary", "callers", "call_chain": + require.Equal(t, false, body["truncated"]) + require.Contains(t, body, "nodes") + require.Contains(t, body, "edges") + if format == "toon" && call.operation != "summary" { + require.Contains(t, body, "total") + require.NotContains(t, body, "total_nodes", "TOON does not promise a JSON total_nodes field") + require.NotContains(t, body, "total_edges", "complete TOON edge rows omit this optional full-count field") + for _, raw := range body["nodes"].([]any) { + node := raw.(map[string]any) + require.NotContains(t, node, "meta", "compact rows do not expose signature metadata") + require.Contains(t, node, "id") + require.Contains(t, node, "file_path") + require.Contains(t, node, "start_line") + } + } else { + require.Contains(t, body, "total_nodes") + require.Contains(t, body, "total_edges") + if call.operation == "summary" { + for _, raw := range body["nodes"].([]any) { + node := raw.(map[string]any) + require.Contains(t, node, "end_line") + require.Contains(t, node, "meta") + } + } + } + case "editing_context": + require.NotEmpty(t, body["defines"]) + case "symbols": + require.NotEmpty(t, body["results"]) + require.Contains(t, body, "total") + case "text": + require.NotEmpty(t, body["matches"]) + } + } + }) + } +} + +// This pins a possible visible TOON witness before enabling the signature +// adapter: the unchanged following caller and target share the same graph +// response, so upstream line shifts cancel and no end_line/signature is made up. +func TestEditProbeTOONSignatureFollowingCallerWireWitness(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + file, header := "main.go", "package main\n\n" + example, before := "func Example() int { return 1 }", "func Target() int { return Example() }" + after, following := "func Target(optional ...int) int {\n return Example()\n}", "func Following() int { return Target() }" + if language == "typescript" { + file, header = "main.ts", "" + example, before = "export function Example(): number { return 1; }", "export function Target(): number { return Example(); }" + after, following = "export function Target(optional?: number): number {\n return Example();\n}", "export function Following(): number { return Target(); }" + } + if language == "python" { + file, header = "main.py", "" + example, before = "def Example():\n return 1", "def Target():\n return Example()" + after, following = "def Target(optional=0):\n\n return Example()", "def Following():\n return Target()" + } + source := header + example + "\n\n" + before + "\n\n" + following + "\n" + s, _ := setupMoveInlineRepo(t, map[string]string{file: source}) + t.Cleanup(s.DrainBackground) + s.ToolCallTimeout = -1 + logCore, captured := observer.New(zap.DebugLevel) + s.logger = zap.New(logCore) + const session = "toon-signature-witness" + s.NoteSessionToolPolicy(session, "facade-v1", "hide") + // The in-memory fixture serves its base corpus; it has no checkout catalog. + ctx := WithSessionID(t.Context(), session) + output := map[string]any{"format": "toon", "max_bytes": 0} + decode := func(result *mcpgo.CallToolResult) map[string]any { + if result.IsError { + for _, entry := range captured.All() { + if entry.Message != "mcp: handler panic recovered" && entry.Message != "tool handler panic recovered" { + continue + } + t.Logf("actual handler panic stack: %v", entry.ContextMap()["stack"]) + } + } + require.False(t, result.IsError) + text := toolText(result) + require.False(t, json.Valid([]byte(text)), "requested TOON fell back to JSON") + decoded, err := toon.Decode([]byte(text), toon.WithStrictMode(true)) + require.NoError(t, err) + body, ok := decoded.(map[string]any) + require.True(t, ok) + return body + } + summaryArgs := map[string]any{"operation": "summary", "target": map[string]any{"file": file}, "output": output} + gap := func(body map[string]any, compact bool) int { + positions := map[string]int{} + for _, raw := range body["nodes"].([]any) { + node := raw.(map[string]any) + if compact { + require.NotContains(t, node, "end_line") + require.NotContains(t, node, "meta") + } else { + require.Contains(t, node, "end_line") + require.Contains(t, node, "meta") + } + id, _ := node["id"].(string) + if id == file+"::Target" || id == file+"::Following" { + number, ok := node["start_line"].(float64) + require.True(t, ok) + require.NotContains(t, positions, id, "duplicate scoped fixture node") + positions[id] = int(number) + } + } + require.Len(t, positions, 2) + return positions[file+"::Following"] - positions[file+"::Target"] + } + require.Equal(t, strings.Count(before, "\n")+2, gap(decode(callEditOperationFrame(t, s, ctx, "read", summaryArgs)), false)) + ack := decode(callEditOperationFrame(t, s, ctx, "edit", map[string]any{"operation": "symbol", "target": map[string]any{"symbol": file + "::Target"}, "match": before, "replacement": after, "output": output})) + require.NotEmpty(t, ack["mutation_receipt"]) + for _, call := range []struct { + name, operation, target string + depth int + }{{"read", "summary", file, 1}, {"relations", "callers", file + "::Target", 1}, {"trace", "call_chain", file + "::Following", 2}} { + target := map[string]any{"symbol": call.target} + if call.operation == "summary" { + target = map[string]any{"file": call.target} + } + body := decode(callEditOperationFrame(t, s, ctx, call.name, map[string]any{"operation": call.operation, "target": target, "options": map[string]any{"depth": call.depth, "limit": 1000}, "output": output})) + require.Equal(t, false, body["truncated"]) + require.Equal(t, strings.Count(after, "\n")+2, gap(body, call.operation != "summary")) + if call.operation == "summary" { + continue + } + incoming, outgoing := false, false + for _, raw := range body["edges"].([]any) { + edge := raw.(map[string]any) + if edge["kind"] != "calls" { + continue + } + incoming = incoming || edge["from"] == file+"::Following" && edge["to"] == file+"::Target" + outgoing = outgoing || edge["from"] == file+"::Target" && edge["to"] == file+"::Example" + } + require.True(t, incoming, "unchanged actual caller relation missing") + if call.operation == "call_chain" { + require.True(t, outgoing, "actual target-to-callee relation missing") + } + } + }) + } +} + +// Pin the actual compact text an edit probe may request. Compact is one line +// per symbol with no node IDs, edge rows, signatures or freshness fields, so it +// can prove only presence, and only where the operation offers it. The other +// probe operations and the edit ACK ignore output.compact and answer JSON, +// which is pinned too: the harness keeps those requests JSON. +func TestEditProbeActualCompactTextContracts(t *testing.T) { + original := "package main\n\nfunc Example() int { return 1 }\nfunc Caller() int { return Example() }\n" + s, _ := setupMoveInlineRepo(t, map[string]string{"main.go": original}) + t.Cleanup(s.DrainBackground) + s.ToolCallTimeout = -1 + const session = "edit-compact-contract" + s.NoteSessionToolPolicy(session, "facade-v1", "hide") + // The in-memory fixture serves its base corpus; it has no checkout catalog. + ctx := WithSessionID(t.Context(), session) + plain := map[string]any{"format": "json", "max_bytes": 0} + compact := map[string]any{"format": "json", "compact": true, "max_bytes": 0} + edit := func(name string, output map[string]any) *mcpgo.CallToolResult { + return callEditOperationFrame(t, s, ctx, "edit", map[string]any{"operation": "file", "target": map[string]any{"file": "main.go"}, "match": "func Caller() int { return Example() }", "replacement": "func Caller() int { return Example() }\nfunc " + name + "() int { return Example() }", "options": map[string]any{"mutation_id": "compact-" + name}, "output": output}) + } + ack := edit("Added", plain) + require.False(t, ack.IsError, "%s", toolText(ack)) + first := func(result *mcpgo.CallToolResult) string { + require.NotEmpty(t, result.Content) + text, ok := result.Content[0].(mcpgo.TextContent) + require.True(t, ok) + return text.Text + } + row := regexp.MustCompile(`^([a-z_]+) (\S+) (\S+):([1-9][0-9]*)( \(in \S+\))?$`) + footer := regexp.MustCompile(`^edges: [0-9]+( of [0-9]+)? total(, [0-9]+ (EXTRACTED|INFERRED|AMBIGUOUS))*$`) + lines := func(result *mcpgo.CallToolResult) (rows []string, edges string) { + require.False(t, result.IsError, "%s", toolText(result)) + text := first(result) + require.False(t, json.Valid([]byte(text)), "compact silently returned JSON: %s", text) + require.True(t, strings.HasSuffix(text, "\n"), "every compact line ends with a newline: %q", text) + require.NotContains(t, text, "fresh", "compact text carries no freshness claim") + for _, line := range strings.Split(strings.TrimSuffix(text, "\n"), "\n") { + switch { + case row.MatchString(line): + rows = append(rows, line) + case footer.MatchString(line): + require.Empty(t, edges, "one edge footer: %s", text) + edges = line + default: + t.Fatalf("compact line outside the known grammar: %q in %q", line, text) + } + } + return rows, edges + } + // The four operations that offer compact, with the probes' own options. + marker, callee, caller := "function Added main.go:5", "function Example main.go:3", "function Caller main.go:4" + rows, edges := lines(callEditOperationFrame(t, s, ctx, "read", map[string]any{"operation": "summary", "target": map[string]any{"file": "main.go"}, "output": compact})) + require.Contains(t, rows, marker) + require.Contains(t, rows, callee) + require.NotEmpty(t, edges, "summary keeps its edge footer, never edge rows") + rows, edges = lines(callEditOperationFrame(t, s, ctx, "search", map[string]any{"operation": "symbols", "query": "Added", "output": compact})) + require.Contains(t, rows, marker) + require.Empty(t, edges) + rows, edges = lines(callEditOperationFrame(t, s, ctx, "relations", map[string]any{"operation": "callers", "target": map[string]any{"symbol": "main.go::Example"}, "options": map[string]any{"depth": 1, "limit": 1000}, "output": compact})) + require.Contains(t, rows, marker) + require.Contains(t, rows, caller) + require.NotEmpty(t, edges) + rows, edges = lines(callEditOperationFrame(t, s, ctx, "trace", map[string]any{"operation": "call_chain", "target": map[string]any{"symbol": "main.go::Added"}, "options": map[string]any{"depth": 1, "limit": 1000}, "output": compact})) + require.Contains(t, rows, callee) + require.NotEmpty(t, edges) + // Operations without a compact producer, and the edit ACK. + outcome := func(result *mcpgo.CallToolResult) string { + text := first(result) + switch { + case result.IsError: + return "refused: " + text + case json.Valid([]byte(text)): + return "json" + default: + return "text" + } + } + observed := map[string]string{ + "read.source": outcome(callEditOperationFrame(t, s, ctx, "read", map[string]any{"operation": "source", "target": map[string]any{"symbol": "main.go::Added"}, "output": compact})), + "read.editing_context": outcome(callEditOperationFrame(t, s, ctx, "read", map[string]any{"operation": "editing_context", "target": map[string]any{"file": "main.go"}, "output": compact})), + "read.file": outcome(callEditOperationFrame(t, s, ctx, "read", map[string]any{"operation": "file", "target": map[string]any{"file": "main.go"}, "output": compact})), + "search.text": outcome(callEditOperationFrame(t, s, ctx, "search", map[string]any{"operation": "text", "query": "Added", "output": compact})), + "edit.file": outcome(edit("AddedCompact", compact)), + } + for operation, got := range observed { + require.Equal(t, "json", got, "%s answered output.compact; all observed: %v", operation, observed) + } +} diff --git a/internal/mcp/edit_serialization.go b/internal/mcp/edit_serialization.go index 5891958e0..958884cfc 100644 --- a/internal/mcp/edit_serialization.go +++ b/internal/mcp/edit_serialization.go @@ -253,6 +253,16 @@ func (s *Server) trackScopedMutationTicket(ticket *indexer.MutationTicket, repo, checkoutID: checkoutID, checkoutIncarnation: incarnation, } + // Canonical admission already bound this record before waking the watcher. + // Keep only this existing consumer's local hold until Done reports; do not + // add completed timing history to the TTL receipt map. + publication := ticket.PublicationRecord + if checkoutID != "" { + publication = nil // checkout publication retains its existing lifecycle + } + if publication != nil { + indexer.DefaultPublicationPhases().Rekey(publication.Key(), receipt.id) + } s.mutationReceipts.Store(receipt.id, receipt) go func() { result, ok := <-ticket.Done @@ -266,6 +276,7 @@ func (s *Server) trackScopedMutationTicket(ticket *indexer.MutationTicket, repo, receipt.result = result receipt.completed = true receipt.mu.Unlock() + completeCanonicalPublication(publication, result) if result.Err == nil && result.Reindexed && !receipt.checkoutScoped { s.resolveSupersededFailedReceipts(receipt.path, receipt.generation, result) } @@ -502,17 +513,17 @@ func noDispatchSlotRelease(ctx context.Context) bool { return marked } -// releaseDispatchSlotForWait hands the request's daemon dispatch slot back at -// a passive post-commit wait. Outside a daemon dispatch it does nothing. -func (s *Server) releaseDispatchSlotForWait(ctx context.Context) { +// releaseDispatchSlotForWait returns mutation capacity and admits a passive +// post-commit wait to the slow lane. False leaves the committed receipt pending; +// it must never become a retryable busy result. Embedded calls are unchanged. +func (s *Server) releaseDispatchSlotForWait(ctx context.Context, published <-chan struct{}) bool { if noDispatchSlotRelease(ctx) { - return + return true } if s.releaseDispatchSlot != nil { - s.releaseDispatchSlot(ctx) - return + return s.releaseDispatchSlot(ctx, published) } - daemon.ReleaseMCPDispatchSlot(ctx) + return daemon.MoveMCPDispatchToSlowLane(ctx, published) == nil } func (s *Server) mutationWaitDuration() time.Duration { @@ -548,18 +559,24 @@ func (s *Server) mutationReindexState(ctx context.Context, absPath string) mutat // admitted work and let the receipt hold the pin until the ticket // reports. Nil whenever no generation stack answered this request. pin := handoffRequestView(ctx, viewmetrics.HandoffFileMutation) - ticket, scheduleErr := scheduler.EnqueueFileMutation(context.WithoutCancel(ctx), absPath) + publication := s.committedPublicationRecord(ctx, absPath) + admissionLap := s.beginAcknowledgementLap(ctx, editAckTicketAdmission) + ticket, scheduleErr := scheduler.EnqueueFileMutation(indexer.WithPublicationRecord(context.WithoutCancel(ctx), publication), absPath) + admissionLap.finish() if scheduleErr != nil { + failMutationPhases(publication) pin.release() return mutationReindexOutcome{Err: scheduleErr} } if ticket == nil { + failMutationPhases(publication) // Admission declined to schedule anything, so nothing outlives // this frame and the fall-through below reindexes inline. pin.release() } if ticket != nil { receipt := s.trackMutationTicket(ticket) + s.bindReadRepairAdmission(absPath, ticket.PublicationRecord, receipt) receipt.pinView(pin) // The disk write and ticket admission are complete. This wait // must not announce another ordinary writer while the watcher @@ -567,18 +584,22 @@ func (s *Server) mutationReindexState(ctx context.Context, absPath string) mutat resumeIntent := suspendSourceMutationWriteIntent(ctx) defer resumeIntent() // Only waiting from here on: let other requests use this - // dispatch slot. - s.releaseDispatchSlotForWait(ctx) - timer := time.NewTimer(s.mutationWaitDuration()) - defer timer.Stop() + // dispatch slot. A publication that lands first needs none. + waitCtx, cancel := context.WithTimeout(ctx, s.mutationWaitDuration()) + defer cancel() + waitLap := s.beginAcknowledgementLap(ctx, editAckReindexWait) + if !s.releaseDispatchSlotForWait(waitCtx, receipt.done) { + waitLap.finish() + return receipt.outcome(true) + } select { case <-receipt.done: + waitLap.finish() outcome := receipt.outcome(false) outcome.Receipt = "" return outcome - case <-timer.C: - return receipt.outcome(true) - case <-ctx.Done(): + case <-waitCtx.Done(): + waitLap.finish() return receipt.outcome(true) } } diff --git a/internal/mcp/edit_symbol_measurement_contract_test.go b/internal/mcp/edit_symbol_measurement_contract_test.go new file mode 100644 index 000000000..0eac772f8 --- /dev/null +++ b/internal/mcp/edit_symbol_measurement_contract_test.go @@ -0,0 +1,284 @@ +package mcp + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +// The benchmark adapters use existing arbitrary text splicing within an actual +// indexed declaration. Empty replacement and sibling declarations are product +// semantics, not invented operation enums or fake benchmark success. +func TestEditSymbolMeasurementFixtureUsesActualSpliceAndExplicitAbsence(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + for _, operation := range []string{"sibling", "rename", "delete", "append_leaf", "modify_body", "change_signature"} { + t.Run(language+"/"+operation, func(t *testing.T) { + s, root := setupTestServer(t) + t.Cleanup(s.DrainBackground) + var file, header, declaration, survivor, added, renamed, leaf, modified, signatureChanged, caller string + switch language { + case "go": + file = "symbol-cell.go" + header = "package main\n" + declaration = "func Target() int { return 1 }" + survivor = "func Sentinel() int { return 2 }" + added = "func Added() int { return Target() }" + renamed = "func Renamed() int { return 1 }" + leaf = "func Added() int { return 7 }" + modified = "func Target() int {\n value := 7\n return value\n}" + signatureChanged = "func Target(optional ...int) int { return 1 }" + caller = "func Caller() int { return Target() }" + case "typescript": + file = "symbol-cell.ts" + declaration = "export function Target(): number { return 1; }" + survivor = "export function Sentinel(): number { return 2; }" + added = "export function Added(): number { return Target(); }" + renamed = "export function Renamed(): number { return 1; }" + leaf = "export function Added(): number { return 7; }" + modified = "export function Target(): number {\n const value = 7;\n return value;\n}" + signatureChanged = "export function Target(optional?: number): number { return 1; }" + caller = "export function Caller(): number { return Target(); }" + case "python": + file = "symbol-cell.py" + declaration = "def Target():\n return 1" + survivor = "def Sentinel():\n return 2" + added = "def Added():\n return Target()" + renamed = "def Renamed():\n return 1" + leaf = "def Added():\n return 7" + modified = "def Target():\n value = 7\n return value" + signatureChanged = "def Target(optional=0):\n return 1" + caller = "def Caller():\n return Target()" + } + source := header + declaration + "\n\n" + survivor + "\n" + if operation == "modify_body" || operation == "change_signature" { + source += "\n" + caller + "\n" + } + path := filepath.Join(root, file) + require.NoError(t, os.WriteFile(path, []byte(source), 0600)) + require.NoError(t, s.indexer.IndexFile(path)) + id := file + "::Target" + beforeNode := s.engineFor(t.Context()).GetSymbol(id) + require.NotNil(t, beforeNode, "fixture must start with actual producer-owned symbol") + beforeSpan := beforeNode.EndLine - beforeNode.StartLine + replacement := declaration + "\n" + added + if operation == "rename" { + replacement = renamed + } + if operation == "delete" { + replacement = "" + } + if operation == "append_leaf" { + replacement = declaration + "\n" + leaf + } + if operation == "modify_body" { + replacement = modified + } + if operation == "change_signature" { + replacement = signatureChanged + } + s.facades.capture(mcpgo.NewTool("edit_symbol"), s.handleEditSymbol) + req := editOperationRequest("edit", "symbol") + req.Params.Arguments = map[string]any{"operation": "symbol", "target": map[string]any{"symbol": id}, "match": declaration, "replacement": replacement, "options": map[string]any{"mutation_id": "symbol-cell-" + language + "-" + operation}, "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "edit", r) + })(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError, "%v", result.Content) + response := unmarshalResult(t, result) + require.Equal(t, "applied", response["status"]) + require.NotEmpty(t, response["mutation_receipt"]) + record, found := s.mutationCommits.byReceipt(response["mutation_receipt"].(string)) + require.True(t, found) + status := s.mutationStatusPayload(record) + require.Equal(t, "edit_symbol", status["tool"]) + ack := status["acknowledgement"].(acknowledgementPayload) + require.Equal(t, "not_applicable_text_splice", ack.ParserScope) + require.Equal(t, "not_applicable", ack.PhaseStates["parse"]) + parse, present := ack.ObservedExecutionMS["parse"] + require.True(t, present) + require.Nil(t, parse) + actual, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, strings.Replace(source, declaration, replacement, 1), string(actual)) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(file+"::Sentinel"), "neighbor must survive every splice") + switch operation { + case "sibling", "append_leaf": + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(id)) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(file+"::Added")) + case "modify_body", "change_signature": + afterNode := s.engineFor(t.Context()).GetSymbol(id) + require.NotNil(t, afterNode) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(file+"::Caller"), "unchanged zero-argument caller must survive") + sourceResult := callTool(t, s, "get_symbol_source", map[string]any{"id": id, "context_lines": 0}) + require.False(t, sourceResult.IsError) + sourceBody := unmarshalResult(t, sourceResult) + require.Equal(t, strings.TrimSpace(replacement), strings.TrimSpace(sourceBody["source"].(string))) + if operation == "modify_body" { + require.Greater(t, afterNode.EndLine-afterNode.StartLine, beforeSpan) + } + if operation == "change_signature" { + switch language { + case "go": + require.Contains(t, afterNode.Meta["signature"], "optional") + case "typescript": + require.Equal(t, "function Target()", afterNode.Meta["signature"], "producer parameter detail is proved by exact source, not invented metadata") + case "python": + require.Equal(t, "def Target(...)", afterNode.Meta["signature"], "producer parameter detail is proved by exact source, not invented metadata") + } + } + default: + require.Nil(t, s.engineFor(t.Context()).GetSymbol(id), "old identity must really be absent, not an empty successful source") + missing := callTool(t, s, "get_symbol_source", map[string]any{"id": id}) + require.True(t, missing.IsError, "expected absence is an explicit producer refusal") + require.Contains(t, mustErrText(t, missing), id) + if operation == "rename" { + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(file+"::Renamed")) + } + if operation == "delete" { + chain := callTool(t, s, "get_call_chain", map[string]any{"id": id, "depth": 1, "limit": 1000}) + require.False(t, chain.IsError) + chainBody := unmarshalResult(t, chain) + require.Contains(t, chainBody, "nodes") + require.Contains(t, chainBody, "edges") + require.Empty(t, chainBody["nodes"]) + require.Empty(t, chainBody["edges"]) + require.Zero(t, chainBody["total_nodes"]) + require.Zero(t, chainBody["total_edges"]) + require.Equal(t, false, chainBody["truncated"]) + search := callTool(t, s, "search_symbols", map[string]any{"query": "GortexDeletedMeasurementNoSuchSymbol", "query_class": "symbol", "expand": "off"}) + require.False(t, search.IsError) + searchBody := unmarshalResult(t, search) + require.Contains(t, searchBody, "results") + results, _ := searchBody["results"].([]any) + require.EqualValues(t, len(results), searchBody["total"]) + for _, raw := range results { + require.NotEqual(t, id, raw.(map[string]any)["id"], "deleted identity must remain absent even in complete fallback output") + } + require.Equal(t, "symbol", searchBody["query_class"]) + require.Equal(t, false, searchBody["truncated"]) + } + + } + }) + } + } +} + +func TestDeleteMeasurementSearchCoversHundredDeclarationFixture(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + s, root := setupTestServer(t) + t.Cleanup(s.DrainBackground) + file, source := "fixture.go", "package main\n" + declaration := func(name string) string { return "func " + name + "() int { return 1 }" } + if language == "typescript" { + file = "fixture.ts" + source = "" + declaration = func(name string) string { return "export function " + name + "(): number { return 1; }" } + } + if language == "python" { + file = "fixture.py" + source = "" + declaration = func(name string) string { return "def " + name + "():\n return 1" } + } + for i := 0; i < 100; i++ { + source += declaration(fmt.Sprintf("GortexEditProbe000000000000002aN%03dBefore", i)) + "\n\n" + } + path := filepath.Join(root, file) + require.NoError(t, os.WriteFile(path, []byte(source), 0600)) + require.NoError(t, s.indexer.IndexFile(path)) + name := "GortexEditProbe000000000000002aN099Before" + id := file + "::" + name + // Like the edit-probe harness, request complete output: absolute fixture + // paths can push a hundred declarations past the default byte budget. + args := map[string]any{"query": name, "query_class": "symbol", "expand": "off", "limit": 1000, "max_per_file": 0, "max_bytes": 0} + before := callTool(t, s, "search_symbols", args) + require.False(t, before.IsError) + beforeBody := unmarshalResult(t, before) + found := false + for _, raw := range beforeBody["results"].([]any) { + if raw.(map[string]any)["id"] == id { + found = true + } + } + require.True(t, found, "fixture must prove actual old ID was searchable before deletion") + s.facades.capture(mcpgo.NewTool("edit_symbol"), s.handleEditSymbol) + req := editOperationRequest("edit", "symbol") + req.Params.Arguments = map[string]any{"operation": "symbol", "target": map[string]any{"symbol": id}, "match": declaration(name), "replacement": "", "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "edit", r) + })(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + after := callTool(t, s, "search_symbols", args) + require.False(t, after.IsError) + body := unmarshalResult(t, after) + require.Contains(t, body, "results") + require.NotContains(t, body, "_truncated_by_budget", "complete deletion evidence must not be a budget-trimmed page") + results, _ := body["results"].([]any) + require.EqualValues(t, len(results), body["total"]) + require.Equal(t, false, body["truncated"]) + require.Empty(t, body["next_cursor"]) + for _, raw := range results { + require.NotEqual(t, id, raw.(map[string]any)["id"], "complete fallback population retained deleted exact ID") + } + }) + } +} + +func TestEditSymbolCrossFileMeasurementUsesActualResolvedCallee(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + s, root := setupTestServer(t) + t.Cleanup(s.DrainBackground) + file, hub, header, declaration, added, hubSource := "caller.go", "hub.go", "package main\n", "func CrossAnchor() int { return 1 }", "func CrossMarker() int { return Hub() }", "package main\nfunc Hub() int { return 2 }\n" + if language == "typescript" { + file = "caller.ts" + hub = "hub.ts" + header = "import { Hub } from \"./hub\";\n" + declaration = "export function CrossAnchor(): number { return 1; }" + added = "export function CrossMarker(): number { return Hub(); }" + hubSource = "export function Hub(): number { return 2; }\n" + } + if language == "python" { + file = "caller.py" + hub = "hub.py" + header = "from hub import Hub\n" + declaration = "def CrossAnchor():\n return 1" + added = "def CrossMarker():\n return Hub()" + hubSource = "def Hub():\n return 2\n" + } + require.NoError(t, os.WriteFile(filepath.Join(root, hub), []byte(hubSource), 0600)) + require.NoError(t, s.indexer.IndexFile(filepath.Join(root, hub))) + require.NoError(t, os.WriteFile(filepath.Join(root, file), []byte(header+declaration+"\n"), 0600)) + require.NoError(t, s.indexer.IndexFile(filepath.Join(root, file))) + s.indexer.ResolveAll() + s.facades.capture(mcpgo.NewTool("edit_symbol"), s.handleEditSymbol) + req := editOperationRequest("edit", "symbol") + req.Params.Arguments = map[string]any{"operation": "symbol", "target": map[string]any{"symbol": file + "::CrossAnchor"}, "match": declaration, "replacement": declaration + "\n" + added, "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "edit", r) + })(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + chain := callTool(t, s, "get_call_chain", map[string]any{"id": file + "::CrossMarker", "depth": 1, "limit": 1000}) + require.False(t, chain.IsError) + body := unmarshalResult(t, chain) + edges, _ := body["edges"].([]any) + found := false + for _, raw := range edges { + edge := raw.(map[string]any) + if edge["from"] == file+"::CrossMarker" && edge["to"] == hub+"::Hub" && edge["kind"] == "calls" { + found = true + } + } + require.True(t, found, "actual public edit must publish exact crossfile Hub edge") + }) + } +} diff --git a/internal/mcp/ensure_fresh_self_heal_test.go b/internal/mcp/ensure_fresh_self_heal_test.go index 193cf393a..8df3c63f5 100644 --- a/internal/mcp/ensure_fresh_self_heal_test.go +++ b/internal/mcp/ensure_fresh_self_heal_test.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "testing" "github.com/stretchr/testify/assert" @@ -44,8 +45,12 @@ func TestEnsureFresh_MultiRepoSelfHealsStaleFile(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // These fixtures test path ownership/freshness, not cross-repo batch + // resolution. Register their explicit roots through the production path. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) diff --git a/internal/mcp/facade_shorthand_scope_test.go b/internal/mcp/facade_shorthand_scope_test.go index 251c74a86..2fdf4ca42 100644 --- a/internal/mcp/facade_shorthand_scope_test.go +++ b/internal/mcp/facade_shorthand_scope_test.go @@ -93,8 +93,7 @@ func newShorthandScopeFixture(t *testing.T) shorthandScopeFixture { g := graph.New() mi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) for i := range 40 { repo := fmt.Sprintf("far-%02d", i) for _, name := range []string{"Needle", "Twin", "Shared"} { diff --git a/internal/mcp/facade_tools.go b/internal/mcp/facade_tools.go index 5cf4d24a0..474cefcf4 100644 --- a/internal/mcp/facade_tools.go +++ b/internal/mcp/facade_tools.go @@ -520,10 +520,9 @@ func parseLocalizationNewUserBoundary(facade, operation string, arguments map[st return boundary, nil } -func (s *Server) handleFacade(ctx context.Context, facade string, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { - started := time.Now() - input, _ := req.Params.Arguments.(map[string]any) - localizationAuthToken := localizationauth.TakeArgument(input) +// facadeRequestOperation is the side-effect-free operation selection shared +// by admission classification and the actual facade dispatcher. +func facadeRequestOperation(facade string, req mcpgo.CallToolRequest) string { operation := resolveFacadeOperationAlias(facade, normalizeFacadeOperation(req.GetString("operation", ""))) if facade == "analyze" { operation = requestedAnalyzeKind(req.GetArguments()) @@ -540,6 +539,14 @@ func (s *Server) handleFacade(ctx context.Context, facade string, req mcpgo.Call if facade == "read" { operation = normalizeFacadeReadOperation(operation, req.GetArguments()) } + return operation +} + +func (s *Server) handleFacade(ctx context.Context, facade string, req mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + started := time.Now() + input, _ := req.Params.Arguments.(map[string]any) + localizationAuthToken := localizationauth.TakeArgument(input) + operation := facadeRequestOperation(facade, req) terminal := s.localizationFor(ctx) freshLocalizeFlow := facade == "explore" && operation == "localize" newUserTask, invalidBoundary := parseLocalizationNewUserBoundary(facade, operation, req.GetArguments()) diff --git a/internal/mcp/freshness_rider_extractor_lang_test.go b/internal/mcp/freshness_rider_extractor_lang_test.go index d5bd0bb66..ab7c8c735 100644 --- a/internal/mcp/freshness_rider_extractor_lang_test.go +++ b/internal/mcp/freshness_rider_extractor_lang_test.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "encoding/json" "os" "path/filepath" @@ -64,8 +65,10 @@ func TestFreshnessRider_ExtractorStalenessIsScopedToTheFilesLanguage(t *testing. g := graph.New() store := &riderIndexStateStore{Store: g} mi := indexer.NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } // Use the actual policy snapshot persisted by this index, so unrelated // extractor or global policy bumps cannot accidentally stale the Go file. diff --git a/internal/mcp/freshness_rider_test.go b/internal/mcp/freshness_rider_test.go index dca6a327f..213148f8c 100644 --- a/internal/mcp/freshness_rider_test.go +++ b/internal/mcp/freshness_rider_test.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "encoding/json" "os" "path/filepath" @@ -106,8 +107,12 @@ func TestMultiRepoRiderAndMissingFileFlag(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // These fixtures test path ownership/freshness, not cross-repo batch + // resolution. Register their explicit roots through the production path. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/graph_stats_per_repo_test.go b/internal/mcp/graph_stats_per_repo_test.go index 6401705ef..8690161bd 100644 --- a/internal/mcp/graph_stats_per_repo_test.go +++ b/internal/mcp/graph_stats_per_repo_test.go @@ -41,8 +41,10 @@ func setupMultiRepoStatsServer(t *testing.T) *Server { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) diff --git a/internal/mcp/index_health_empty_repo_test.go b/internal/mcp/index_health_empty_repo_test.go index 323b5355b..090ace518 100644 --- a/internal/mcp/index_health_empty_repo_test.go +++ b/internal/mcp/index_health_empty_repo_test.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "os" "path/filepath" "testing" @@ -43,8 +44,10 @@ func healthServerWithIndexedRepos(t *testing.T, named map[string]string) *Server reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) mi := indexer.NewMultiIndexer(graph.New(), reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv.configManager = cm srv.multiIndexer = mi diff --git a/internal/mcp/instructions_test.go b/internal/mcp/instructions_test.go index 825054787..3447eea77 100644 --- a/internal/mcp/instructions_test.go +++ b/internal/mcp/instructions_test.go @@ -179,8 +179,10 @@ func TestStateAwareInstructionsVariants(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/move_file_observation.go b/internal/mcp/move_file_observation.go new file mode 100644 index 000000000..675241a1c --- /dev/null +++ b/internal/mcp/move_file_observation.go @@ -0,0 +1,450 @@ +package mcp + +import ( + "context" + "strconv" + "strings" + "time" + "weak" + + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/indexer" +) + +// These records describe the existing move's two real writes and direct +// reconciles. They are neither a durable transaction nor physical mutation +// receipts, and never participate in replay/pending-source/freshness indexes. +// The first measured fixture is a same-package, two-file Go relocation. +type moveFileObservation struct { + operationID, sourceID, targetID, role string + moveSession, moveRequestKey string + writeVerdict, directState string + directClaimed bool + readClaimed bool + readState string + readAssisted *moveDirectObservation + direct *moveDirectObservation +} +type moveDirectObservation struct { + generation uint64 + windowNS int64 + writerToAcceptanceNS int64 + acceptedBlob string + causes [8]mutationReadRepairCause + causeCount int + accounting indexer.ReadRepairPhaseAccounting + requestCause mutationReadRepairCause + startedFromAckNS, finishedFromAckNS, acceptedFromStartNS int64 +} +type moveFileObservationReference struct { + Path string `json:"path"` + Role string `json:"role"` + Receipt string `json:"receipt"` +} +type moveObservation struct { + server *Server + id string + records map[string]*mutationCommitRecord + refs []moveFileObservationReference +} + +func (s *Server) beginMoveObservation(ctx context.Context, sourceID, targetID, sourceAbs, targetAbs, sourceRel, targetRel string, twoFileSamePackage bool) *moveObservation { + ack := editAcknowledgementFrom(ctx, s) + if ack == nil || !twoFileSamePackage { + return nil + } + checkout, incarnation := mutationCheckoutScope(ctx) + moveRequestKey, _ := daemon.MCPRequestIDKeyForContext(ctx) + moveSession := SessionIDFromContext(ctx) + out := &moveObservation{server: s, id: "move-" + strconv.FormatUint(ack.id, 10), records: make(map[string]*mutationCommitRecord, 2)} + for _, file := range []struct{ abs, rel, role string }{{sourceAbs, sourceRel, "source"}, {targetAbs, targetRel, "target"}} { + record := &mutationCommitRecord{kind: moveFileObservationCommit, owner: s, id: "move-file-" + strconv.FormatUint(mutationCommitSequence.Add(1), 10), tool: "move_symbol", key: out.id, absPath: file.abs, relPath: file.rel, startedAt: ack.entered, acknowledgement: ack, disk: mutationDiskInFlight, checkoutID: checkout, checkoutIncarnation: incarnation, moveFile: &moveFileObservation{operationID: out.id, sourceID: sourceID, targetID: targetID, role: file.role, moveSession: moveSession, moveRequestKey: moveRequestKey, writeVerdict: "pending", directState: "not_started", readState: "not_started"}} + s.mutationCommits.put(record) + out.records[file.abs] = record + out.refs = append(out.refs, moveFileObservationReference{Path: file.rel, Role: file.role, Receipt: record.id}) + } + return out +} + +func (o *moveObservation) write(ack *editAcknowledgement, path string, content []byte) error { + if o == nil { + return writeMoveFileObserved(ack, path, content) + } + started := time.Now() + err := writeMoveFileObserved(ack, path, content) + returned := time.Now() + record := o.records[path] + if record == nil { + return err + } + blob := "" + if err == nil { + blob = gitBlobSHA(content) + } // no existing fingerprint is produced by the move writer + ledger := &o.server.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[record.id] != record || record.owner != o.server || record.kind != moveFileObservationCommit || record.moveFile == nil || record.key != o.id { + return err + } + if !record.diskWriteStartedAt.IsZero() { + record.disk = mutationDiskFailed + record.errText = "unavailable_repeated_writer" + return err + } + record.diskWriteStartedAt, record.committedAt = started, returned + if err != nil { + record.disk = mutationDiskFailed + record.errText = err.Error() + return err + } + record.disk, record.newSHA, record.bytesWritten = mutationDiskCommitted, blob, len(content) + record.commitSeq = mutationDiskCommitSequence.Add(1) + return err // observation never changes the real writer outcome +} + +func (o *moveObservation) finishWrites(success bool) { + if o == nil { + return + } + ledger := &o.server.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + for _, record := range o.records { + record.mu.Lock() + if ledger.byID[record.id] == record && record.moveFile != nil && record.key == o.id && record.moveFile.writeVerdict == "pending" { + record.moveFile.writeVerdict = "partial_or_failed" + if success { + record.moveFile.writeVerdict = "all_writes_completed" + } + } + record.mu.Unlock() + } +} + +func (o *moveObservation) references() []moveFileObservationReference { + if o == nil { + return nil + } + out := make([]moveFileObservationReference, len(o.refs)) + copy(out, o.refs) + return out +} + +type moveDirectWitness struct { + server weak.Pointer[Server] + record weak.Pointer[mutationCommitRecord] + indexer weak.Pointer[indexer.Indexer] + id, operation, path, blob, session, requestKey string + committed time.Time + ackStarted time.Time + commitSeq uint64 + readAssisted bool +} + +func (o *moveObservation) observer(ctx context.Context, path string, idx *indexer.Indexer) *indexer.ReadRepairObserver { + if o == nil || idx == nil { + return nil + } + record := o.records[path] + if record == nil { + return nil + } + key, _ := daemon.MCPRequestIDKeyForContext(ctx) + witness := moveDirectWitness{server: weak.Make(o.server), record: weak.Make(record), indexer: weak.Make(idx), session: SessionIDFromContext(ctx), requestKey: key, ackStarted: record.startedAt} + ledger := &o.server.mutationCommits + ledger.mu.Lock() + record.mu.Lock() + valid := ledger.byID[record.id] == record && record.kind == moveFileObservationCommit && record.owner == o.server && record.moveFile != nil && record.moveFile.writeVerdict == "all_writes_completed" && !record.moveFile.directClaimed && record.disk == mutationDiskCommitted && record.newSHA != "" && !record.committedAt.IsZero() && record.commitSeq != 0 + if valid { + witness.id, witness.operation, witness.path, witness.blob, witness.committed, witness.commitSeq = record.id, record.key, record.absPath, record.newSHA, record.committedAt, record.commitSeq + record.moveFile.directClaimed = true + record.moveFile.directState = "pending_observer" + } + record.mu.Unlock() + ledger.mu.Unlock() + if !valid { + return nil + } + return &indexer.ReadRepairObserver{Session: witness.session, RequestIDKey: witness.requestKey, Complete: witness.complete} +} + +// The newest same-path publication owner is authoritative. Move observations +// remain excluded from physical/batch ticket binding; this is a separate actual +// reader cause attached to the same retained move file and returned write. +func (s *Server) moveReadRepairObserver(ctx context.Context, path string, idx *indexer.Indexer) (*indexer.ReadRepairObserver, bool) { + if idx == nil { + return nil, false + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + for i := len(ledger.ordered) - 1; i >= 0; i-- { + record := ledger.ordered[i] + if !record.isPublicationOwner() && record.kind != moveFileObservationCommit || record.absPath != path { + continue + } + if record.kind != moveFileObservationCommit { + return nil, false + } + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[record.id] != record || record.owner != s || record.moveFile == nil || record.moveFile.readClaimed || record.disk != mutationDiskCommitted || record.newSHA == "" || record.committedAt.IsZero() || record.commitSeq == 0 { + return nil, true + } + // Capture can precede the overall write verdict. Completion still requires + // that exact record's final successful verdict; no later retry can rebind it. + if record.moveFile.writeVerdict != "pending" && record.moveFile.writeVerdict != "all_writes_completed" { + return nil, true + } + key, _ := daemon.MCPRequestIDKeyForContext(ctx) + witness := moveDirectWitness{server: weak.Make(s), record: weak.Make(record), indexer: weak.Make(idx), id: record.id, operation: record.key, path: record.absPath, blob: record.newSHA, committed: record.committedAt, commitSeq: record.commitSeq, session: SessionIDFromContext(ctx), requestKey: key, readAssisted: true, ackStarted: record.startedAt} + record.moveFile.readClaimed = true + record.moveFile.readState = "pending_observer" + return &indexer.ReadRepairObserver{Session: witness.session, RequestIDKey: witness.requestKey, Complete: witness.complete}, true + } + return nil, false +} + +// Both routes retain their own immutable result and actual request cause. +// The slot is accessed only while the owning record mutex is held. +func (w moveDirectWitness) slot(file *moveFileObservation) (**moveDirectObservation, *string) { + if w.readAssisted { + return &file.readAssisted, &file.readState + } + return &file.direct, &file.directState +} + +func (w moveDirectWitness) complete(observed indexer.ReadRepairObservation) { + s, record, idx := w.server.Value(), w.record.Value(), w.indexer.Value() + if s == nil || record == nil { + return + } + if idx == nil { + w.unavailable("unavailable_observer_owner") + return + } + start, end := observed.Bounds() + if !observed.AcceptedFor(idx, w.blob) || observed.AcceptedAt().Before(w.committed) { + w.unavailable("unavailable_owner_bytes_or_acceptance") + return + } + accepted := moveDirectObservation{generation: observed.Generation(), windowNS: end.Sub(start).Nanoseconds(), acceptedBlob: w.blob, writerToAcceptanceNS: observed.AcceptedAt().Sub(w.committed).Nanoseconds(), accounting: observed.PhaseAccounting(), startedFromAckNS: start.Sub(w.ackStarted).Nanoseconds(), finishedFromAckNS: end.Sub(w.ackStarted).Nanoseconds(), acceptedFromStartNS: observed.AcceptedAt().Sub(start).Nanoseconds()} + ownedCause := false + for i := range accepted.causes { + session, key, generation, ok := observed.Cause(i) + if !ok { + break + } + accepted.causes[i] = mutationReadRepairCause{Session: session, RequestIDKey: key, AdmissionGeneration: generation} + accepted.causeCount++ + if session == w.session && key == w.requestKey && generation != 0 { + if ownedCause && accepted.requestCause.AdmissionGeneration != generation { + w.unavailable("unavailable_ambiguous_original_cause") + return + } + ownedCause = true + accepted.requestCause = mutationReadRepairCause{Session: session, RequestIDKey: key, AdmissionGeneration: generation} + } + } + if _, _, _, more := observed.Cause(len(accepted.causes)); more || !ownedCause { + w.unavailable("unavailable_original_reconcile_cause") + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + if ledger.byID[w.id] != record { + return + } + // A later real writer, including another observed move, cannot donate its + // accepted publication to this earlier file record. + // Keep competing rows read-locked through binding. An older admitted + // physical writer cannot start/commit between this check and acceptance. + locked := make([]*mutationCommitRecord, 0) + defer func() { + for i := len(locked) - 1; i >= 0; i-- { + locked[i].mu.RUnlock() + } + }() + firstPath := true + refused := "" + for i := len(ledger.ordered) - 1; i >= 0; i-- { + candidate := ledger.ordered[i] + if !candidate.isPublicationOwner() && candidate.kind != moveFileObservationCommit || candidate.absPath != w.path { + continue + } + if firstPath { + firstPath = false + if candidate != record { + refused = "unavailable_superseded_owner" + } + } + if candidate == record { + continue + } + candidate.mu.RLock() + locked = append(locked, candidate) + if candidate.disk == mutationDiskCommitted && candidate.commitSeq > w.commitSeq { + refused = "unavailable_competing_commit" + } + if candidate.disk == mutationDiskInFlight && !candidate.diskWriteStartedAt.IsZero() && !candidate.diskWriteStartedAt.After(observed.AcceptedAt()) { + refused = "unavailable_competing_writer" + } + } + if refused != "" { + record.mu.Lock() + if w.matchesWrite(record) { + value, state := w.slot(record.moveFile) + if *value == nil && *state == "pending_observer" { + *state = refused + } + } + record.mu.Unlock() + return + } + record.mu.Lock() + defer record.mu.Unlock() + if !w.matchesWrite(record) { + return + } + value, state := w.slot(record.moveFile) + if *value != nil || *state != "pending_observer" { + return + } + if record.moveFile.writeVerdict != "all_writes_completed" { + *state = "unavailable_move_write_verdict" + return + } + *value = &accepted + *state = "observed" +} +func (w moveDirectWitness) matchesWrite(record *mutationCommitRecord) bool { + return record.owner == w.server.Value() && record.id == w.id && record.kind == moveFileObservationCommit && record.moveFile != nil && record.key == w.operation && record.absPath == w.path && record.newSHA == w.blob && record.committedAt.Equal(w.committed) && record.commitSeq == w.commitSeq && record.disk == mutationDiskCommitted +} +func (w moveDirectWitness) unavailable(reason string) { + s, record := w.server.Value(), w.record.Value() + if s == nil || record == nil { + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[w.id] == record && w.matchesWrite(record) { + value, state := w.slot(record.moveFile) + if *value == nil && *state == "pending_observer" { + *state = reason + } + } +} + +func (o *moveObservation) reindex(ctx context.Context, path string) bool { + if o == nil { + return false + } + var idx *indexer.Indexer + if o.server.multiIndexer != nil { + idx, _ = o.server.multiIndexer.IndexerForFile(path) + } + observer := o.observer(ctx, path, idx) + if observer == nil && idx == nil { + o.noteDirectUnavailable(path, "unavailable_observer_owner") + } + // The existing path and fallback order remain unchanged. No scheduler is + // introduced, and no watcher ticket is invented for this direct reconcile. + success := o.server.reindexFileObserved(path, observer) + if !success && observer != nil { + o.noteDirectUnavailable(path, "unavailable_reconcile_not_confirmed") + } + return success +} +func (o *moveObservation) noteDirectUnavailable(path, reason string) { + record := o.records[path] + if record == nil { + return + } + ledger := &o.server.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + record.mu.Lock() + defer record.mu.Unlock() + if ledger.byID[record.id] == record && record.kind == moveFileObservationCommit && record.moveFile != nil && record.key == o.id && record.moveFile.direct == nil && (record.moveFile.directState == "not_started" || record.moveFile.directState == "pending_observer") { + record.moveFile.directClaimed = true + record.moveFile.directState = reason + } +} + +func (s *Server) moveFileObservationPayload(ctx context.Context, id string) (map[string]any, bool) { + if !strings.HasPrefix(id, "move-file-") { + return nil, false + } + payload := map[string]any{"found": false, "receipt": id, "receipt_kind": "move_file_observation", "guidance": "observation only; source_id/target_id/touched and real write outcomes remain the move authority"} + checkout, incarnation := mutationCheckoutScope(ctx) + var ack *editAcknowledgement + var direct, readAssisted *moveDirectObservation + ledger := &s.mutationCommits + ledger.mu.Lock() + record := ledger.byID[id] + if record != nil { + record.mu.RLock() + if record.kind == moveFileObservationCommit && record.owner == s && record.moveFile != nil && mutationCheckoutScopeMatches(checkout, incarnation, record.checkoutID, record.checkoutIncarnation) { + data := record.moveFile + payload["found"], payload["operation_id"], payload["source_id"], payload["target_id"], payload["path"], payload["role"] = true, data.operationID, data.sourceID, data.targetID, record.relPath, data.role + payload["move_write_verdict"], payload["file_write_status"], payload["writer_return_observed"], payload["new_sha"] = data.writeVerdict, record.disk, !record.committedAt.IsZero(), record.newSHA + payload["direct_reconcile_state"], payload["requested_background_state"] = data.directState, "not_applicable_direct_reconcile" + payload["read_assisted_reconcile_state"] = data.readState + payload["move_request_cause"] = map[string]string{"session": data.moveSession, "request_id_key": data.moveRequestKey} + // This handler calls the existing synchronous reconcile, never queue + // admission/wait. Preserve raw ACK nulls/counts and name applicability + // explicitly instead of manufacturing measured zero scheduler work. + payload["acknowledgement_phase_applicability"] = map[string]string{"publish": "not_applicable_direct_reconcile", "reindex_wait": "not_applicable_direct_reconcile"} + timing := map[string]any{"scope": "move_file_write", "clock": "monotonic", "state": "not_observed", "outcome": record.disk} + if !record.diskWriteStartedAt.IsZero() && !record.committedAt.IsZero() { + timing["state"] = "invalid_order" + if !record.diskWriteStartedAt.Before(record.startedAt) && !record.committedAt.Before(record.diskWriteStartedAt) { + timing["state"] = "measured" + timing["started_ns"] = record.diskWriteStartedAt.Sub(record.startedAt).Nanoseconds() + timing["returned_ns"] = record.committedAt.Sub(record.startedAt).Nanoseconds() + timing["window_ns"] = record.committedAt.Sub(record.diskWriteStartedAt).Nanoseconds() + } + } + payload["write_timing"] = timing + if data.direct != nil { + copy := *data.direct + direct = © + } + if data.readAssisted != nil { + copy := *data.readAssisted + readAssisted = © + } + ack = record.acknowledgement + } + record.mu.RUnlock() + } + ledger.mu.Unlock() + if direct != nil { + payload["direct_reconcile"] = direct.payload("move_direct_reconcile") + } + if readAssisted != nil { + payload["read_assisted_reconcile"] = readAssisted.payload("move_read_assisted_reconcile") + } + var snapshot *editAcknowledgementSnapshot + if ack != nil { + copy := ack.snapshot() + snapshot = © + } + payload["acknowledgement"] = buildAcknowledgementPayload(snapshot, "unavailable_ledger_entry_or_scope") + return payload, true +} + +func (observed *moveDirectObservation) payload(scope string) map[string]any { + phases := observed.accounting.Payload() + phases["scope"] = scope + causes := make([]mutationReadRepairCause, observed.causeCount) + copy(causes, observed.causes[:observed.causeCount]) + return map[string]any{"scope": scope, "clock": "monotonic", "owner_generation": observed.generation, "window_ns": observed.windowNS, "accepted_blob_sha": observed.acceptedBlob, "writer_return_to_acceptance_ns": observed.writerToAcceptanceNS, "reconcile_causes": causes, "phase_accounting": phases, "additive_with_acknowledgement": false, "request_cause": observed.requestCause, "started_from_ack_ns": observed.startedFromAckNS, "finished_from_ack_ns": observed.finishedFromAckNS, "accepted_from_start_ns": observed.acceptedFromStartNS} +} diff --git a/internal/mcp/move_file_observation_test.go b/internal/mcp/move_file_observation_test.go new file mode 100644 index 000000000..e95b1d765 --- /dev/null +++ b/internal/mcp/move_file_observation_test.go @@ -0,0 +1,366 @@ +package mcp + +import ( + "context" + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "go.uber.org/zap" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + "time" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func TestMoveFileObservationActualDirectReconcileIsBoundPerFile(t *testing.T) { + s, _ := setupContractMultiRepoServer(t, "fixture") + t.Cleanup(func() { + s.DrainBackground() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, s.multiIndexer.Close(ctx)) + }) + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + sourcePath, targetPath := filepath.Join(root, "source.go"), filepath.Join(root, "target.go") + require.NoError(t, os.WriteFile(sourcePath, []byte("package main\nfunc Relocated() int { return 7 }\nfunc StableCaller() int { return Relocated() }\n"), 0600)) + require.NoError(t, os.WriteFile(targetPath, []byte("package main\nfunc DestinationAnchor() int { return 2 }\n"), 0600)) + _, err := s.multiIndexer.IncrementalReindexRepo("fixture", []string{sourcePath, targetPath}) + require.NoError(t, err) + oldID, newID := "fixture/source.go::Relocated", "fixture/target.go::Relocated" + s.facades.capture(mcpgo.NewTool("move_symbol"), s.handleMoveSymbol) + req := editOperationRequest("refactor", "move") + req.Params.Arguments = map[string]any{"operation": "move", "target": map[string]any{"symbol": oldID}, "destination": "fixture/target.go", "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "refactor", r) + })(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + body := unmarshalResult(t, result) + require.Equal(t, oldID, body["source_id"]) + require.Equal(t, newID, body["target_id"]) + operation, ok := body["move_observation_id"].(string) + require.True(t, ok) + require.NotEmpty(t, operation) + refs, ok := body["file_observations"].([]any) + require.True(t, ok) + require.Len(t, refs, 2) + sharedRPC := "" + for _, raw := range refs { + ref := raw.(map[string]any) + id := ref["receipt"].(string) + var status map[string]any + require.Eventually(t, func() bool { + status, _ = s.moveFileObservationPayload(t.Context(), id) + return status["direct_reconcile_state"] == "observed" + }, time.Second, time.Millisecond, "actual direct owner evidence missing: %v", status) + require.Equal(t, "move_file_observation", status["receipt_kind"]) + require.Equal(t, operation, status["operation_id"]) + require.Equal(t, oldID, status["source_id"]) + require.Equal(t, newID, status["target_id"]) + require.Equal(t, ref["path"], status["path"]) + require.Equal(t, ref["role"], status["role"]) + require.Equal(t, "all_writes_completed", status["move_write_verdict"]) + require.Equal(t, mutationDiskCommitted, status["file_write_status"]) + require.Equal(t, true, status["writer_return_observed"]) + require.Equal(t, "not_applicable_direct_reconcile", status["requested_background_state"]) + applicability := status["acknowledgement_phase_applicability"].(map[string]string) + require.Equal(t, map[string]string{"publish": "not_applicable_direct_reconcile", "reindex_wait": "not_applicable_direct_reconcile"}, applicability) + require.NotContains(t, status, "mutation_id") + require.NotContains(t, status, "transaction_id") + require.NotContains(t, status, "reindex_receipt") + ack := status["acknowledgement"].(acknowledgementPayload) + require.NotNil(t, ack.RPCID) + for _, name := range []string{"publish", "reindex_wait"} { + value, present := ack.ObservedExecutionMS[name] + require.True(t, present) + require.Nil(t, value) + require.Zero(t, ack.Counts[name].Started) + require.Zero(t, ack.Counts[name].Completed) + } + if sharedRPC == "" { + sharedRPC = *ack.RPCID + } else { + require.Equal(t, sharedRPC, *ack.RPCID, "shared A must be deduplicated by its actual RPC owner") + } + direct := status["direct_reconcile"].(map[string]any) + require.Equal(t, "move_direct_reconcile", direct["scope"]) + require.Equal(t, "monotonic", direct["clock"]) + require.Equal(t, status["new_sha"], direct["accepted_blob_sha"]) + require.Positive(t, direct["owner_generation"]) + require.Positive(t, direct["window_ns"]) + require.Equal(t, false, direct["additive_with_acknowledgement"]) + phases := direct["phase_accounting"].(map[string]any) + require.Equal(t, "move_direct_reconcile", phases["scope"]) + require.Equal(t, true, phases["observed_partition_complete"]) + require.Equal(t, direct["window_ns"], phases["window_ns"]) + require.Equal(t, phases["window_ns"], phases["observed_subtotal_ns"]) + states := phases["phase_states"].(map[string]string) + require.Len(t, states, 9) + for name, state := range states { + require.Equal(t, "measured", state, "missing actual move executor phase %s", name) + } + direct["accepted_blob_sha"] = "foreign" + again, _ := s.moveFileObservationPayload(t.Context(), id) + require.Equal(t, status["new_sha"], again["direct_reconcile"].(map[string]any)["accepted_blob_sha"], "caller mutation changed retained observation") + } +} + +func moveObservationExecutionFixture(t *testing.T) (*Server, *moveObservation, context.Context, *editAcknowledgement, string, string) { + t.Helper() + s, _ := setupContractMultiRepoServer(t, "fixture") + t.Cleanup(func() { + s.DrainBackground() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, s.multiIndexer.Close(ctx)) + }) + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + source, target := filepath.Join(root, "source.go"), filepath.Join(root, "target.go") + require.NoError(t, os.WriteFile(source, []byte("package main\nfunc Before() {}\n"), 0600)) + require.NoError(t, os.WriteFile(target, []byte("package main\nfunc OldTarget() {}\n"), 0600)) + _, err := s.multiIndexer.IncrementalReindexRepo("fixture", []string{source, target}) + require.NoError(t, err) + ctx, ack, _ := s.withEditAcknowledgement(t.Context()) + t.Cleanup(ack.finishActual) + observation := s.beginMoveObservation(ctx, "fixture/source.go::Before", "fixture/target.go::Before", source, target, "fixture/source.go", "fixture/target.go", true) + require.NotNil(t, observation) + require.NoError(t, observation.write(ack, target, []byte("package main\nfunc Moved() {}\n"))) + require.NoError(t, observation.write(ack, source, []byte("package main\nfunc After() { Existing() }\n"))) + observation.finishWrites(true) + return s, observation, ctx, ack, source, target +} +func captureMoveActualExecution(t *testing.T, s *Server, path string, observer *indexer.ReadRepairObserver) indexer.ReadRepairObservation { + t.Helper() + done := make(chan indexer.ReadRepairObservation, 1) + wrapped := &indexer.ReadRepairObserver{Complete: func(o indexer.ReadRepairObservation) { + if observer != nil { + observer.Complete(o) + } + done <- o + }} + if observer != nil { + wrapped.Session, wrapped.RequestIDKey = observer.Session, observer.RequestIDKey + } + _, err := s.multiIndexer.IncrementalReindexRepoObserved("fixture", []string{path}, wrapped) + require.NoError(t, err) + select { + case actual := <-done: + return actual + case <-time.After(5 * time.Second): + t.Fatal("actual direct observer did not join") + return indexer.ReadRepairObservation{} + } +} + +func TestMoveDirectRefusesOlderAdmissionWhichActuallyCommitsLaterSameBytes(t *testing.T) { + s, _ := setupContractMultiRepoServer(t, "fixture") + t.Cleanup(func() { s.DrainBackground(); require.NoError(t, s.multiIndexer.Close(context.Background())) }) + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + source, target := filepath.Join(root, "source.go"), filepath.Join(root, "target.go") + bytes := []byte("package main\nfunc After() { Existing() }\n") + require.NoError(t, os.WriteFile(source, []byte("package main\nfunc Before() {}\n"), 0600)) + require.NoError(t, os.WriteFile(target, []byte("package main\nfunc OldTarget() {}\n"), 0600)) + _, err := s.multiIndexer.IncrementalReindexRepo("fixture", []string{source, target}) + require.NoError(t, err) + entered, release := make(chan *mutationCommitRecord, 1), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + s.mutationPreCommitHook = func(record *mutationCommitRecord) { entered <- record; <-release } + type result struct { + record *mutationCommitRecord + err error + } + done := make(chan result, 1) + joined := false + go func() { + record, err := s.commitFileMutation(t.Context(), "edit_file", "older-admission", "same-bytes", "fixture/source.go", source, bytes, 0600) + done <- result{record, err} + }() + t.Cleanup(func() { + unblock() + if !joined { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("competing real writer was not joined") + } + } + }) + var older *mutationCommitRecord + select { + case older = <-entered: + case <-time.After(5 * time.Second): + t.Fatal("older writer not admitted") + } + ctx, ack, _ := s.withEditAcknowledgement(t.Context()) + defer ack.finishActual() + observation := s.beginMoveObservation(ctx, "fixture/source.go::Before", "fixture/target.go::Before", source, target, "fixture/source.go", "fixture/target.go", true) + require.NoError(t, observation.write(ack, target, []byte("package main\nfunc Moved() {}\n"))) + require.NoError(t, observation.write(ack, source, bytes)) + observation.finishWrites(true) + idx := s.multiIndexer.GetIndexer("fixture") + observer := observation.observer(ctx, source, idx) + require.NotNil(t, observer) + unblock() + var finished result + select { + case finished = <-done: + joined = true + case <-time.After(5 * time.Second): + t.Fatal("older actual writer did not return") + } + require.NoError(t, finished.err) + require.Same(t, older, finished.record) + record := observation.records[source] + require.Greater(t, older.commitSeq, record.commitSeq, "guard must exercise actual commit order opposite admission order") + actual := captureMoveActualExecution(t, s, source, observer) + require.True(t, actual.AcceptedFor(idx, gitBlobSHA(bytes)), "wrong-stage rejection would not test competing commit fence") + status, _ := s.moveFileObservationPayload(t.Context(), record.id) + require.Equal(t, "unavailable_competing_commit", status["direct_reconcile_state"]) + require.NotContains(t, status, "direct_reconcile") + // A repeated callback must not revise the first final refusal even when + // competing evidence has since changed from committed to in-flight. + older.mu.Lock() + older.disk = mutationDiskInFlight + older.diskWriteStartedAt = actual.AcceptedAt() + older.mu.Unlock() + observer.Complete(actual) + repeated, _ := s.moveFileObservationPayload(t.Context(), record.id) + require.Equal(t, "unavailable_competing_commit", repeated["direct_reconcile_state"]) + require.NotContains(t, repeated, "direct_reconcile") +} + +func TestMoveDirectRejectsActualEarlyAcceptanceForeignOwnerBytesAndEviction(t *testing.T) { + for _, kind := range []string{"early_acceptance", "foreign_owner", "foreign_bytes", "evicted"} { + t.Run(kind, func(t *testing.T) { + s, observation, ctx, ack, source, target := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + expectedOwner := idx + if kind == "foreign_owner" { + expectedOwner = indexer.New(graph.New(), testRegistry(), config.Default().Index, zap.NewNop()) + } + observer := observation.observer(ctx, source, expectedOwner) + require.NotNil(t, observer) + record := observation.records[source] + if kind == "foreign_bytes" { + require.NoError(t, writeMoveFileObserved(nil, source, []byte("package main\nfunc OtherBytes() { Existing() }\n"))) + } + actual := captureMoveActualExecution(t, s, source, nil) + if kind == "early_acceptance" { + ctx2, ack2, _ := s.withEditAcknowledgement(t.Context()) + defer ack2.finishActual() + second := s.beginMoveObservation(ctx2, "fixture/source.go::Before", "fixture/target.go::Before", source, target, "fixture/source.go", "fixture/target.go", true) + require.NoError(t, second.write(ack2, target, []byte("package main\nfunc Moved() {}\n"))) + require.NoError(t, second.write(ack2, source, []byte("package main\nfunc After() { Existing() }\n"))) + second.finishWrites(true) + observation, record = second, second.records[source] + observer = second.observer(ctx2, source, idx) + require.NotNil(t, observer) + require.True(t, actual.AcceptedFor(idx, record.newSHA)) + require.True(t, actual.AcceptedAt().Before(record.committedAt), "actual acceptance must precede the new real writer return") + } + if kind == "evicted" { + require.True(t, actual.AcceptedFor(idx, record.newSHA)) + record.mu.Lock() + record.startedAt = time.Now().Add(-mutationCommitRetention - time.Second) + record.mu.Unlock() + s.mutationCommits.mu.Lock() + s.mutationCommits.evictLocked(time.Now()) + s.mutationCommits.mu.Unlock() + } + observer.Complete(actual) + status, _ := s.moveFileObservationPayload(t.Context(), record.id) + if kind == "evicted" { + require.Equal(t, false, status["found"]) + } else { + require.NotContains(t, status, "direct_reconcile") + require.Equal(t, "unavailable_owner_bytes_or_acceptance", status["direct_reconcile_state"]) + } + runtime.KeepAlive(expectedOwner) + _ = ack // shared owner remains live until cleanup + _ = observation + }) + } +} + +func TestMoveFileWriteWindowsPartialFailureAndUnobservedIsolation(t *testing.T) { + s, _, ctx, ack, source, target := moveObservationExecutionFixture(t) + observation := s.beginMoveObservation(ctx, "fixture/source.go::Before", "fixture/target.go::Before", source, target, "fixture/source.go", "fixture/target.go", true) + targetBytes := []byte("package main\nfunc ActuallyWritten() {}\n") + require.NoError(t, observation.write(ack, target, targetBytes)) + require.NoError(t, os.Remove(source)) + require.NoError(t, os.Mkdir(source, 0700)) + require.Error(t, observation.write(ack, source, []byte("package main\nfunc CannotReplaceDirectory() {}\n"))) + observation.finishWrites(false) + targetStatus, _ := s.moveFileObservationPayload(t.Context(), observation.records[target].id) + sourceStatus, _ := s.moveFileObservationPayload(t.Context(), observation.records[source].id) + require.Equal(t, "partial_or_failed", targetStatus["move_write_verdict"]) + require.Equal(t, "partial_or_failed", sourceStatus["move_write_verdict"]) + require.Equal(t, mutationDiskCommitted, targetStatus["file_write_status"]) + require.Equal(t, mutationDiskFailed, sourceStatus["file_write_status"]) + timing := targetStatus["write_timing"].(map[string]any) + require.Equal(t, "measured", timing["state"]) + require.Equal(t, timing["window_ns"], timing["returned_ns"].(int64)-timing["started_ns"].(int64)) + require.Positive(t, timing["window_ns"]) + require.Nil(t, observation.observer(ctx, target, s.multiIndexer.GetIndexer("fixture")), "partial move must not gain successful source verdict") + require.NotContains(t, targetStatus, "direct_reconcile") + before := len(s.mutationCommits.ordered) + require.Nil(t, s.beginMoveObservation(t.Context(), "old", "new", source, target, "source.go", "target.go", true)) + require.Equal(t, before, len(s.mutationCommits.ordered), "unobserved path allocated timing records") + for _, ref := range observation.references() { + _, physical := s.mutationCommits.byReceipt(ref.Receipt) + require.False(t, physical, "move timing acquired physical receipt authority") + } +} + +func TestMoveOriginalDirectAdmissionCannotBeReissued(t *testing.T) { + s, observation, ctx, _, source, _ := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + first := observation.observer(ctx, source, idx) + require.NotNil(t, first) + require.Nil(t, observation.observer(ctx, source, idx), "an original direct observation cannot borrow a second reconcile") + actual := captureMoveActualExecution(t, s, source, first) + require.True(t, actual.AcceptedFor(idx, observation.records[source].newSHA)) + before, _ := s.moveFileObservationPayload(t.Context(), observation.records[source].id) + require.Equal(t, "observed", before["direct_reconcile_state"]) + require.Nil(t, observation.observer(ctx, source, idx)) + after, _ := s.moveFileObservationPayload(t.Context(), observation.records[source].id) + require.Equal(t, before["direct_reconcile"], after["direct_reconcile"], "completed original owner changed after re-admission attempt") +} + +// The source and target are written before either direct reconciliation. +// An independent public reader may therefore publish the target first. The +// later direct no-op cannot donate a new acceptance window to that write. +func TestMovePublicReadPublishesTargetBeforeDirectAdmission(t *testing.T) { + s, observation, ctx, _, _, target := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + const movedID = "fixture/target.go::Moved" + require.Nil(t, s.engineFor(t.Context()).GetSymbol(movedID), "fixture must require an actual target publication") + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + readerCtx := WithSessionCWD(WithSessionID(ctx, "move-independent-reader"), root) + req := mcpgo.CallToolRequest{} + req.Params.Arguments = map[string]any{"id": movedID, "context_lines": 0} + result, err := s.handleGetSymbolSource(readerCtx, req) + require.NoError(t, err) + require.False(t, result.IsError) + require.Contains(t, unmarshalResult(t, result)["source"], "func Moved()") + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(movedID), "actual public read did not apply the target") + // Keep real executor proof: unchanged direct work cannot be relabeled as + // applied acceptance, regardless of how the earlier reader is retained. + later := captureMoveActualExecution(t, s, target, observation.observer(ctx, target, idx)) + require.False(t, later.AcceptedFor(idx, observation.records[target].newSHA), "fixture must exercise the later no-op, not another actual apply") + status, _ := s.moveFileObservationPayload(t.Context(), observation.records[target].id) + require.Equal(t, "observed", status["read_assisted_reconcile_state"], "the actual earlier public publisher must retain its own independent window") + require.Contains(t, status, "read_assisted_reconcile") + require.NotEqual(t, "observed", status["direct_reconcile_state"], "read-assisted publication must not relabel the later direct no-op") + require.NotContains(t, status, "direct_reconcile") +} diff --git a/internal/mcp/move_measurement_contract_test.go b/internal/mcp/move_measurement_contract_test.go new file mode 100644 index 000000000..099d8b32f --- /dev/null +++ b/internal/mcp/move_measurement_contract_test.go @@ -0,0 +1,171 @@ +package mcp + +import ( + "context" + "encoding/json" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" + "testing" + "time" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func TestMoveMeasurementUsesActualGoRelocationAndExplicitLanguageRefusal(t *testing.T) { + for _, language := range []string{"go", "typescript", "python"} { + t.Run(language, func(t *testing.T) { + s, root := setupTestServer(t) + t.Cleanup(s.DrainBackground) + file, destination, source, target := "move_source.go", "move_target.go", "package main\nfunc Relocated() int { return 7 }\nfunc StableCaller() int { return Relocated() }\n", "package main\nfunc DestinationAnchor() int { return 2 }\n" + if language == "typescript" { + file = "move_source.ts" + destination = "move_target.ts" + source = "export function Relocated(): number { return 7; }\n" + target = "export function DestinationAnchor(): number { return 2; }\n" + } + if language == "python" { + file = "move_source.py" + destination = "move_target.py" + source = "def Relocated():\n return 7\n" + target = "def DestinationAnchor():\n return 2\n" + } + sourcePath, targetPath := filepath.Join(root, file), filepath.Join(root, destination) + require.NoError(t, os.WriteFile(sourcePath, []byte(source), 0600)) + require.NoError(t, os.WriteFile(targetPath, []byte(target), 0600)) + require.NoError(t, s.indexer.IndexFile(sourcePath)) + require.NoError(t, s.indexer.IndexFile(targetPath)) + s.indexer.ResolveAll() + oldID, newID := file+"::Relocated", destination+"::Relocated" + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(oldID), "language refusal must operate on an actual indexed declaration") + s.facades.capture(mcpgo.NewTool("move_symbol"), s.handleMoveSymbol) + req := editOperationRequest("refactor", "move") + req.Params.Arguments = map[string]any{"operation": "move", "target": map[string]any{"symbol": oldID}, "destination": destination, "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "refactor", r) + })(t.Context(), req) + require.NoError(t, err) + if language != "go" { + require.True(t, result.IsError) + require.Contains(t, mustErrText(t, result), "unsupported language: move_symbol only supports Go files") + actualSource, err := os.ReadFile(sourcePath) + require.NoError(t, err) + require.Equal(t, source, string(actualSource)) + actualTarget, err := os.ReadFile(targetPath) + require.NoError(t, err) + require.Equal(t, target, string(actualTarget)) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(oldID)) + require.Nil(t, s.engineFor(t.Context()).GetSymbol(newID)) + return + } + require.False(t, result.IsError) + body := unmarshalResult(t, result) + require.Equal(t, oldID, body["source_id"]) + require.Equal(t, newID, body["target_id"]) + require.NotContains(t, body, "mutation_receipt", "multi-file relocation must not be represented as one physical edit receipt") + require.Nil(t, s.engineFor(t.Context()).GetSymbol(oldID)) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(newID)) + oldSource := callTool(t, s, "get_symbol_source", map[string]any{"id": oldID}) + require.True(t, oldSource.IsError) + // Preserve the real producer wire shape consumed by expected-absence + // probes: suggestions share the line after an exact ID boundary. + oldWire, marshalErr := json.Marshal(oldSource) + require.NoError(t, marshalErr) + var oldEnvelope struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + require.NoError(t, json.Unmarshal(oldWire, &oldEnvelope)) + require.Len(t, oldEnvelope.Content, 1) + require.Equal(t, "text", oldEnvelope.Content[0].Type) + require.True(t, strings.HasPrefix(oldEnvelope.Content[0].Text, "symbol not found: "+oldID+" — did you mean: ")) + require.Contains(t, oldEnvelope.Content[0].Text, newID) + newSource := callTool(t, s, "get_symbol_source", map[string]any{"id": newID, "context_lines": 0}) + require.False(t, newSource.IsError) + require.Contains(t, unmarshalResult(t, newSource)["source"], "return 7") + chain := callTool(t, s, "get_call_chain", map[string]any{"id": file + "::StableCaller", "depth": 1, "limit": 1000}) + require.False(t, chain.IsError) + found := false + for _, raw := range unmarshalResult(t, chain)["edges"].([]any) { + edge := raw.(map[string]any) + if edge["from"] == file+"::StableCaller" && edge["to"] == newID && edge["kind"] == "calls" { + found = true + } + require.NotEqual(t, oldID, edge["to"], "caller retained old location") + } + rawOut, _ := json.Marshal(s.graph.GetOutEdges(file + "::StableCaller")) + require.True(t, found, "unchanged Go caller must resolve new location; chain=%v raw_out=%s", unmarshalResult(t, chain), string(rawOut)) + }) + } +} + +func TestMoveMeasurementActualMultiIndexerPreservesUnchangedCaller(t *testing.T) { + s, _ := setupContractMultiRepoServer(t, "fixture") + t.Cleanup(func() { + s.DrainBackground() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + require.NoError(t, s.multiIndexer.Close(ctx)) + }) + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + sourcePath, targetPath := filepath.Join(root, "move_source.go"), filepath.Join(root, "move_target.go") + require.NoError(t, os.WriteFile(sourcePath, []byte("package main\nfunc Relocated() int { return 7 }\nfunc StableCaller() int { return Relocated() }\n"), 0600)) + require.NoError(t, os.WriteFile(targetPath, []byte("package main\nfunc DestinationAnchor() int { return 2 }\n"), 0600)) + _, err := s.multiIndexer.IncrementalReindexRepo("fixture", []string{sourcePath, targetPath}) + require.NoError(t, err) + oldID, newID, callerID := "fixture/move_source.go::Relocated", "fixture/move_target.go::Relocated", "fixture/move_source.go::StableCaller" + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(callerID), "actual caller must exist before move") + s.facades.capture(mcpgo.NewTool("move_symbol"), s.handleMoveSymbol) + req := editOperationRequest("refactor", "move") + req.Params.Arguments = map[string]any{"operation": "move", "target": map[string]any{"symbol": oldID}, "destination": "fixture/move_target.go", "output": map[string]any{"format": "json"}} + result, err := s.boundToolHandler(func(ctx context.Context, r mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error) { + return s.handleFacade(ctx, "refactor", r) + })(t.Context(), req) + require.NoError(t, err) + require.False(t, result.IsError) + source, err := os.ReadFile(sourcePath) + require.NoError(t, err) + require.Contains(t, string(source), "func StableCaller()") + require.Nil(t, s.engineFor(t.Context()).GetSymbol(oldID)) + require.NotNil(t, s.engineFor(t.Context()).GetSymbol(newID)) + chain := callTool(t, s, "get_call_chain", map[string]any{"id": callerID, "depth": 1, "limit": 1000}) + require.False(t, chain.IsError) + body := unmarshalResult(t, chain) + edges, _ := body["edges"].([]any) + found := false + for _, raw := range edges { + edge := raw.(map[string]any) + if edge["from"] == callerID && edge["to"] == newID && edge["kind"] == "calls" { + found = true + } + } + caller, _ := json.Marshal(s.graph.GetNode(callerID)) + rawOut, _ := json.Marshal(s.graph.GetOutEdges(callerID)) + require.True(t, found, "multi move chain=%v caller=%s raw_out=%s source=%s", body, string(caller), string(rawOut), string(source)) +} + +func TestMoveSplicePreservesRequiredDeclarationSeparator(t *testing.T) { + for _, source := range []string{ + "package main\nfunc Relocated() int { return 7 }\nfunc StableCaller() int { return Relocated() }\n", + "package main\n\nfunc Relocated() int { return 7 }\n\nfunc StableCaller() int { return Relocated() }\n", + "package main\n// Relocated is moved.\nfunc Relocated() int { return 7 }\nfunc StableCaller() int { return Relocated() }\n", + } { + set := token.NewFileSet() + file, err := parser.ParseFile(set, "source.go", source, parser.ParseComments) + require.NoError(t, err) + decl, _, _, err := extractGoTopLevelDecl(set, file, []byte(source), "Relocated") + require.NoError(t, err) + after, err := spliceOutGoDecl([]byte(source), set, decl) + require.NoError(t, err) + parsed, err := parser.ParseFile(token.NewFileSet(), "after.go", after, 0) + require.NoError(t, err, "splice emitted invalid Go: %s", after) + require.Len(t, parsed.Decls, 1) + require.Contains(t, string(after), "func StableCaller()") + } +} diff --git a/internal/mcp/move_read_assisted_observation_test.go b/internal/mcp/move_read_assisted_observation_test.go new file mode 100644 index 000000000..3a7dd9491 --- /dev/null +++ b/internal/mcp/move_read_assisted_observation_test.go @@ -0,0 +1,208 @@ +package mcp + +import ( + "context" + "fmt" + "os" + "runtime" + "strings" + "sync" + "testing" + "time" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func TestMoveReadAssistedOriginalFileWinnerAndImmutableCallback(t *testing.T) { + s, move, ctx, _, _, target := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + readerCtx := WithSessionID(ctx, "independent-move-reader") + observer := s.readRepairObserver(readerCtx, target, idx) + require.NotNil(t, observer) + actual := captureMoveActualExecution(t, s, target, observer) + require.True(t, actual.AcceptedFor(idx, move.records[target].newSHA)) + status, _ := s.moveFileObservationPayload(t.Context(), move.records[target].id) + require.Equal(t, "observed", status["read_assisted_reconcile_state"]) + require.Equal(t, "not_started", status["direct_reconcile_state"]) + repair := status["read_assisted_reconcile"].(map[string]any) + require.Equal(t, "move_read_assisted_reconcile", repair["scope"]) + require.Equal(t, false, repair["additive_with_acknowledgement"]) + require.Equal(t, "independent-move-reader", repair["request_cause"].(mutationReadRepairCause).Session) + start, end := actual.Bounds() + require.Equal(t, end.Sub(start).Nanoseconds(), repair["finished_from_ack_ns"].(int64)-repair["started_from_ack_ns"].(int64)) + require.Equal(t, actual.AcceptedAt().Sub(start).Nanoseconds(), repair["accepted_from_start_ns"]) + require.GreaterOrEqual(t, repair["accepted_from_start_ns"].(int64), int64(0)) + require.LessOrEqual(t, repair["accepted_from_start_ns"].(int64), repair["window_ns"].(int64)) + require.Equal(t, "independent-move-reader", repair["reconcile_causes"].([]mutationReadRepairCause)[0].Session) + require.Nil(t, s.readRepairObserver(readerCtx, target, idx), "original read admission cannot be reissued") + observer.Complete(actual) + again, _ := s.moveFileObservationPayload(t.Context(), move.records[target].id) + require.Equal(t, repair, again["read_assisted_reconcile"]) +} + +func TestMoveReadAssistedRejectsNewerPublicationOwners(t *testing.T) { + for _, kind := range []mutationCommitKind{physicalMutationCommit, batchFileObservationCommit, moveFileObservationCommit} { + t.Run(fmt.Sprintf("kind-%d", kind), func(t *testing.T) { + s, move, ctx, _, _, target := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + observer := s.readRepairObserver(WithSessionID(ctx, "original-reader"), target, idx) + require.NotNil(t, observer) + // Complete a real later same-byte write. Its separately retained ledger + // owner has the actual later commit order for each supported owner kind. + bytes, err := os.ReadFile(target) + require.NoError(t, err) + require.NoError(t, writeMoveFileObserved(nil, target, bytes)) + foreign := &mutationCommitRecord{kind: kind, owner: s, id: "foreign-newer", absPath: target, disk: mutationDiskCommitted, newSHA: gitBlobSHA(bytes), committedAt: time.Now(), commitSeq: mutationDiskCommitSequence.Add(1)} + s.mutationCommits.put(foreign) + actual := captureMoveActualExecution(t, s, target, observer) + require.True(t, actual.AcceptedFor(idx, gitBlobSHA(bytes)), "test must reach actual accepted-byte ownership rejection") + status, _ := s.moveFileObservationPayload(t.Context(), move.records[target].id) + require.Equal(t, "unavailable_competing_commit", status["read_assisted_reconcile_state"]) + require.NotContains(t, status, "read_assisted_reconcile") + // A repeated callback cannot rewrite this final refusal. + observer.Complete(actual) + repeated, _ := s.moveFileObservationPayload(t.Context(), move.records[target].id) + require.Equal(t, status["read_assisted_reconcile_state"], repeated["read_assisted_reconcile_state"]) + _, moveOwner := s.moveReadRepairObserver(ctx, target, idx) + require.Equal(t, kind == moveFileObservationCommit, moveOwner, "latest non-move authority must not fall back to an older move") + }) + } +} + +func TestMoveReadAssistedPendingVerdictAndUnknownOwnerRemainUnavailable(t *testing.T) { + s, move, ctx, _, _, target := moveObservationExecutionFixture(t) + idx := s.multiIndexer.GetIndexer("fixture") + require.Nil(t, s.readRepairObserver(ctx, target, nil)) + record := move.records[target] + record.mu.Lock() + record.moveFile.writeVerdict = "pending" + record.mu.Unlock() + observer := s.readRepairObserver(WithSessionID(ctx, "pending-move-reader"), target, idx) + require.NotNil(t, observer, "capture only a weak known-writer-return candidate") + actual := captureMoveActualExecution(t, s, target, observer) + require.True(t, actual.AcceptedFor(idx, record.newSHA)) + status, _ := s.moveFileObservationPayload(context.Background(), record.id) + require.Equal(t, "unavailable_move_write_verdict", status["read_assisted_reconcile_state"]) + require.NotContains(t, status, "read_assisted_reconcile") + move.finishWrites(true) + observer.Complete(actual) + after, _ := s.moveFileObservationPayload(t.Context(), record.id) + require.Equal(t, "unavailable_move_write_verdict", after["read_assisted_reconcile_state"], "later final verdict cannot manufacture a callback window") +} + +func TestMoveActualCoalescedPublicReaderAndDirectKeepBothCausesOneExecution(t *testing.T) { + s, move, ctx, _, _, target := moveObservationExecutionFixture(t) + root, ok := s.multiIndexer.RepoRoot("fixture") + require.True(t, ok) + entered, resume := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(resume) }) } + gateDone := make(chan error, 1) + go func() { + gateDone <- s.multiIndexer.RunRepositoryTopologyBatch(t.Context(), func(context.Context) error { close(entered); <-resume; return nil }) + }() + type readerResult struct { + result *mcpgo.CallToolResult + err error + } + readerDone, directDone := make(chan readerResult, 1), make(chan bool, 1) + gateJoined, readerJoined, directJoined := false, false, false + readerStarted, directStarted := false, false + t.Cleanup(func() { + release() + if !gateJoined { + select { + case <-gateDone: + case <-time.After(5 * time.Second): + t.Error("coalescing gate did not join") + } + } + if readerStarted && !readerJoined { + select { + case <-readerDone: + case <-time.After(5 * time.Second): + t.Error("actual public reader did not join") + } + } + if directStarted && !directJoined { + select { + case <-directDone: + case <-time.After(5 * time.Second): + t.Error("actual direct caller did not join") + } + } + }) + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("existing topology gate was not held") + } + readerStarted = true + go func() { + request := mcpgo.CallToolRequest{} + request.Params.Arguments = map[string]any{"id": "fixture/target.go::Moved", "context_lines": 0} + result, err := s.handleGetSymbolSource(WithSessionCWD(WithSessionID(ctx, "actual-coalesced-reader"), root), request) + readerDone <- readerResult{result, err} + }() + directStarted = true + go func() { directDone <- move.reindex(ctx, target) }() + // The existing gate blocks drain before it takes pending paths. Observe the + // two real callers parked after coordinator admission, not just goroutine + // launch or observer construction; then release one genuinely coalesced batch. + require.Eventually(t, func() bool { + record := move.records[target] + record.mu.RLock() + pending := record.moveFile.directState == "pending_observer" && record.moveFile.readState == "pending_observer" + record.mu.RUnlock() + if !pending { + return false + } + stack := make([]byte, 128<<10) + n := runtime.Stack(stack, true) + return strings.Count(string(stack[:n]), "(*repositoryMutationCoordinator).reconcileObserved(") >= 2 + }, 5*time.Second, time.Millisecond) + release() + select { + case err := <-gateDone: + gateJoined = true + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("gate did not finish") + } + select { + case result := <-readerDone: + readerJoined = true + require.NoError(t, result.err) + require.False(t, result.result.IsError) + case <-time.After(5 * time.Second): + t.Fatal("public read did not finish") + } + select { + case success := <-directDone: + directJoined = true + require.True(t, success) + case <-time.After(5 * time.Second): + t.Fatal("direct reconcile did not finish") + } + status, _ := s.moveFileObservationPayload(t.Context(), move.records[target].id) + require.Equal(t, "observed", status["direct_reconcile_state"]) + require.Equal(t, "observed", status["read_assisted_reconcile_state"]) + direct := status["direct_reconcile"].(map[string]any) + repair := status["read_assisted_reconcile"].(map[string]any) + for _, key := range []string{"owner_generation", "window_ns", "accepted_blob_sha", "writer_return_to_acceptance_ns", "started_from_ack_ns", "finished_from_ack_ns", "accepted_from_start_ns", "reconcile_causes"} { + require.Equal(t, direct[key], repair[key], "different executed %s", key) + } + require.Equal(t, "actual-coalesced-reader", repair["request_cause"].(mutationReadRepairCause).Session) + require.NotEqual(t, repair["request_cause"], direct["request_cause"]) + require.Len(t, repair["reconcile_causes"], 2) + directPhases := direct["phase_accounting"].(map[string]any) + repairPhases := repair["phase_accounting"].(map[string]any) + require.Equal(t, "move_direct_reconcile", directPhases["scope"]) + require.Equal(t, "move_read_assisted_reconcile", repairPhases["scope"]) + for key, value := range directPhases { + if key != "scope" { + require.Equal(t, value, repairPhases[key], "different partition %s", key) + } + } +} diff --git a/internal/mcp/mutation_commit.go b/internal/mcp/mutation_commit.go index c736a0708..ebea6fb81 100644 --- a/internal/mcp/mutation_commit.go +++ b/internal/mcp/mutation_commit.go @@ -13,6 +13,7 @@ import ( "sync" "sync/atomic" "time" + "weak" "github.com/zzet/gortex/internal/agents" "github.com/zzet/gortex/internal/indexer" @@ -98,7 +99,19 @@ var mutationDiskCommitSequence atomic.Uint64 // the client knows retrying is safe. var errMutationNotApplied = errors.New("mutation not applied") +type mutationCommitKind uint8 + +const ( + physicalMutationCommit mutationCommitKind = iota + batchAcknowledgementCommit + // Measurement-only file owner. Durable batch verdicts remain authoritative; + // this kind cannot enter ordinary commit/replay/pending-source paths. + batchFileObservationCommit + moveFileObservationCommit +) + type mutationCommitRecord struct { + kind mutationCommitKind // immutable; zero retains the existing physical-record contract mu sync.RWMutex owner *Server pendingSourceRecovered bool @@ -137,6 +150,23 @@ type mutationCommitRecord struct { startedAt time.Time committedAt time.Time + // Each physical file commit owns its disk boundary and timing record. + // Request-wide first-wins stamps cannot identify a second file's write. + diskWriteStartedAt time.Time + publication *indexer.PublicationPhaseRecord + // Files written by one measured RPC share one acknowledgement, retained + // only while this record belongs to the bounded ledger. Pending-source + // evidence can outlive that ownership, so eviction must clear this too. + acknowledgement *editAcknowledgement + readRepair *mutationReadRepairObservation + // Measurement-only admission witness. Public reindex fields are filled + // after waiting and deliberately omit Receipt on immediate success. + readRepairAdmission weak.Pointer[mutationReceipt] + // Batch observations use this fixed-size owner identity without entering + // any physical mutation index or durable transaction manifest. + batchOwner [32]byte + batchAfterSHA256 [32]byte + moveFile *moveFileObservation // response is the successful tool payload, retained only for records that // carry a caller-chosen mutation_id. Without a key there is nothing to @@ -168,9 +198,13 @@ type mutationCommitSnapshot struct { AppliedGeneration uint64 `json:"applied_generation,omitempty"` CheckoutID string `json:"checkout_id,omitempty"` CheckoutIncarnation string `json:"checkout_incarnation,omitempty"` + readRepair *mutationReadRepairObservation } func (r *mutationCommitRecord) snapshot() mutationCommitSnapshot { + if !r.isPhysicalCommit() { + return mutationCommitSnapshot{} + } r.mu.RLock() defer r.mu.RUnlock() snap := mutationCommitSnapshot{ @@ -192,6 +226,10 @@ func (r *mutationCommitRecord) snapshot() mutationCommitSnapshot { CheckoutID: r.checkoutID, CheckoutIncarnation: r.checkoutIncarnation, } + if r.readRepair != nil { + observation := *r.readRepair + snap.readRepair = &observation + } if !r.startedAt.IsZero() { snap.StartedAt = r.startedAt.UTC().Format(time.RFC3339Nano) } @@ -202,12 +240,18 @@ func (r *mutationCommitRecord) snapshot() mutationCommitSnapshot { } func (r *mutationCommitRecord) diskStatus() string { + if !r.isPhysicalCommit() { + return "" + } r.mu.RLock() defer r.mu.RUnlock() return r.disk } func (r *mutationCommitRecord) markNotApplied(err error) { + if !r.isPhysicalCommit() { + return + } r.mu.Lock() defer func() { r.mu.Unlock() @@ -224,6 +268,9 @@ func (r *mutationCommitRecord) markNotApplied(err error) { } func (r *mutationCommitRecord) markFailed(err error) { + if !r.isPhysicalCommit() { + return + } r.mu.Lock() defer func() { r.mu.Unlock() @@ -240,6 +287,9 @@ func (r *mutationCommitRecord) markFailed(err error) { } func (r *mutationCommitRecord) markCommitted(newSHA string, bytesWritten int) { + if !r.isPhysicalCommit() { + return + } r.mu.Lock() defer r.mu.Unlock() r.disk = mutationDiskCommitted @@ -262,7 +312,7 @@ func (r *mutationCommitRecord) recordGraph(outcome mutationReindexOutcome) { // An asynchronous observer may apply only to the receipt it sampled. Empty // expectedReceipt is reserved for the mutation handler binding its outcome. func (r *mutationCommitRecord) recordGraphForReceipt(expectedReceipt string, outcome mutationReindexOutcome) { - if r == nil { + if !r.isPhysicalCommit() { return } // Keep the binding and the outcome atomic with respect to ledger readers. @@ -297,6 +347,22 @@ func (r *mutationCommitRecord) recordGraphForReceipt(expectedReceipt string, out if r.reindexReceipt != outcome.Receipt { r.pendingSourceRecovered = false } + admission := r.readRepairAdmission.Value() + var admittedReceipt string + var admittedGeneration uint64 + if admission != nil { + admittedReceipt, admittedGeneration = admission.id, admission.generation + } else if r.readRepair != nil { + admittedReceipt, admittedGeneration = r.readRepair.requestedReceipt, r.readRepair.requestedGeneration + } + compatibleAdmission := admittedGeneration != 0 && admittedGeneration == outcome.Generation && + (outcome.Receipt == "" || outcome.Receipt == admittedReceipt) && + (outcome.checkoutID == "" || (r.checkoutID == outcome.checkoutID && r.checkoutIncarnation == outcome.checkoutIncarnation)) + if !compatibleAdmission && (admittedGeneration != 0 || r.reindexReceipt != outcome.Receipt || r.reindexGeneration != outcome.Generation || + (outcome.checkoutID != "" && (r.checkoutID != outcome.checkoutID || r.checkoutIncarnation != outcome.checkoutIncarnation))) { + r.readRepair = nil + r.readRepairAdmission = weak.Pointer[mutationReceipt]{} + } if ledger != nil { ledger.unbindFreshnessLocked(r) // A detached completion must not resurrect an evicted commit. @@ -320,6 +386,9 @@ func (r *mutationCommitRecord) recordGraphForReceipt(expectedReceipt string, out } func (r *mutationCommitRecord) markSourceRecoveredForReceipt(expectedReceipt string) { + if !r.isPhysicalCommit() { + return + } r.mu.Lock() defer r.mu.Unlock() if expectedReceipt != "" && r.reindexReceipt == expectedReceipt { @@ -330,6 +399,9 @@ func (r *mutationCommitRecord) markSourceRecoveredForReceipt(expectedReceipt str // A missing bound ticket is terminal failure, never publication evidence. The // identity/state recheck avoids overriding a concurrent rebind or completion. func (r *mutationCommitRecord) markReceiptLost(id string) { + if !r.isPhysicalCommit() { + return + } r.mu.Lock() defer r.mu.Unlock() if id == "" || r.reindexReceipt != id || @@ -345,7 +417,7 @@ func (r *mutationCommitRecord) markReceiptLost(id string) { // retainResponse stores the successful payload for idempotent replay. Only // keyed records retain it — see mutationCommitRecord.response. func (r *mutationCommitRecord) retainResponse(resp map[string]any) { - if r == nil { + if !r.isPhysicalCommit() { return } r.mu.Lock() @@ -361,6 +433,9 @@ func (r *mutationCommitRecord) retainResponse(resp map[string]any) { } func (r *mutationCommitRecord) replayResponse() (map[string]any, bool) { + if !r.isPhysicalCommit() { + return nil, false + } r.mu.RLock() defer r.mu.RUnlock() if r.response == nil { @@ -373,6 +448,14 @@ func (r *mutationCommitRecord) replayResponse() (map[string]any, bool) { return clone, true } +func (r *mutationCommitRecord) isPhysicalCommit() bool { + return r != nil && r.kind == physicalMutationCommit +} + +func (r *mutationCommitRecord) isPublicationOwner() bool { + return r != nil && (r.kind == physicalMutationCommit || r.kind == batchFileObservationCommit) +} + // graphStatusFor collapses a reindex outcome into the graph half of a receipt. func graphStatusFor(outcome mutationReindexOutcome) string { switch { @@ -391,6 +474,9 @@ func graphStatusFor(outcome mutationReindexOutcome) string { // rather than a sync.Map because eviction needs a total order over entries, and // bounding it matters more here than lock-free reads on a path that already // performs a filesystem write. +// A distinct batch-only acknowledgement kind shares the same bound, but is +// excluded from every physical/source/replay index and listing. This ledger +// is not a general-purpose timing registry. type mutationCommitLedger struct { mu sync.Mutex byID map[string]*mutationCommitRecord @@ -409,11 +495,11 @@ func (l *mutationCommitLedger) put(record *mutationCommitRecord) { } l.evictLocked(time.Now()) l.byID[record.id] = record - if record.key != "" { + if record.isPhysicalCommit() && record.key != "" { l.byKey[record.key] = record } record.mu.RLock() - if record.reindexReceipt != "" { + if record.isPhysicalCommit() && record.reindexReceipt != "" { l.byReindexReceipt[record.reindexReceipt] = record } record.mu.RUnlock() @@ -423,6 +509,9 @@ func (l *mutationCommitLedger) put(record *mutationCommitRecord) { // unbindFreshnessLocked requires the ledger lock and the record's read or // write lock. Another commit may have claimed the same receipt in the interim. func (l *mutationCommitLedger) unbindFreshnessLocked(record *mutationCommitRecord) { + if !record.isPublicationOwner() { + return + } if l.byReindexReceipt[record.reindexReceipt] == record { delete(l.byReindexReceipt, record.reindexReceipt) } @@ -440,6 +529,13 @@ func (l *mutationCommitLedger) evictLocked(now time.Time) { } record.mu.RUnlock() if expired { + record.mu.Lock() + record.publication = nil + record.acknowledgement = nil + record.readRepair = nil + record.moveFile = nil + record.readRepairAdmission = weak.Pointer[mutationReceipt]{} + record.mu.Unlock() delete(l.byID, record.id) if record.key != "" && l.byKey[record.key] == record { delete(l.byKey, record.key) @@ -452,9 +548,14 @@ func (l *mutationCommitLedger) evictLocked(now time.Time) { for len(l.ordered) >= maxMutationCommits { oldest := l.ordered[0] l.ordered = l.ordered[1:] - oldest.mu.RLock() + oldest.mu.Lock() l.unbindFreshnessLocked(oldest) - oldest.mu.RUnlock() + oldest.publication = nil + oldest.acknowledgement = nil + oldest.readRepair = nil + oldest.moveFile = nil + oldest.readRepairAdmission = weak.Pointer[mutationReceipt]{} + oldest.mu.Unlock() delete(l.byID, oldest.id) if oldest.key != "" && l.byKey[oldest.key] == oldest { delete(l.byKey, oldest.key) @@ -466,6 +567,9 @@ func (l *mutationCommitLedger) byReceipt(id string) (*mutationCommitRecord, bool l.mu.Lock() defer l.mu.Unlock() record, ok := l.byID[id] + if !ok || !record.isPhysicalCommit() { + return nil, false + } return record, ok } @@ -473,6 +577,9 @@ func (l *mutationCommitLedger) byMutationID(key string) (*mutationCommitRecord, l.mu.Lock() defer l.mu.Unlock() record, ok := l.byKey[key] + if !ok || !record.isPhysicalCommit() { + return nil, false + } return record, ok } @@ -480,6 +587,9 @@ func (l *mutationCommitLedger) byFreshnessReceipt(id string) (*mutationCommitRec l.mu.Lock() defer l.mu.Unlock() record, ok := l.byReindexReceipt[id] + if !ok || !record.isPhysicalCommit() { + return nil, false + } return record, ok } @@ -491,6 +601,9 @@ func (l *mutationCommitLedger) recentForPath(relPath string) (*mutationCommitRec defer l.mu.Unlock() for i := len(l.ordered) - 1; i >= 0; i-- { record := l.ordered[i] + if !record.isPhysicalCommit() { + continue + } record.mu.RLock() match := record.relPath == relPath || record.absPath == relPath record.mu.RUnlock() @@ -509,7 +622,9 @@ func (l *mutationCommitLedger) recent(limit int) []*mutationCommitRecord { } out := make([]*mutationCommitRecord, 0, limit) for i := len(l.ordered) - 1; i >= 0 && len(out) < limit; i-- { - out = append(out, l.ordered[i]) + if l.ordered[i].isPhysicalCommit() { + out = append(out, l.ordered[i]) + } } return out } @@ -535,7 +650,7 @@ func mutationCommitNoteFrom(ctx context.Context) *mutationCommitNote { } func (n *mutationCommitNote) observe(record *mutationCommitRecord) { - if n == nil || record == nil { + if n == nil || !record.isPhysicalCommit() { return } n.mu.Lock() @@ -594,8 +709,9 @@ func (s *Server) beginMutationCommit(ctx context.Context, tool, mutationID, fing disk: mutationDiskInFlight, // Seed, not an outcome: graphRecorded stays false until something // reports back, and that is what makes the difference visible. - graph: mutationGraphStale, - startedAt: time.Now(), + graph: mutationGraphStale, + startedAt: time.Now(), + acknowledgement: editAcknowledgementFrom(ctx, s), } if state := checkoutMutationFromContext(ctx); state != nil { record.checkoutID = state.checkoutID @@ -656,8 +772,14 @@ func (s *Server) commitFileMutation( record.markNotApplied(err) return record, fmt.Errorf("%w: %w", errMutationNotApplied, err) } + record.mu.Lock() + record.diskWriteStartedAt = time.Now() + record.mu.Unlock() indexer.StampPublicationPhase(ctx, indexer.PublicationDiskWriteStarted) - if err := agents.AtomicWriteFile(absPath, data, perm); err != nil { + diskLap := s.beginAcknowledgementLap(ctx, editAckDiskWrite) + writeErr := agents.AtomicWriteFile(absPath, data, perm) + diskLap.finish() // actual call end, before committed-hash/receipt bookkeeping + if err := writeErr; err != nil { record.markFailed(err) return record, err } @@ -709,7 +831,7 @@ func mutationNotAppliedMessage(what string, record *mutationCommitRecord, err er // attachMutationCommit puts the disk half of the receipt on a success payload. // The graph half rides on the same response via attachMutationFreshness. func attachMutationCommit(resp map[string]any, record *mutationCommitRecord) { - if record == nil { + if !record.isPhysicalCommit() { return } resp["mutation_receipt"] = record.id diff --git a/internal/mcp/mutation_phase_record.go b/internal/mcp/mutation_phase_record.go new file mode 100644 index 000000000..66f3edd07 --- /dev/null +++ b/internal/mcp/mutation_phase_record.go @@ -0,0 +1,117 @@ +package mcp + +import ( + "context" + "time" + + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/pathkey" +) + +// committedPublicationRecord opens one timeline for an actual single-file +// commit, after disk success but before refresh admission. The commit note may +// contain several writes; choose the latest committed version of this path, +// never the first request-wide disk stamp or an earlier file's record. +// Atomic batch transactions have a separate durable lifecycle and do not enter +// this helper. No filesystem/store observation is added here. +func (s *Server) committedPublicationRecord(ctx context.Context, path string) *indexer.PublicationPhaseRecord { + note := mutationCommitNoteFrom(ctx) + if note == nil { + return nil + } + note.mu.Lock() + records := append([]*mutationCommitRecord(nil), note.records...) + note.mu.Unlock() + var selected *mutationCommitRecord + var sequence uint64 + for _, record := range records { + record.mu.RLock() + match := record.disk == mutationDiskCommitted && record.commitSeq > sequence && pathkey.EqualPaths(record.absPath, path) + if match { + selected, sequence = record, record.commitSeq + } + record.mu.RUnlock() + } + if selected == nil { + return nil + } + return s.openCommittedPublicationRecord(ctx, selected) +} + +func (s *Server) openCommittedPublicationRecord(ctx context.Context, record *mutationCommitRecord) *indexer.PublicationPhaseRecord { + if !record.isPhysicalCommit() { + return nil + } + return s.openOwnedPublicationRecord(ctx, record) +} + +// The batch's distinct observation kind can share the bounded recorder without +// gaining physical mutation authority or entering committedPublicationRecord. +func (s *Server) openOwnedPublicationRecord(ctx context.Context, record *mutationCommitRecord) *indexer.PublicationPhaseRecord { + if !record.isPublicationOwner() { + return nil + } + ledger := &s.mutationCommits + ledger.mu.Lock() + record.mu.RLock() + owned := ledger.byID[record.id] == record + existing := record.publication + started, committed, checkoutID, key := record.diskWriteStartedAt, record.committedAt, record.checkoutID, record.id + committedVerdict := record.kind != batchFileObservationCommit || record.disk == mutationDiskCommitted + record.mu.RUnlock() + ledger.mu.Unlock() + if !owned || !committedVerdict || started.IsZero() || committed.IsZero() { + return nil + } + if existing != nil { + return existing + } + origin := toolReceivedAt(ctx) + if origin.IsZero() { + origin = started + } + publication := indexer.DefaultPublicationPhases().Begin(checkoutID, key, indexer.PublicationSourceMCPEdit, origin) + // Stamp this physical file's boundaries before absorbing the shared + // request timeline. Its first file's disk marks must not win for this one. + publication.MarkAt(indexer.PublicationDiskWriteStarted, started) + publication.MarkAt(indexer.PublicationReceiptCommitted, committed) + publication.Absorb(indexer.PublicationStampsFrom(ctx)) + + // Pending-source evidence can retain a record after ledger eviction. Do + // not resurrect its timing pointer: completed timing retention is bounded + // by the existing ledger/recorder, not that evidence's longer lifetime. + ledger.mu.Lock() + record.mu.Lock() + if ledger.byID[record.id] == record && record.publication == nil { + record.publication = publication + } + selected := record.publication + record.mu.Unlock() + ledger.mu.Unlock() + return selected +} + +func completeCanonicalPublication(record *indexer.PublicationPhaseRecord, result indexer.MutationResult) { + if record == nil { + return + } + // A coalesced newer application can complete the older requested ticket. + // Keep its original binding; canonical AppliedGeneration is a watcher + // sequence, not a checkout dirty-generation ID or a new per-edit patch span. + if result.Err == nil && result.Reindexed { + record.Mark(indexer.PublicationTicketCompleted) + } else { + record.Mark(indexer.PublicationTicketFailed) + } +} + +// retainedMutationPublication is status-only. Holding the pointer in one reply +// does not extend its registry lifetime or mutate the record's binding. +func retainedMutationPublication(record *mutationCommitRecord) (*indexer.PublicationPhaseRecord, time.Time) { + if !record.isPhysicalCommit() { + return nil, time.Time{} + } + record.mu.RLock() + defer record.mu.RUnlock() + return record.publication, record.committedAt +} diff --git a/internal/mcp/mutation_phase_record_test.go b/internal/mcp/mutation_phase_record_test.go new file mode 100644 index 000000000..d188c55f9 --- /dev/null +++ b/internal/mcp/mutation_phase_record_test.go @@ -0,0 +1,239 @@ +package mcp + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/indexer" +) + +// The indexer tests exercise the real bind-before-wake critical section. This +// adapter deterministically makes completion available before admission returns, +// exercising the MCP side without depending on a debounce scheduling race. +type canonicalPublicationWatcher struct { + mutationTestWatcher + hold bool + records []*indexer.PublicationPhaseRecord + snapshots []indexer.PublicationPhaseSnapshot + results []chan indexer.MutationResult +} + +func (w *canonicalPublicationWatcher) EnqueueFileMutation(ctx context.Context, path string) (*indexer.MutationTicket, error) { + if w.scheduleErr != nil { + return nil, w.scheduleErr + } + generation := uint64(len(w.records) + 1) + record := indexer.PublicationRecordFromContext(ctx) + if record != nil { + record.BindTicket(generation) + record.Mark(indexer.PublicationTicketEnqueued) + w.snapshots = append(w.snapshots, record.Snapshot()) + } + done := make(chan indexer.MutationResult, 1) + w.records = append(w.records, record) + w.results = append(w.results, done) + if !w.hold { + done <- indexer.MutationResult{RequestedGeneration: generation, AppliedGeneration: generation, Reindexed: true} + close(done) + } + return &indexer.MutationTicket{Path: path, Generation: generation, Done: done, PublicationRecord: record}, nil +} + +func canonicalTimingContext() context.Context { + ctx := withToolReceivedAt(context.Background(), time.Now()) + ctx, _ = withMutationCommitNote(ctx) + return indexer.WithPublicationStamps(ctx) +} + +func commitTimingFixture(t *testing.T, s *Server, ctx context.Context, path, content string) *mutationCommitRecord { + t.Helper() + record, err := s.commitFileMutation(ctx, "write_file", "", "", filepath.Base(path), path, []byte(content), 0o600) + require.NoError(t, err) + return record +} + +func TestCanonicalCommitImmediateSuccessKeepsPublication(t *testing.T) { + s := newMutationServer() + w := &canonicalPublicationWatcher{} + s.watcher = w + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + record := commitTimingFixture(t, s, ctx, path, "package edited\n") + outcome := s.mutationReindexState(ctx, path) + require.True(t, outcome.Reindexed) + require.False(t, outcome.Pending) + require.Empty(t, outcome.Receipt, "keep the existing immediate-success public contract") + record.recordGraph(outcome) + require.Len(t, w.snapshots, 1) + require.EqualValues(t, 1, w.snapshots[0].Ticket) + require.Contains(t, publicationOffsets(w.snapshots[0]), indexer.PublicationReceiptCommitted) + require.Contains(t, publicationOffsets(w.snapshots[0]), indexer.PublicationTicketEnqueued) + status := s.mutationStatusPayload(record) + publication := status["publication_phases"].(indexer.PublicationPhaseSnapshot) + require.True(t, publication.Terminal) + require.EqualValues(t, 1, publication.Ticket) + require.Zero(t, publication.DirtyGenerationID, "a watcher sequence is not a checkout view ID") + require.Equal(t, "measured", status["phase_states"].(map[string]string)["disk_write"]) + require.Equal(t, "measured", status["phase_states"].(map[string]string)["publish"]) + require.Empty(t, record.snapshot().ReindexReceipt) +} + +func TestCanonicalCommitsKeepPerFileAndVersionTiming(t *testing.T) { + s := newMutationServer() + w := &canonicalPublicationWatcher{} + s.watcher = w + ctx := canonicalTimingContext() + root := t.TempDir() + firstPath, secondPath := filepath.Join(root, "first.go"), filepath.Join(root, "second.go") + first := commitTimingFixture(t, s, ctx, firstPath, "package first\n") + first.recordGraph(s.mutationReindexState(ctx, firstPath)) + before := w.records[0].Snapshot() + second := commitTimingFixture(t, s, ctx, secondPath, "package second\n") + second.recordGraph(s.mutationReindexState(ctx, secondPath)) + newer := commitTimingFixture(t, s, ctx, firstPath, "package newer\n") + newer.recordGraph(s.mutationReindexState(ctx, firstPath)) + require.Len(t, w.records, 3) + require.NotSame(t, w.records[0], w.records[1]) + require.NotSame(t, w.records[0], w.records[2]) + require.Equal(t, before, w.records[0].Snapshot(), "the newer file/version must not extend or retarget the first receipt") + for i, commit := range []*mutationCommitRecord{first, second, newer} { + publication := w.records[i].Snapshot() + commit.mu.RLock() + started := commit.diskWriteStartedAt + commit.mu.RUnlock() + require.Equal(t, started.Sub(toolReceivedAt(ctx)).Nanoseconds(), publicationOffsets(publication)[indexer.PublicationDiskWriteStarted], + "a request's first disk stamp must not be reused for another physical write") + require.EqualValues(t, i+1, publication.Ticket) + } +} + +func TestCanonicalCoalescedCompletionKeepsOriginalReceiptIdentity(t *testing.T) { + s := newMutationServer() + s.mutationReindexWait = time.Nanosecond + w := &canonicalPublicationWatcher{hold: true} + s.watcher = w + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + for _, source := range []string{"package older\n", "package newer\n"} { + record := commitTimingFixture(t, s, ctx, path, source) + outcome := s.mutationReindexState(ctx, path) + require.True(t, outcome.Pending) + record.recordGraph(outcome) + } + for i, done := range w.results { + done <- indexer.MutationResult{RequestedGeneration: uint64(i + 1), AppliedGeneration: 2, Reindexed: true} + close(done) + id := w.records[i].Key() + value, found := s.mutationReceipts.Load(id) + require.True(t, found) + select { + case <-value.(*mutationReceipt).done: + case <-time.After(5 * time.Second): + t.Fatal("the existing receipt consumer did not finish") + } + publication := w.records[i].Snapshot() + require.True(t, publication.Terminal) + require.EqualValues(t, i+1, publication.Ticket) + require.Zero(t, publication.DirtyGenerationID) + require.NotContains(t, publicationOffsets(publication), indexer.PublicationCycleStarted, "do not invent a per-receipt patch interval from a shared completion") + } +} + +func TestCommittedPublicationRetentionEndsWithLedgerOwnership(t *testing.T) { + for _, eviction := range []string{"expiry", "capacity"} { + t.Run(eviction, func(t *testing.T) { + s := newMutationServer() + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + record := commitTimingFixture(t, s, ctx, path, "package retained\n") + publication := s.committedPublicationRecord(ctx, path) + require.NotNil(t, publication) + if eviction == "expiry" { + record.mu.Lock() + record.startedAt = time.Now().Add(-mutationCommitRetention - time.Second) + record.mu.Unlock() + s.mutationCommits.mu.Lock() + s.mutationCommits.evictLocked(time.Now()) + s.mutationCommits.mu.Unlock() + } else { + for i := range maxMutationCommits { + s.mutationCommits.put(&mutationCommitRecord{id: fmt.Sprintf("timing-cap-%d", i), startedAt: time.Now()}) + } + } + _, present := s.mutationCommits.byReceipt(record.id) + require.False(t, present) + retained, _ := retainedMutationPublication(record) + require.Nil(t, retained, "source evidence or the commit note may still hold this object, but not its timing history") + require.Nil(t, s.committedPublicationRecord(ctx, path), "a late callback must not resurrect evicted timing retention") + require.Equal(t, mutationDiskCommitted, record.snapshot().DiskStatus, "timing eviction must not change the disk verdict") + }) + } +} + +func TestReboundCommitDoesNotBorrowReplacementPublication(t *testing.T) { + s := newMutationServer() + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + record := commitTimingFixture(t, s, ctx, path, "package edited\n") + original := s.committedPublicationRecord(ctx, path) + require.NotNil(t, original) + newID := "replacement-" + record.id + replacement := indexer.DefaultPublicationPhases().Begin("", newID, indexer.PublicationSourceMCPEdit, time.Now()) + replacement.Mark(indexer.PublicationTicketEnqueued) + s.mutationReceipts.Store(newID, &mutationReceipt{id: newID, generation: 2, done: make(chan struct{})}) + record.recordGraph(mutationReindexOutcome{Pending: true, Receipt: newID, Generation: 2}) + original.Mark(indexer.PublicationTicketCompleted) + status := s.mutationStatusPayload(record) + require.Equal(t, mutationGraphPending, status["graph_status"]) + require.NotContains(t, status, "publication_phases", "do not substitute another receipt's timeline for this physical commit") + for name, value := range status["phases_ms"].(map[string]*float64) { + require.Nil(t, value, name) + } + retained, _ := retainedMutationPublication(record) + require.Same(t, original, retained, "the historical record itself is never retargeted") +} + +func TestCanonicalAdmissionFailureClosesItsCommittedRecord(t *testing.T) { + s := newMutationServer() + sentinel := errors.New("admission stopped") + s.watcher = &canonicalPublicationWatcher{mutationTestWatcher: mutationTestWatcher{scheduleErr: sentinel}} + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + record := commitTimingFixture(t, s, ctx, path, "package committed\n") + outcome := s.mutationReindexState(ctx, path) + require.ErrorIs(t, outcome.Err, sentinel) + record.recordGraph(outcome) + status := s.mutationStatusPayload(record) + require.Equal(t, mutationDiskCommitted, status["disk_status"]) + require.Equal(t, mutationGraphFailed, status["graph_status"]) + publication := status["publication_phases"].(indexer.PublicationPhaseSnapshot) + require.True(t, publication.Terminal) + require.Zero(t, publication.Ticket, "no ticket was admitted") + require.Contains(t, publicationOffsets(publication), indexer.PublicationTicketFailed) + require.Nil(t, status["phases_ms"].(map[string]*float64)["publish"]) +} + +func TestCanonicalTimingSurvivesBoundedRecorderHistoryEviction(t *testing.T) { + s := newMutationServer() + s.watcher = &canonicalPublicationWatcher{} + ctx := canonicalTimingContext() + path := filepath.Join(t.TempDir(), "edit.go") + record := commitTimingFixture(t, s, ctx, path, "package edited\n") + record.recordGraph(s.mutationReindexState(ctx, path)) + publication, _ := retainedMutationPublication(record) + require.NotNil(t, publication) + for i := range 40 { + indexer.DefaultPublicationPhases().Begin("", fmt.Sprintf("recorder-cap-%s-%d", record.id, i), indexer.PublicationSourceMCPEdit, time.Now()) + } + _, found := indexer.DefaultPublicationPhases().Lookup(publication.Key()) + require.False(t, found, "the recorder's independent count bound should have evicted the original entry") + status := s.mutationStatusPayload(record) + require.Equal(t, publication.Key(), status["publication_phases"].(indexer.PublicationPhaseSnapshot).Key) + require.Equal(t, "measured", status["phase_states"].(map[string]string)["disk_write"]) +} diff --git a/internal/mcp/mutation_phase_timings_test.go b/internal/mcp/mutation_phase_timings_test.go new file mode 100644 index 000000000..5016e6970 --- /dev/null +++ b/internal/mcp/mutation_phase_timings_test.go @@ -0,0 +1,83 @@ +package mcp + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/indexer" +) + +func TestMutationStatusWithoutTimelineKeepsMissingTimingNull(t *testing.T) { + s := newMutationServer() + record := s.beginMutationCommit(context.Background(), "edit_file", "", "fp", "edit.go", filepath.Join(t.TempDir(), "edit.go")) + payload := s.mutationStatusPayload(record) + require.NotContains(t, payload, "publication_phases", "do not fabricate an old timeline for a canonical/in-flight receipt") + encoded, err := json.Marshal(payload) + require.NoError(t, err) + var wire map[string]any + require.NoError(t, json.Unmarshal(encoded, &wire)) + phases := wire["phases_ms"].(map[string]any) + states := wire["phase_states"].(map[string]any) + require.Len(t, phases, 19) // original15 plus two admissions, leaf control and caller-result finalization + require.Contains(t, phases, "cohort_admission") + require.Contains(t, phases, "batch_admission") + require.Contains(t, phases, "edit_control") + require.Contains(t, phases, "caller_result") + for name, ms := range phases { + require.Nil(t, ms, name) + require.Equal(t, "not_observed", states[name]) + } + window := wire["phase_timing"].(map[string]any) + require.Equal(t, "unavailable", window["clock"]) + require.Equal(t, "partial", window["coverage"]) + require.Nil(t, window["ack_wall_ms"]) + require.Nil(t, window["publication_wall_ms"]) +} + +func TestMutationStatusTimingAndLegacyTimelineShareSnapshot(t *testing.T) { + s := newMutationServer() + record := s.beginMutationCommit(context.Background(), "write_file", "", "fp", "edit.go", filepath.Join(t.TempDir(), "edit.go")) + record.markCommitted("sha", 10) + id := "timed-" + record.id + ticket := &mutationReceipt{id: id, done: make(chan struct{}), generation: 7} + s.mutationReceipts.Store(id, ticket) + record.recordGraph(mutationReindexOutcome{Pending: true, Receipt: id, Generation: 7}) + origin := time.Now() + timeline := indexer.DefaultPublicationPhases().Begin("timing-"+record.id, id, indexer.PublicationSourceMCPEdit, origin) + timeline.MarkAt(indexer.PublicationDiskWriteStarted, origin.Add(time.Millisecond)) + timeline.MarkAt(indexer.PublicationReceiptCommitted, origin.Add(2*time.Millisecond)) + timeline.MarkAt(indexer.PublicationTicketEnqueued, origin.Add(4*time.Millisecond)) + before := timeline.Snapshot() + payload := s.mutationStatusPayload(record) + require.Equal(t, before, payload["publication_phases"], "keep the old snapshot type and contents") + phases := payload["phases_ms"].(map[string]*float64) + require.Equal(t, 1.0, *phases["disk_write"]) + require.Equal(t, 2.0, *phases["publish"]) + require.Nil(t, payload["phase_timing"].(indexer.EditTimingWindow).PublicationWallMS) + timeline.MarkAt(indexer.PublicationTicketCompleted, origin.Add(10*time.Millisecond)) + final := s.mutationStatusPayload(record) + require.Equal(t, "partial", final["phase_timing"].(indexer.EditTimingWindow).Coverage) + require.Equal(t, 10.0, *final["phase_timing"].(indexer.EditTimingWindow).PublicationWallMS) + require.Nil(t, payload["phase_timing"].(indexer.EditTimingWindow).PublicationWallMS, "an earlier response owns an immutable snapshot") +} + +func TestGraphOnlyReceiptTimingDoesNotInventAcknowledgement(t *testing.T) { + s := newMutationServer() + ticket := &mutationReceipt{id: "refresh-" + t.Name(), done: make(chan struct{})} + s.mutationReceipts.Store(ticket.id, ticket) + payload, found := s.graphRefreshReceiptPayload(context.Background(), ticket.id) + require.True(t, found) + states := payload["phase_states"].(map[string]string) + phases := payload["phases_ms"].(map[string]*float64) + for _, name := range []string{"dispatcher_queue", "middleware", "parse", "disk_write", "publish", "reindex_wait"} { + require.Equal(t, "not_applicable", states[name]) + require.Nil(t, phases[name]) + } + require.Equal(t, "not_observed", states["patch_parse"]) + require.NotContains(t, payload, "publication_phases") +} diff --git a/internal/mcp/overlay.go b/internal/mcp/overlay.go index d2b4d64e5..09ec77aec 100644 --- a/internal/mcp/overlay.go +++ b/internal/mcp/overlay.go @@ -96,6 +96,7 @@ func (s *Server) wrapToolHandlerMode(h mcpserver.ToolHandlerFunc, injectOverlay // leaf handler — the overlay build, the freshness sweep, and the response // capture below all touch the graph and can block on the same locks. return s.boundToolHandler(s.retryFreshSymbolSearch(func(ctx context.Context, req mcp.CallToolRequest) (res *mcp.CallToolResult, retErr error) { + s.noteAcknowledgementMiddleware(ctx) // The instant the call entered the middleware: the origin a mutation's // or a fresh request's publication phases are measured from. ctx = withToolReceivedAt(ctx, time.Now()) @@ -353,7 +354,7 @@ func (s *Server) wrapToolHandlerMode(h mcpserver.ToolHandlerFunc, injectOverlay ctx = withContractCoreReadErrors(ctx) } indexer.StampPublicationPhase(ctx, indexer.PublicationHandlerStarted) - res, hErr := h(ctx, req) + res, hErr := s.invokeAcknowledgedLeaf(ctx, req, h) if hErr == nil && res != nil && !res.IsError && lateExactnessRefusalIsSafe(req.Params.Name) { if err := contractCoreReadError(ctx); err != nil { res = mcp.NewToolResultError(err.Error()) diff --git a/internal/mcp/overlay_view_request_test.go b/internal/mcp/overlay_view_request_test.go index e29753dbd..e50d5e490 100644 --- a/internal/mcp/overlay_view_request_test.go +++ b/internal/mcp/overlay_view_request_test.go @@ -86,8 +86,7 @@ func newOverlayRequestFixture(t *testing.T) overlayRequestFixture { g := graph.New() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, testRegistry(), bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) diff --git a/internal/mcp/overlay_workspace_stamp_test.go b/internal/mcp/overlay_workspace_stamp_test.go index 084c86495..083ba59fe 100644 --- a/internal/mcp/overlay_workspace_stamp_test.go +++ b/internal/mcp/overlay_workspace_stamp_test.go @@ -64,8 +64,7 @@ func newOverlayStampFixture(t *testing.T) stampFixture { g := graph.New() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, testRegistry(), bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) diff --git a/internal/mcp/path_indexability_test.go b/internal/mcp/path_indexability_test.go index c77a59582..2ab93da4c 100644 --- a/internal/mcp/path_indexability_test.go +++ b/internal/mcp/path_indexability_test.go @@ -1,6 +1,7 @@ package mcp import ( + "context" "os" "path/filepath" "testing" @@ -151,8 +152,12 @@ func newPrefixFixture(t *testing.T) (*Server, string, string) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // These fixtures test path ownership/freshness, not cross-repo batch + // resolution. Register their explicit roots through the production path. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } return &Server{multiIndexer: mi}, repoA, repoB } diff --git a/internal/mcp/pending_source.go b/internal/mcp/pending_source.go index 1e4d2f02f..5aca021ae 100644 --- a/internal/mcp/pending_source.go +++ b/internal/mcp/pending_source.go @@ -29,6 +29,9 @@ type pendingSourceKey struct { } func (p *pendingSourceRegistry) register(record *mutationCommitRecord) { + if !record.isPhysicalCommit() { + return + } p.mu.Lock() defer p.mu.Unlock() if p.versions == nil { @@ -119,6 +122,9 @@ func (p *pendingSourceRegistry) paths(checkoutID, incarnation string) []string { // Resolve a late ticket immediately when the commit binds its receipt. The // receipt can finish before the mutation handler records its pending outcome. func (s *Server) refreshPendingSourceRecord(record *mutationCommitRecord) { + if !record.isPhysicalCommit() { + return + } defer s.retireSettledPendingSource(record) snap := record.snapshot() if snap.GraphStatus == mutationGraphFresh || snap.ReindexReceipt == "" { @@ -153,6 +159,9 @@ func (s *Server) applyPendingSourceReceipt(record *mutationCommitRecord, snap mu } func (s *Server) retireSettledPendingSource(record *mutationCommitRecord) { + if !record.isPhysicalCommit() { + return + } record.mu.RLock() retire := record.disk == mutationDiskNotApplied || record.disk == mutationDiskFailed || (record.graphRecorded && (record.graph == mutationGraphFresh || (record.checkoutID == "" && record.graph == mutationGraphFailed))) diff --git a/internal/mcp/read_repair_observation.go b/internal/mcp/read_repair_observation.go new file mode 100644 index 000000000..f196bd12a --- /dev/null +++ b/internal/mcp/read_repair_observation.go @@ -0,0 +1,295 @@ +package mcp + +import ( + "context" + "time" + "weak" + + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/indexer" +) + +// One immutable, independent coordinator execution. This is neither the +// original receipt's P nor B: read repair can overlap that receipt's quiet +// wait, and adding these durations would count the same time twice. Retention +// belongs only to the existing mutation ledger, independently of the original +// publication recorder's index. That pointer is an incarnation witness only. +type mutationReadRepairObservation struct { + generation uint64 + windowNS int64 + causes [8]mutationReadRepairCause + causeCount int + accounting indexer.ReadRepairPhaseAccounting + // Frozen admission identity survives that existing receipt's shorter TTL; + // accepted R is retained by the mutation ledger, not by the receipt map. + requestedReceipt string + requestedGeneration uint64 + acceptedBlobSHA string +} + +type mutationReadRepairCause struct { + Session string `json:"session,omitempty"` + RequestIDKey string `json:"request_id_key,omitempty"` + AdmissionGeneration uint64 `json:"admission_generation"` +} + +func (o mutationReadRepairObservation) payload() map[string]any { + causes := make([]mutationReadRepairCause, o.causeCount) + copy(causes, o.causes[:o.causeCount]) + return map[string]any{ + "scope": "read_assisted_reconcile", "clock": "monotonic", + "owner_generation": o.generation, "window_ns": o.windowNS, + "reader_causes": causes, "additive_with_requested_background": false, + "background_relationship": "unavailable", + "phase_accounting": o.accounting.Payload(), + "requested_receipt": o.requestedReceipt, + "requested_generation": o.requestedGeneration, + "accepted_blob_sha": o.acceptedBlobSHA, + } +} + +// A witness retains only immutable identities and weak owners during the +// existing work lifetime. It cannot keep an evicted receipt or Server alive. +type mutationReadRepairWitness struct { + server weak.Pointer[Server] + record weak.Pointer[mutationCommitRecord] + indexer weak.Pointer[indexer.Indexer] + publication weak.Pointer[indexer.PublicationPhaseRecord] + admission weak.Pointer[mutationReceipt] + id string + fingerprint string + absPath string + blobSHA string + receipt string + checkout string + incarnation string + commitSeq uint64 + generation uint64 + committedAt time.Time +} + +func (s *Server) readRepairObserver(ctx context.Context, absPath string, idx *indexer.Indexer) *indexer.ReadRepairObserver { + if observer, moveOwner := s.moveReadRepairObserver(ctx, absPath, idx); moveOwner { + return observer + } + witness, ok := s.captureReadRepairWitness(absPath, idx) + if !ok { + return nil + } + key, _ := daemon.MCPRequestIDKeyForContext(ctx) + return &indexer.ReadRepairObserver{Session: SessionIDFromContext(ctx), RequestIDKey: key, Complete: witness.complete} +} + +// Bind the actual existing ticket before waiting, without changing public +// pending/result fields. All receipt identity fields read here are immutable. +// No publication/recorder methods or mutable receipt state are read under the +// existing ledger -> record locks. +func (s *Server) bindReadRepairAdmission(absPath string, publication *indexer.PublicationPhaseRecord, admission *mutationReceipt) { + if publication == nil || admission == nil || admission.generation == 0 || admission.checkoutScoped || admission.path != absPath { + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + for i := len(ledger.ordered) - 1; i >= 0; i-- { + record := ledger.ordered[i] + if !record.isPublicationOwner() { + continue + } + record.mu.Lock() + match := record.absPath == absPath + if match && ledger.byID[record.id] == record && record.owner == s && record.disk == mutationDiskCommitted && + record.commitSeq != 0 && record.newSHA != "" && record.publication == publication && + record.readRepairAdmission == (weak.Pointer[mutationReceipt]{}) { + record.readRepairAdmission = weak.Make(admission) + } + record.mu.Unlock() + if match { + return + } + } +} + +func (s *Server) captureReadRepairWitness(absPath string, idx *indexer.Indexer) (mutationReadRepairWitness, bool) { + return s.readRepairWitness(absPath, idx, true) +} + +func (s *Server) readRepairWitness(absPath string, idx *indexer.Indexer, pendingOnly bool) (mutationReadRepairWitness, bool) { + if idx == nil { + return mutationReadRepairWitness{}, false + } + ledger := &s.mutationCommits + ledger.mu.Lock() + var witness mutationReadRepairWitness + batch := false + for i := len(ledger.ordered) - 1; i >= 0; i-- { + record := ledger.ordered[i] + if !record.isPublicationOwner() { + continue + } + record.mu.RLock() + if record.absPath != absPath { + record.mu.RUnlock() + continue + } + // The newest file publication owner is authoritative even if it is not yet + // eligible. Never fall back to an older edit of the same path. + committed := record.disk == mutationDiskCommitted && record.commitSeq != 0 && record.newSHA != "" + writing := pendingOnly && record.disk == mutationDiskInFlight + // Capture may precede final batch durability and ticket admission, but + // only after the real forward writer has returned these known bytes. + // This is a weak candidate, never a committed source claim. Completion + // still requires this exact record's successful transaction verdict, + // actual accepted bytes and original publication/admission identity. + batchReady := record.kind != batchFileObservationCommit || (committed && record.publication != nil && record.readRepairAdmission.Value() != nil) + if pendingOnly && record.kind == batchFileObservationCommit && !record.committedAt.IsZero() && record.newSHA != "" && (committed || writing) { + batchReady = true + } + if batchReady && ledger.byID[record.id] == record && record.owner == s && (committed || writing) && record.readRepair == nil && + (!pendingOnly || !record.graphRecorded || record.graph == mutationGraphPending) { + witness = mutationReadRepairWitness{ + server: weak.Make(s), record: weak.Make(record), indexer: weak.Make(idx), + publication: weak.Make(record.publication), id: record.id, fingerprint: record.fingerprint, + absPath: absPath, blobSHA: record.newSHA, + checkout: record.checkoutID, incarnation: record.checkoutIncarnation, + commitSeq: record.commitSeq, committedAt: record.committedAt, + } + batch = record.kind == batchFileObservationCommit + if admission := record.readRepairAdmission.Value(); admission != nil && admission.generation != 0 && admission.path == absPath && !admission.checkoutScoped { + witness.admission = weak.Make(admission) + witness.receipt, witness.generation = admission.id, admission.generation + } + } + record.mu.RUnlock() + break + } + ledger.mu.Unlock() + // Observation-only owners do not participate in physical recordGraph or + // pending-source lifecycle updates. Read the actual admission outside ledger + // locks rather than treating its old pending seed as an endless eligibility. + if pendingOnly && batch { + if admission := witness.admission.Value(); admission != nil && !admission.outcome(true).Pending { + return mutationReadRepairWitness{}, false + } + } + return witness, witness.id != "" +} + +func (w mutationReadRepairWitness) complete(observation indexer.ReadRepairObservation) { + s, record, idx := w.server.Value(), w.record.Value(), w.indexer.Value() + if s == nil || record == nil || idx == nil { + return + } + ready, ok := w.completionWitness(s, record, idx) + if !ok || !observation.AcceptedFor(idx, ready.blobSHA) { + return + } + started, finished := observation.Bounds() + if ready.committedAt.IsZero() || observation.AcceptedAt().Before(ready.committedAt) { + return + } + w = ready + accepted := mutationReadRepairObservation{generation: observation.Generation(), windowNS: finished.Sub(started).Nanoseconds(), accounting: observation.PhaseAccounting()} + for i := range accepted.causes { + session, key, generation, ok := observation.Cause(i) + if !ok { + break + } + accepted.causes[i] = mutationReadRepairCause{Session: session, RequestIDKey: key, AdmissionGeneration: generation} + accepted.causeCount++ + } + if _, _, _, more := observation.Cause(len(accepted.causes)); more { + return // Never silently truncate a future larger producer bound. + } + w.bind(accepted) +} + +func (w mutationReadRepairWitness) completionWitness(s *Server, record *mutationCommitRecord, idx *indexer.Indexer) (mutationReadRepairWitness, bool) { + // A reader can observe the committed write before the editor has opened + // its publication and enqueued its watcher ticket. Resolve only that same + // retained incarnation now; never borrow a later edit's admission. + ready, ok := s.readRepairWitness(w.absPath, idx, false) + if !ok || ready.record.Value() != record || ready.id != w.id || ready.fingerprint != w.fingerprint || + (w.commitSeq != 0 && ready.commitSeq != w.commitSeq) || (w.blobSHA != "" && ready.blobSHA != w.blobSHA) || + (!w.committedAt.IsZero() && ready.committedAt != w.committedAt) || + ready.checkout != w.checkout || ready.incarnation != w.incarnation || + ready.publication.Value() == nil || ready.admission.Value() == nil || ready.generation == 0 { + return mutationReadRepairWitness{}, false + } + if (w.publication != (weak.Pointer[indexer.PublicationPhaseRecord]{}) && ready.publication != w.publication) || + (w.admission != (weak.Pointer[mutationReceipt]{}) && ready.admission != w.admission) || + (w.receipt != "" && ready.receipt != w.receipt) || (w.generation != 0 && ready.generation != w.generation) { + return mutationReadRepairWitness{}, false + } + return ready, true +} + +// bind is the acceptance fence after the execution's content/owner proof. It +// performs no recorder callbacks or registry lookups under ledger locks. +func (w mutationReadRepairWitness) bind(accepted mutationReadRepairObservation) { + s, record := w.server.Value(), w.record.Value() + if s == nil || record == nil { + return + } + ledger := &s.mutationCommits + ledger.mu.Lock() + defer ledger.mu.Unlock() + if ledger.byID[w.id] != record { + return + } + // Supersession before acceptance refuses the link. Once accepted, later + // edits do not erase this historical execution or mutate earlier snapshots. + for i := len(ledger.ordered) - 1; i >= 0; i-- { + candidate := ledger.ordered[i] + if !candidate.isPublicationOwner() { + continue + } + candidate.mu.RLock() + match := candidate.absPath == w.absPath + candidate.mu.RUnlock() + if match { + if candidate != record { + return + } + break + } + } + record.mu.Lock() + defer record.mu.Unlock() + if record.owner != s || record.id != w.id || record.commitSeq != w.commitSeq || record.newSHA != w.blobSHA || + record.absPath != w.absPath || record.fingerprint != w.fingerprint || record.disk != mutationDiskCommitted || + (record.reindexReceipt != "" && record.reindexReceipt != w.receipt) || + (record.reindexGeneration != 0 && record.reindexGeneration != w.generation) || + w.admission.Value() == nil || record.readRepairAdmission != w.admission || + record.checkoutID != w.checkout || record.checkoutIncarnation != w.incarnation || + record.publication == nil || record.publication != w.publication.Value() || record.readRepair != nil { + return + } + accepted.requestedReceipt, accepted.requestedGeneration = w.receipt, w.generation + accepted.acceptedBlobSHA = w.blobSHA + record.readRepair = &accepted +} + +// Only the already-needed multi-repository read repair is observed. Embedded +// and unobserved callers preserve their existing fallback and nil behavior. +func (s *Server) reindexFileForRead(ctx context.Context, absPath string, idx *indexer.Indexer) bool { + if s.multiIndexer != nil { + if prefix := s.multiIndexer.RepoForFile(absPath); prefix != "" { + result, err := s.multiIndexer.IncrementalReindexRepoObserved(prefix, []string{absPath}, s.readRepairObserver(ctx, absPath, idx)) + if incrementalReindexSucceeded(result, err) { + return true + } + } + } + // Match reindexFile's existing standalone fallback without submitting the + // multi-repository reconcile twice after a failure. + if s.indexer != nil { + if root := s.indexer.RootPath(); root != "" { + if _, ok := relativeWithinRoot(root, absPath); ok { + result, err := s.indexer.IncrementalReindexPaths(root, []string{absPath}) + return incrementalReindexSucceeded(result, err) + } + } + } + return false +} diff --git a/internal/mcp/read_repair_observation_test.go b/internal/mcp/read_repair_observation_test.go new file mode 100644 index 000000000..68eb0ca75 --- /dev/null +++ b/internal/mcp/read_repair_observation_test.go @@ -0,0 +1,467 @@ +package mcp + +import ( + "context" + "path/filepath" + "runtime" + "strconv" + "sync" + "testing" + "time" + "weak" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/search" +) + +func newReadRepairWitnessFixture(t *testing.T) (*Server, *mutationCommitRecord, mutationReadRepairWitness, *indexer.PublicationPhaseRecorder) { + t.Helper() + s := newMutationServer() + phases := indexer.NewPublicationPhaseRecorder(4, 1) + now := time.Now() + publication := phases.Begin("checkout", "publication", indexer.PublicationSourceMCPEdit, now) + record := &mutationCommitRecord{ + owner: s, id: "commit", fingerprint: "edit fingerprint", absPath: "/repo/file.go", + disk: mutationDiskCommitted, graph: mutationGraphPending, graphRecorded: true, + commitSeq: 1, newSHA: gitBlobSHA([]byte("committed bytes")), startedAt: now, committedAt: now, + reindexReceipt: "publication", reindexGeneration: 1, checkoutID: "checkout", checkoutIncarnation: "incarnation", + publication: publication, + } + s.mutationCommits.put(record) + admission := &mutationReceipt{id: record.reindexReceipt, generation: record.reindexGeneration, path: record.absPath} + s.mutationReceipts.Store(admission.id, admission) + record.readRepairAdmission = weak.Make(admission) + witness := mutationReadRepairWitness{ + server: weak.Make(s), record: weak.Make(record), publication: weak.Make(publication), + id: record.id, fingerprint: record.fingerprint, absPath: record.absPath, + blobSHA: record.newSHA, receipt: record.reindexReceipt, checkout: record.checkoutID, + incarnation: record.checkoutIncarnation, commitSeq: record.commitSeq, + generation: record.reindexGeneration, committedAt: now, + admission: weak.Make(admission), + } + return s, record, witness, phases +} + +// Reuse the existing canonical-ticket adapter, but expose its actual enqueue +// boundary so the test can hold Done through reader admission and acceptance. +type readRepairHeldWatcher struct { + canonicalPublicationWatcher + enqueued chan struct{} +} + +func (w *readRepairHeldWatcher) EnqueueFileMutation(ctx context.Context, path string) (*indexer.MutationTicket, error) { + ticket, err := w.canonicalPublicationWatcher.EnqueueFileMutation(ctx, path) + close(w.enqueued) + return ticket, err +} + +func TestReadRepairCapturedBeforeWatcherAdmissionObservesActualExecutor(t *testing.T) { + for _, inFlight := range []bool{false, true} { + name := "committed_before_enqueue" + if inFlight { + name = "in_flight_before_commit" + } + t.Run(name, func(t *testing.T) { testReadRepairBeforeWatcherAdmission(t, inFlight, false) }) + } + t.Run("execution_before_commit_refused", func(t *testing.T) { testReadRepairBeforeWatcherAdmission(t, true, true) }) +} + +func testReadRepairBeforeWatcherAdmission(t *testing.T, inFlight, earlyExecution bool) { + root := setupMiniRepo(t, "reader-admission") + path := filepath.Join(root, "main.go") + gc := &config.GlobalConfig{Repos: []config.RepoEntry{{Path: root, Name: "fixture"}}} + gc.SetConfigPath(filepath.Join(t.TempDir(), "config.yaml")) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(gc.ConfigPath()) + require.NoError(t, err) + g := graph.New() + mi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + registerMCPFixtureRepos(t, mi, cm) + idx := mi.GetIndexer("fixture") + require.NotNil(t, idx) + s := newMutationServer() + w := &readRepairHeldWatcher{canonicalPublicationWatcher: canonicalPublicationWatcher{hold: true}, enqueued: make(chan struct{})} + s.watcher = w + ctx, cancel := context.WithCancel(canonicalTimingContext()) + t.Cleanup(cancel) + after := []byte("package main\nfunc Hello() {}\nfunc AddedByReader() {}\n") + var observer *indexer.ReadRepairObserver + var early indexer.ReadRepairObservation + if inFlight { + s.mutationPreCommitHook = func(record *mutationCommitRecord) { + require.Equal(t, mutationDiskInFlight, record.diskStatus()) + observer = s.readRepairObserver(ctx, path, idx) + require.NotNil(t, observer, "capture the exact in-flight record before the real write") + if earlyExecution { + bumpFile(t, path, string(after)) + _, repairErr := mi.IncrementalReindexRepoObserved("fixture", []string{path}, &indexer.ReadRepairObserver{ + Complete: func(o indexer.ReadRepairObservation) { early = o }, + }) + require.NoError(t, repairErr) + require.True(t, early.AcceptedFor(idx, gitBlobSHA(after)), "real early executor applied these bytes") + } + } + } + record, err := s.commitFileMutation(ctx, "write_file", "logical-reader-admission", "reader admission fixture", "main.go", path, after, 0o600) + require.NoError(t, err) + require.Nil(t, record.publication) + require.Nil(t, record.readRepairAdmission.Value()) + if !inFlight { + observer = s.readRepairObserver(ctx, path, idx) + } + require.NotNil(t, observer, "committed bytes can become stale before publication/ticket binding") + finished := make(chan mutationReindexOutcome, 1) + go func() { finished <- s.mutationReindexState(ctx, path) }() + var releaseOnce sync.Once + release := func() { + releaseOnce.Do(func() { + select { + case <-w.enqueued: + w.results[0] <- indexer.MutationResult{RequestedGeneration: 1, AppliedGeneration: 1, Reindexed: true} + close(w.results[0]) + case <-time.After(5 * time.Second): + cancel() + } + }) + } + t.Cleanup(func() { + release() + cancel() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("held watcher waiter was not joined") + } + }) + require.Eventually(t, func() bool { + record.mu.RLock() + defer record.mu.RUnlock() + return record.readRepairAdmission.Value() != nil + }, 5*time.Second, time.Millisecond) + // The watcher may finish its no-op before the independent executor's + // callback. Terminal receipt state must not erase that historical owner. + record.recordGraph(mutationReindexOutcome{Generation: 1, Reindexed: true}) + if earlyExecution { + observer.Complete(early) + require.Nil(t, record.snapshot().readRepair, "same bytes do not authorize an execution predating the actual commit") + return + } + result, err := mi.IncrementalReindexRepoObserved("fixture", []string{path}, observer) + require.NoError(t, err) + require.NotNil(t, result) + require.NotEmpty(t, g.FindNodesByName("AddedByReader"), "real executor applied the committed bytes") + accepted := record.snapshot() + require.NotNil(t, accepted.readRepair, "actual executor callback must resolve the same newly bound admission") + require.Equal(t, gitBlobSHA(after), accepted.readRepair.acceptedBlobSHA) + require.Equal(t, true, accepted.readRepair.accounting.Payload()["observed_partition_complete"]) + release() + runtime.KeepAlive(idx) +} + +func TestReadRepairInFlightCandidateRequiresSuccessfulRetainedCommit(t *testing.T) { + for _, state := range []string{"in_flight", "failed", "not_applied", "evicted", "replaced", "superseded"} { + t.Run(state, func(t *testing.T) { + s := newMutationServer() + idx := &indexer.Indexer{} + record := s.beginMutationCommit(canonicalTimingContext(), "write_file", "logical", "fingerprint", "file.go", "/repo/file.go") + candidate, ok := s.captureReadRepairWitness(record.absPath, idx) + require.True(t, ok) + if state == "evicted" || state == "replaced" || state == "superseded" { + record.markCommitted(gitBlobSHA([]byte("committed bytes")), len("committed bytes")) + phases := indexer.NewPublicationPhaseRecorder(4, 1) + publication := phases.Begin("", "publication", indexer.PublicationSourceMCPEdit, time.Now()) + admission := &mutationReceipt{id: "publication", generation: 1, path: record.absPath} + s.mutationReceipts.Store(admission.id, admission) + record.mu.Lock() + record.publication = publication + record.readRepairAdmission = weak.Make(admission) + record.mu.Unlock() + _, ready := candidate.completionWitness(s, record, idx) + require.True(t, ready, "ownership refusal must start from a fully eligible committed record") + } + switch state { + case "failed": + record.markFailed(context.Canceled) + case "not_applied": + record.markNotApplied(context.Canceled) + case "evicted": + s.mutationCommits.mu.Lock() + delete(s.mutationCommits.byID, record.id) + s.mutationCommits.mu.Unlock() + case "replaced": + s.mutationCommits.mu.Lock() + s.mutationCommits.byID[record.id] = &mutationCommitRecord{} + s.mutationCommits.mu.Unlock() + case "superseded": + s.beginMutationCommit(canonicalTimingContext(), "write_file", "later", "new fingerprint", "file.go", record.absPath) + } + _, ok = candidate.completionWitness(s, record, idx) + require.False(t, ok) + require.Nil(t, record.snapshot().readRepair) + }) + } +} + +func TestReadRepairCompletionOnlyFillsPreviouslyMissingIdentity(t *testing.T) { + for _, field := range []string{"publication", "admission", "receipt", "generation"} { + t.Run(field, func(t *testing.T) { + s, record, witness, phases := newReadRepairWitnessFixture(t) + idx := &indexer.Indexer{} + record.mu.Lock() + switch field { + case "publication": + record.publication = phases.Begin("checkout", "foreign-publication", indexer.PublicationSourceMCPEdit, time.Now()) + case "admission": + other := &mutationReceipt{id: witness.receipt, generation: witness.generation, path: witness.absPath} + s.mutationReceipts.Store("replacement", other) + record.readRepairAdmission = weak.Make(other) + case "receipt": + witness.receipt = "previously-witnessed-other-receipt" + case "generation": + witness.generation++ + } + record.mu.Unlock() + _, ok := witness.completionWitness(s, record, idx) + require.False(t, ok, "completion cannot replace an already witnessed %s", field) + require.Nil(t, record.snapshot().readRepair) + runtime.KeepAlive(phases) + }) + } +} + +func TestReadRepairSeesCommittedAdmissionBeforeACKAndSurvivesSuccessClear(t *testing.T) { + s := newMutationServer() + s.mutationReindexWait = 30 * time.Second + w := &readRepairHeldWatcher{canonicalPublicationWatcher: canonicalPublicationWatcher{hold: true}, enqueued: make(chan struct{})} + s.watcher = w + ctx, cancel := context.WithCancel(canonicalTimingContext()) + t.Cleanup(cancel) + path := filepath.Join(t.TempDir(), "edit.go") + record, err := s.commitFileMutation(ctx, "write_file", "caller-logical-id", "distinct-id fixture", filepath.Base(path), path, []byte("package edited\n"), 0o600) + require.NoError(t, err) + idx := &indexer.Indexer{} + outcome := make(chan mutationReindexOutcome, 1) + finished := make(chan struct{}) + go func() { + defer close(finished) + outcome <- s.mutationReindexState(ctx, path) + }() + var releaseOnce sync.Once + release := func() { + releaseOnce.Do(func() { + select { + case <-w.enqueued: + w.results[0] <- indexer.MutationResult{RequestedGeneration: 1, AppliedGeneration: 1, Reindexed: true} + close(w.results[0]) + case <-time.After(5 * time.Second): + cancel() + } + }) + } + t.Cleanup(func() { + release() + cancel() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Error("held mutation waiter was not reaped") + } + }) + require.Eventually(t, func() bool { + witness, ok := s.captureReadRepairWitness(path, idx) + return ok && witness.generation != 0 + }, 5*time.Second, time.Millisecond, + "a pre-ACK reader must see the real admitted ticket, not post-wait metadata") + pending := record.snapshot() + require.Empty(t, pending.ReindexReceipt) + require.Zero(t, pending.ReindexGeneration, "05ab's required nonzero post-wait generation is absent at this real boundary") + require.False(t, pending.GraphRecorded) + select { + case <-outcome: + t.Fatal("Done was not held through the reader observation") + default: + } + witness, ok := s.captureReadRepairWitness(path, idx) + require.True(t, ok) + witness.bind(mutationReadRepairObservation{generation: 2, windowNS: 10}) + accepted := record.snapshot() + require.NotNil(t, accepted.readRepair) + release() + var result mutationReindexOutcome + select { + case result = <-outcome: + case <-time.After(5 * time.Second): + t.Fatal("actual ticket completion did not release the original wait") + } + require.True(t, result.Reindexed) + require.False(t, result.Pending) + require.Empty(t, result.Receipt, "immediate-success public output must still clear its receipt") + record.recordGraph(result) + late := record.snapshot() + require.Empty(t, late.ReindexReceipt) + require.EqualValues(t, 1, late.ReindexGeneration) + require.NotNil(t, late.readRepair, "expected initial metadata fill/Receipt clearing must not erase accepted R") + require.Equal(t, accepted.readRepair, late.readRepair) + // Exercise the production status serialization after success clears the + // transient receipt: the three identifier domains must remain distinct. + status := mutationStatusFor(t, s, map[string]any{"mutation_receipt": record.id}) + original := status["publication_phases"].(map[string]any) + repair := status["read_assisted_publication"].(map[string]any) + require.Equal(t, record.id, status["receipt"]) + require.Equal(t, "caller-logical-id", status["mutation_id"]) + require.NotEqual(t, status["receipt"], original["key"]) + require.NotEqual(t, status["mutation_id"], original["key"]) + require.Equal(t, original["key"], repair["requested_receipt"]) + require.Equal(t, "mcp_edit", original["source"]) + require.Equal(t, "monotonic", original["clock"]) + require.Equal(t, float64(1), original["ticket"]) + require.Equal(t, original["ticket"], repair["requested_generation"]) + record.recordGraph(mutationReindexOutcome{Receipt: "foreign", Generation: 99, Pending: true}) + require.Nil(t, record.snapshot().readRepair, "actual foreign rebind must still clear R") + require.EqualValues(t, 2, accepted.readRepair.generation, "prior snapshots remain immutable") + runtime.KeepAlive(idx) +} + +func TestReadRepairAcceptedAdmissionSurvivesReceiptExpiryButRejectsForeignRebind(t *testing.T) { + for _, test := range []struct { + name string + outcome mutationReindexOutcome + keep bool + }{ + {"success_clear", mutationReindexOutcome{Generation: 1, Reindexed: true}, true}, + {"pending_original", mutationReindexOutcome{Generation: 1, Receipt: "publication", Pending: true}, true}, + {"foreign_receipt", mutationReindexOutcome{Generation: 1, Receipt: "foreign", Pending: true}, false}, + {"foreign_generation", mutationReindexOutcome{Generation: 2, Receipt: "publication", Pending: true}, false}, + {"unknown_admission", mutationReindexOutcome{Reindexed: true}, false}, + {"foreign_incarnation", mutationReindexOutcome{Generation: 1, Receipt: "publication", Pending: true, checkoutID: "checkout", checkoutIncarnation: "other"}, false}, + } { + t.Run(test.name, func(t *testing.T) { + s, record, witness, phases := newReadRepairWitnessFixture(t) + witness.bind(mutationReadRepairObservation{generation: 2, windowNS: 10}) + early := record.snapshot() + require.NotNil(t, early.readRepair) + // Model disappearance of the weak admission after its existing + // map retention, without depending on GC or adding lifecycle work. + s.mutationReceipts.Delete("publication") + record.mu.Lock() + record.readRepairAdmission = weak.Pointer[mutationReceipt]{} + record.mu.Unlock() + record.recordGraph(test.outcome) + require.Equal(t, test.keep, record.snapshot().readRepair != nil) + require.EqualValues(t, 2, early.readRepair.generation) + runtime.KeepAlive(s) + runtime.KeepAlive(phases) + }) + } +} + +func TestReadRepairBindingRejectsEvictionRebindAndNewerCommit(t *testing.T) { + for _, test := range []struct { + name string + change func(*Server, *mutationCommitRecord, *indexer.PublicationPhaseRecorder) + }{ + {"evicted", func(s *Server, _ *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { + s.mutationCommits.mu.Lock() + s.mutationCommits.evictLocked(time.Now().Add(mutationCommitRetention + time.Second)) + s.mutationCommits.mu.Unlock() + }}, + {"receipt_rebound", func(_ *Server, r *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { + r.reindexReceipt = "replacement" + }}, + {"generation_rebound", func(_ *Server, r *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { r.reindexGeneration++ }}, + {"incarnation_rebound", func(_ *Server, r *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { + r.checkoutIncarnation = "replacement" + }}, + {"newer_bytes", func(_ *Server, r *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { + r.newSHA = gitBlobSHA([]byte("newer bytes")) + }}, + {"newer_commit", func(s *Server, r *mutationCommitRecord, _ *indexer.PublicationPhaseRecorder) { + s.mutationCommits.put(&mutationCommitRecord{owner: s, id: "newer", absPath: r.absPath, startedAt: time.Now()}) + }}, + {"publication_rebound", func(_ *Server, r *mutationCommitRecord, phases *indexer.PublicationPhaseRecorder) { + r.publication = phases.Begin("checkout", "replacement", indexer.PublicationSourceMCPEdit, time.Now()) + }}, + } { + t.Run(test.name, func(t *testing.T) { + s, record, witness, phases := newReadRepairWitnessFixture(t) + test.change(s, record, phases) + witness.bind(mutationReadRepairObservation{generation: 2, windowNS: 10}) + if record.snapshot().readRepair != nil { + t.Fatal("unowned/rebound mutation gained repair evidence") + } + runtime.KeepAlive(s) + runtime.KeepAlive(phases) + }) + } +} + +func TestReadRepairAcceptedHistoryIsImmutableAfterLaterEdit(t *testing.T) { + s, record, witness, phases := newReadRepairWitnessFixture(t) + witness.bind(mutationReadRepairObservation{ + generation: 2, windowNS: 10, causeCount: 1, + causes: [8]mutationReadRepairCause{{Session: "session", RequestIDKey: "n:7", AdmissionGeneration: 2}}, + }) + early := record.snapshot() + if early.readRepair == nil || early.readRepair.generation != 2 { + t.Fatal("accepted actual execution unavailable") + } + s.mutationCommits.put(&mutationCommitRecord{owner: s, id: "later", absPath: record.absPath, startedAt: time.Now()}) + witness.bind(mutationReadRepairObservation{generation: 3, windowNS: 20}) + late := record.snapshot() + if late.readRepair == nil || late.readRepair.generation != 2 || early.readRepair.windowNS != 10 { + t.Fatal("later event erased or rebound accepted historical evidence") + } + late.readRepair.causes[0].Session = "snapshot mutation" + if record.snapshot().readRepair.causes[0].Session != "session" || early.readRepair.causes[0].Session != "session" { + t.Fatal("mutable snapshot escaped into retained evidence") + } + runtime.KeepAlive(s) + runtime.KeepAlive(phases) +} + +func TestReadRepairUnobservedLedgerDoesNotAllocate(t *testing.T) { + s := newMutationServer() + idx := &indexer.Indexer{} + ctx := t.Context() + if allocations := testing.AllocsPerRun(100, func() { + if s.readRepairObserver(ctx, "/repo/file.go", idx) != nil { + panic("unobserved path acquired a witness") + } + }); allocations != 0 { + t.Fatalf("unobserved allocations: %v", allocations) + } +} + +func TestReadRepairLedgerRetentionDoesNotRestoreCanonicalEvidence(t *testing.T) { + s, record, witness, phases := newReadRepairWitnessFixture(t) + witness.bind(mutationReadRepairObservation{generation: 2, windowNS: 10}) + early := record.snapshot() + for i := range 5 { + phases.Begin("checkout", "later-"+strconv.Itoa(i), indexer.PublicationSourceMCPEdit, time.Now()) + } + if _, retained := phases.Lookup("publication"); retained { + t.Fatal("fixture did not evict the original recorder entry") + } + if record.snapshot().readRepair == nil || record.publication.CoalescedPublication() != nil || + record.publication.Snapshot().EditTimings().Window.PatchWallMS != nil { + t.Fatal("recorder eviction changed R retention or R fabricated canonical P/B") + } + s.mutationCommits.mu.Lock() + s.mutationCommits.evictLocked(time.Now().Add(mutationCommitRetention + time.Second)) + s.mutationCommits.mu.Unlock() + if record.snapshot().readRepair != nil || early.readRepair == nil || early.readRepair.windowNS != 10 { + t.Fatal("ledger eviction retained live R or mutated an earlier immutable snapshot") + } + witness.bind(mutationReadRepairObservation{generation: 3, windowNS: 20}) + if record.snapshot().readRepair != nil { + t.Fatal("late completion resurrected ledger-evicted repair evidence") + } + runtime.KeepAlive(s) + runtime.KeepAlive(phases) +} diff --git a/internal/mcp/read_repair_straddling_commit_test.go b/internal/mcp/read_repair_straddling_commit_test.go new file mode 100644 index 000000000..a7d765cee --- /dev/null +++ b/internal/mcp/read_repair_straddling_commit_test.go @@ -0,0 +1,136 @@ +package mcp + +import ( + "context" + "path/filepath" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/search" +) + +// This intentionally specifies the unresolved boundary: execution begins +// before commit bookkeeping, but its actual applied bytes publish afterward. +// The existing wholly-before-commit rejection remains a separate control. +func TestReadRepairActualExecutorStraddlesCommittedBookkeeping(t *testing.T) { + root := setupMiniRepo(t, "straddling-reader") + path := filepath.Join(root, "main.go") + gc := &config.GlobalConfig{Repos: []config.RepoEntry{{Path: root, Name: "fixture"}}} + gc.SetConfigPath(filepath.Join(t.TempDir(), "config.yaml")) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(gc.ConfigPath()) + require.NoError(t, err) + entered, resume := make(chan struct{}), make(chan struct{}) + var armed atomic.Bool + var enteredOnce sync.Once + core, _ := observer.New(zap.InfoLevel) + logger := zap.New(core, zap.Hooks(func(entry zapcore.Entry) error { + if armed.Load() && entry.Message == "indexer: reconcile phase started" { + enteredOnce.Do(func() { close(entered) }) + <-resume + } + return nil + })) + g := graph.New() + mi := indexer.NewMultiIndexer(g, testRegistry(), search.NewNull(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + registerMCPFixtureRepos(t, mi, cm) + idx := mi.GetIndexer("fixture") + require.NotNil(t, idx) + s := newMutationServer() + w := &readRepairHeldWatcher{canonicalPublicationWatcher: canonicalPublicationWatcher{hold: true}, enqueued: make(chan struct{})} + s.watcher = w + ctx, cancel := context.WithCancel(canonicalTimingContext()) + t.Cleanup(cancel) + after := []byte("package main\nfunc Hello() {}\nfunc AddedByStraddlingReader() {}\n") + type executionResult struct { + observation indexer.ReadRepairObservation + err error + } + done := make(chan executionResult, 1) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(resume) }) } + started := false + joined := false + t.Cleanup(func() { + release() + if started && !joined { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("straddling executor was not joined") + } + } + }) + s.mutationPreCommitHook = func(record *mutationCommitRecord) { + require.Equal(t, mutationDiskInFlight, record.diskStatus()) + observer := s.readRepairObserver(ctx, path, idx) + require.NotNil(t, observer) + // R has started before its first phase log; source discovery/read is + // still ahead. Hold there so the real atomic write supplies the exact + // bytes/inode consumed by the executor. No early test-only write is used. + armed.Store(true) + started = true + go func() { + var observation indexer.ReadRepairObservation + _, repairErr := mi.IncrementalReindexRepoObserved("fixture", []string{path}, &indexer.ReadRepairObserver{ + Complete: func(o indexer.ReadRepairObservation) { + observation = o + observer.Complete(o) + }, + }) + done <- executionResult{observation: observation, err: repairErr} + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("actual executor did not enter its first phase before commit") + } + } + record, err := s.commitFileMutation(ctx, "write_file", "logical-straddling", "straddling fixture", "main.go", path, after, 0o600) + require.NoError(t, err) + waiterDone := make(chan mutationReindexOutcome, 1) + go func() { waiterDone <- s.mutationReindexState(ctx, path) }() + t.Cleanup(func() { + cancel() + select { + case <-waiterDone: + case <-time.After(5 * time.Second): + t.Error("watcher waiter was not joined") + } + }) + require.Eventually(t, func() bool { + record.mu.RLock() + defer record.mu.RUnlock() + return record.readRepairAdmission.Value() != nil + }, 5*time.Second, time.Millisecond) + // Admission is bound, without waiting for the edit ACK or its timer. + release() + var observed executionResult + select { + case observed = <-done: + joined = true + case <-time.After(5 * time.Second): + t.Fatal("actual executor did not finish after commit") + } + require.NoError(t, observed.err) + start, end := observed.observation.Bounds() + require.True(t, start.Before(record.committedAt), "real execution must straddle bookkeeping") + require.True(t, end.After(record.committedAt)) + require.False(t, observed.observation.AcceptedAt().Before(record.committedAt)) + require.True(t, observed.observation.AcceptedFor(idx, gitBlobSHA(after))) + require.NotEmpty(t, g.FindNodesByName("AddedByStraddlingReader")) + require.NotNil(t, record.snapshot().readRepair, "accepted publication after commit must retain its independent executor window") + runtime.KeepAlive(idx) +} diff --git a/internal/mcp/rename_recovery_test.go b/internal/mcp/rename_recovery_test.go index 98783f09f..36df9eac5 100644 --- a/internal/mcp/rename_recovery_test.go +++ b/internal/mcp/rename_recovery_test.go @@ -235,8 +235,7 @@ func setupMultiRepoRenameRecoveryServer( registry := testRegistry() store := graph.New() multi := indexer.NewMultiIndexer(store, registry, search.NewNull(), manager, zap.NewNop()) - _, err = multi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, multi, manager) var base *indexer.Indexer if baseCfg != nil { diff --git a/internal/mcp/repo_identity_pending_test.go b/internal/mcp/repo_identity_pending_test.go new file mode 100644 index 000000000..a054cded4 --- /dev/null +++ b/internal/mcp/repo_identity_pending_test.go @@ -0,0 +1,122 @@ +package mcp + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/config" +) + +func TestStateAwareInstructions_ColdIdentityPendingAndRecovery(t *testing.T) { + srv, mi, configured := newLoadingServer(t) + physical := wsRepo(t, "physical", "alias-workspace", "AliasSymbol") + alias := filepath.Join(t.TempDir(), "different-configured-name") + require.NoError(t, os.Symlink(physical, alias)) + entries := []config.RepoEntry{{Path: configured}, {Path: alias, Name: "alias-repo"}} + mi.DeclareRepos(entries) + for _, cwd := range []string{physical, t.TempDir()} { + for _, client := range []string{"claude-code", "codex", ""} { + instructions := srv.stateAwareInstructionsForClient(cwd, client) + require.Contains(t, instructions, "identity_pending") + require.NotContains(t, instructions, "INACTIVE") + require.NotContains(t, instructions, "already configured") + require.NotContains(t, instructions, "alias-repo") + require.NotContains(t, instructions, "gortex track") + } + } + ctx := sessionCtx("unresolved-alias", physical) + ws, _, bound := srv.sessionScope(ctx) + require.True(t, bound) + require.True(t, strings.HasPrefix(ws, unresolvedWorkspacePrefix), "uncertainty grants no graph scope") + mi.ExpectRepos(entries) + require.Contains(t, srv.stateAwareInstructions(physical), "LOADING alias-repo") + _, err := mi.TrackRepoCtx(t.Context(), config.RepoEntry{Path: physical, Name: "alias-repo"}) + require.NoError(t, err) + ws, _, bound = srv.sessionScope(ctx) + require.True(t, bound) + require.Equal(t, "alias-workspace", ws, "the same session re-resolves after identity and registration complete") + require.NotContains(t, srv.stateAwareInstructions(physical), "identity_pending") + require.Contains(t, srv.stateAwareInstructions(t.TempDir()), "INACTIVE", "only complete negative identity evidence permits inactive advice") +} + +func TestIdentityPendingPreservesRegisteredAutomaticCheckout(t *testing.T) { + v := newViewStack(t) + v.srv.multiIndexer.DeclareRepos([]config.RepoEntry{{Path: v.repoRoot}, {Path: v.otherRoot}, {Path: filepath.Join(t.TempDir(), "unknown")}}) + require.True(t, v.srv.multiIndexer.RepoIdentityPending()) + alias := filepath.Join(t.TempDir(), "registered-checkout-alias") + require.NoError(t, os.Symlink(v.worktreeRoot, alias)) + for _, path := range []string{v.worktreeRoot, alias} { + served, err := v.srv.CheckoutServesCWDChecked(t.Context(), path) + require.NoError(t, err) + require.True(t, served) + require.NotContains(t, v.srv.stateAwareInstructions(path), "identity_pending") + ctx := sessionCtx("pending-other-root-"+filepath.Base(path), path) + ws, _, bound := v.srv.sessionScope(ctx) + require.True(t, bound) + require.Equal(t, "main-ws", ws) + } +} + +func TestStateAwareInstructions_PublicationDuringCheckoutLookup(t *testing.T) { + for _, register := range []bool{false, true} { + name := "identity" + if register { + name = "registration" + } + t.Run(name, func(t *testing.T) { + srv, mi, configured := newLoadingServer(t) + physical := wsRepo(t, "physical", "published-workspace", "Published") + alias := filepath.Join(t.TempDir(), "configured-alias") + require.NoError(t, os.Symlink(physical, alias)) + entries := []config.RepoEntry{{Path: configured}, {Path: alias, Name: "published"}} + mi.DeclareRepos(entries) + calls := 0 + instructions := srv.stateAwareInstructionsWithCheckout(t.Context(), physical, serverInstructions, func(ctx context.Context, path string) (bool, error) { + calls++ + require.Equal(t, physical, path) + mi.ExpectRepos(entries) + if register { + _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: physical, Name: "published"}) + require.NoError(t, err) + } + return false, nil + }) + require.Equal(t, 1, calls) + require.NotContains(t, instructions, "INACTIVE") + require.NotContains(t, instructions, "gortex track") + if register { + require.NotContains(t, instructions, "LOADING") + } else { + require.Contains(t, instructions, "LOADING published") + } + }) + } +} + +func TestStateAwareInstructions_ReadyFamilyWithoutScopeStillRetries(t *testing.T) { + srv, mi, primary := newLoadingServer(t) + linked := t.TempDir() + admin := filepath.Join(primary, ".git", "worktrees", "linked") + require.NoError(t, os.MkdirAll(admin, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(admin, "commondir"), []byte("../.."), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(linked, ".git"), []byte("gitdir: "+admin), 0644)) + calls := 0 + instructions := srv.stateAwareInstructionsWithCheckout(t.Context(), linked, serverInstructions, func(ctx context.Context, path string) (bool, error) { + calls++ + if calls == 1 { + _, err := mi.TrackRepoCtx(ctx, config.RepoEntry{Path: primary, Name: "repo-l"}) + require.NoError(t, err) + } + return false, nil + }) + require.Equal(t, 2, calls) + require.Contains(t, instructions, "view_building") + require.Contains(t, instructions, "checkout discovery is pending") + require.NotContains(t, instructions, "scope_binding_pending", "verified family pending keeps the canonical existing producer") + require.NotContains(t, instructions, "INACTIVE") + require.NotContains(t, instructions, "gortex track "+linked) +} diff --git a/internal/mcp/repo_loading.go b/internal/mcp/repo_loading.go index 7a85efd25..6de08a523 100644 --- a/internal/mcp/repo_loading.go +++ b/internal/mcp/repo_loading.go @@ -36,6 +36,25 @@ func ServerInstructionsLoading(repo, phase string, percent int) string { "the repository is already configured.", repo, loadingPhaseLabel(phase), percent) } +// ServerInstructionsIdentityPending describes uncertainty, not ownership of +// the caller's path. It is shared by initialization and modern discovery. +func ServerInstructionsIdentityPending() string { + return serverInstructionsAdmissionPending("identity_resolution", "identity_pending") +} + +// ServerInstructionsScopeBindingPending applies after registration when the +// ordinary checkout/scope authority has not established the path's route yet. +func ServerInstructionsScopeBindingPending() string { + return serverInstructionsAdmissionPending("scope_binding", "scope_binding_pending") +} + +func serverInstructionsAdmissionPending(phase, reason string) string { + return fmt.Sprintf("Gortex is LOADING graph admission after a restart (%s, 0%%). "+ + "Graph admission for this path is not yet established; calls return repo_loading (retryable, %s). "+ + "Retry shortly or continue work natively while admission completes. No graph scope has been granted; "+ + "do not change tracking configuration or reconnect to resolve this temporary state.", phase, reason) +} + // ServerInstructionsUnavailable is the inactive initialize variant for a // configured repository whose load ended without registering it. func ServerInstructionsUnavailable(repo, reason string) string { diff --git a/internal/mcp/repo_registration_fixture_test.go b/internal/mcp/repo_registration_fixture_test.go new file mode 100644 index 000000000..1d5ea61c5 --- /dev/null +++ b/internal/mcp/repo_registration_fixture_test.go @@ -0,0 +1,44 @@ +package mcp + +import ( + "context" + "sort" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" +) + +// registerMCPFixtureRepos uses the daemon's registration and shared deferred +// stages. Cross-repository queries need all roots published before resolution +// and contract matching; independent TrackRepo calls do not establish that +// boundary. These fixtures start empty and have no concurrent lifecycle writer. +func registerMCPFixtureRepos(t *testing.T, mi *indexer.MultiIndexer, cm *config.ConfigManager) { + t.Helper() + ctx := context.Background() + mi.BeginParallelBatch() + defer mi.ResetBatch() + entries := cm.ActiveRepos() + mi.ExpectRepos(entries) + var prefixes []string + for _, entry := range mi.ResolveConfiguredRepos(ctx, entries) { + mi.StartRepoLoad(entry.Path) + result, err := mi.TrackRepoCtx(ctx, entry) + mi.FinishRepoLoad(entry.Path, err) + require.NoError(t, err) + prefixes = append(prefixes, result.RepoPrefix) + } + require.NoError(t, mi.RunPreEnrichResolve(ctx, nil, nil)) + mi.RunDeferredPassesAllResult(ctx) + mi.ReconcileContractEdges() + // SQLite query fixtures also need their durable resolved-reference sidecar + // after the shared resolution, rather than after each individual root. + if rebuilder, ok := mi.Graph().(graph.RefFactsRebuilder); ok { + sort.Strings(prefixes) + require.NoError(t, rebuilder.RebuildRefFactsForRepos(prefixes)) + } + mi.EndBatch() +} diff --git a/internal/mcp/savings_retrieval_test.go b/internal/mcp/savings_retrieval_test.go index 7f022784f..3fb58778a 100644 --- a/internal/mcp/savings_retrieval_test.go +++ b/internal/mcp/savings_retrieval_test.go @@ -68,8 +68,7 @@ func Widget%d%d(key string, depth int) (string, bool) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/scope_resolve_test.go b/internal/mcp/scope_resolve_test.go index 76bde705e..ce8f13c2a 100644 --- a/internal/mcp/scope_resolve_test.go +++ b/internal/mcp/scope_resolve_test.go @@ -79,8 +79,7 @@ func newSharedWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { reg := testRegistry() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) @@ -117,8 +116,7 @@ func newSplitProjectWorkspaceServer(t *testing.T, flagOn bool) sharedWSOptions { reg := testRegistry() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) @@ -696,8 +694,7 @@ func newLoneRepoServer(t *testing.T, flagOn bool) (*Server, string) { reg := testRegistry() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) @@ -838,8 +835,7 @@ func newTwoRepoServer(t *testing.T) (*Server, string) { backend := search.NewSymbolSearcherBackend(store) mi := indexer.NewMultiIndexer(store, testRegistry(), backend, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) requireSymbolFTS(t, store) eng := query.NewEngine(store) diff --git a/internal/mcp/search_corpus_scope_test.go b/internal/mcp/search_corpus_scope_test.go index 5e51acdd0..786d7a26b 100644 --- a/internal/mcp/search_corpus_scope_test.go +++ b/internal/mcp/search_corpus_scope_test.go @@ -58,8 +58,10 @@ func newWorkspaceRootBoundServer(t *testing.T) (s *Server, root string) { reg := parser.NewRegistry() reg.Register(languages.NewGoExtractor()) mi := indexer.NewMultiIndexer(store, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } // A content node owned by z, with its body in the content index. store.AddNode(&graph.Node{ diff --git a/internal/mcp/server.go b/internal/mcp/server.go index ce4ec8a14..18858dc3b 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -131,7 +131,12 @@ func (sh *symbolHistory) All() map[string][]SymbolModification { // Server wraps the MCP server with Gortex-specific tools. type Server struct { - mcpServer *server.MCPServer + // editOperations covers actual source-edit handler completions across this + // Server's sessions/views, including handlers that outlive their caller. + // Native filesystem stimuli remain harness-owned, not inferred from events. + editOperations editOperationCounters + editCPU *editCPUObservation + mcpServer *server.MCPServer // connSessions tracks daemon-socket connections registered as live // mcp-go client sessions so server-initiated notifications reach // them (ConnectSession / DisconnectSession / WithClientSession in @@ -413,8 +418,8 @@ type Server struct { // mutationRouteWaitEntered observes a real pending source-mutation route // after selection and before polling. Nil outside deterministic tests. mutationRouteWaitEntered func(context.Context) - // releaseDispatchSlot replaces daemon.ReleaseMCPDispatchSlot in tests. - releaseDispatchSlot func(context.Context) bool + // releaseDispatchSlot replaces daemon.MoveMCPDispatchToSlowLane in tests. + releaseDispatchSlot func(context.Context, <-chan struct{}) bool // mutationCommits is the durable disk-commit ledger for the single-file // mutating tools (mutation_commit.go). It answers the question the @@ -1577,6 +1582,16 @@ func ServerInstructionsPendingAutomaticCheckout(cwd string, roots ...string) str return appendTrackedRootsDiagnostic(msg+"\n\n"+toolref.TransportClause()+" "+toolref.DegradedModeLine(), roots) } +// PendingAutomaticCheckoutInstructions captures the verified family verdict +// with its canonical rendered guidance. A later reader must not re-probe the +// filesystem and accidentally turn this pending result into untracked advice. +func PendingAutomaticCheckoutInstructions(cwd string, roots ...string) (string, bool) { + if !agents.PendingAutomaticCheckout(cwd, roots) { + return "", false + } + return ServerInstructionsPendingAutomaticCheckout(cwd, roots...), true +} + // ServerInstructionsPendingFamilyLoad retains the terminal load failure as // context, without suggesting a second track operation for a declared family. func ServerInstructionsPendingFamilyLoad(cwd string, load indexer.RepoLoad, roots ...string) string { @@ -1672,6 +1687,12 @@ func (s *Server) stateAwareInstructionsWithBase(cwd, base string) string { } func (s *Server) stateAwareInstructionsContext(ctx context.Context, cwd, base string) string { + return s.stateAwareInstructionsWithCheckout(ctx, cwd, base, s.CheckoutServesCWDChecked) +} + +// Keep the existing checkout lookup explicit so publication during that +// lookup can be exercised without a global hook or changing the transport. +func (s *Server) stateAwareInstructionsWithCheckout(ctx context.Context, cwd, base string, lookup func(context.Context, string) (bool, error)) string { // The reachability test must match the dispatcher's gate exactly. // Telling a session it is INACTIVE while every call succeeds is the // same divergence as the reverse, just quieter. @@ -1683,24 +1704,33 @@ func (s *Server) stateAwareInstructionsContext(ctx context.Context, cwd, base st // untracked: telling it to track itself and reconnect is the // advice that made agents re-track or stop on every restart. if rec, pending := s.multiIndexer.PendingRepoForPath(cwd); pending { - if !rec.State.Loading() { - roots := agents.FamilyRepoRoots(s.trackedRepoRoots(), s.multiIndexer.ConfiguredRepoRoots()) - if agents.PendingAutomaticCheckout(cwd, roots) { - return ServerInstructionsPendingFamilyLoad(cwd, rec, roots...) - } - return ServerInstructionsUnavailable(rec.Root, rec.Reason) - } - // The ordinary instructions follow the loading preamble: the - // client reads them once, and they must still be right after - // the repository is ready. - phase, percent := s.WarmupProgress() - return ServerInstructionsLoading(rec.Label(), phase, percent) + "\n\n" + base + return s.instructionsForRepoLoad(cwd, base, rec) } - if serves, err := s.CheckoutServesCWDChecked(ctx, cwd); err != nil { + if serves, err := lookup(ctx, cwd); err != nil { return CheckoutAdmissionInstructions(cwd, err) } else if serves { return s.appendLiveInstructions(cwd, base) } + if rec, classified := s.multiIndexer.RepoLoadAdmissionForPath(cwd); classified { + if rec.State.Ready() { + _, _, _, inside = s.multiIndexer.ScopeForCWD(cwd) + _, _, contains = s.multiIndexer.ContainedReposScope(cwd) + if inside || contains { + return s.appendLiveInstructions(cwd, base) + } + if serves, err := lookup(ctx, cwd); err != nil { + return CheckoutAdmissionInstructions(cwd, err) + } else if serves { + return s.appendLiveInstructions(cwd, base) + } + roots := agents.FamilyRepoRoots(s.trackedRepoRoots(), s.multiIndexer.ConfiguredRepoRoots()) + if instructions, known := PendingAutomaticCheckoutInstructions(cwd, roots...); known { + return instructions + } + return ServerInstructionsScopeBindingPending() + "\n\n" + base + } + return s.instructionsForRepoLoad(cwd, base, rec) + } roots := agents.FamilyRepoRoots(s.trackedRepoRoots(), s.multiIndexer.ConfiguredRepoRoots()) if agents.PendingAutomaticCheckout(cwd, roots) { return ServerInstructionsPendingAutomaticCheckout(cwd, roots...) @@ -1711,6 +1741,21 @@ func (s *Server) stateAwareInstructionsContext(ctx context.Context, cwd, base st return s.appendLiveInstructions(cwd, base) } +func (s *Server) instructionsForRepoLoad(cwd, base string, rec indexer.RepoLoad) string { + if rec.State == indexer.RepoLoadIdentityPending { + return ServerInstructionsIdentityPending() + "\n\n" + base + } + if !rec.State.Loading() { + roots := agents.FamilyRepoRoots(s.trackedRepoRoots(), s.multiIndexer.ConfiguredRepoRoots()) + if agents.PendingAutomaticCheckout(cwd, roots) { + return ServerInstructionsPendingFamilyLoad(cwd, rec, roots...) + } + return ServerInstructionsUnavailable(rec.Root, rec.Reason) + } + phase, percent := s.WarmupProgress() + return ServerInstructionsLoading(rec.Label(), phase, percent) + "\n\n" + base +} + func (s *Server) appendLiveInstructions(cwd, base string) string { if facts := s.liveInstructionFacts(cwd); facts != "" { base = base + "\n\n" + facts @@ -2065,6 +2110,9 @@ func NewServer(engine *query.Engine, g graph.Store, idx *indexer.Indexer, watche s.applyDefaultToolScopes() + // Capture process CPU/count0 together, after construction and before this + // Server's registered handlers can be called. Never reset on stats reads. + s.initializeEditCPUObservation(platform.ProcessCPUTime) return s } @@ -2431,7 +2479,7 @@ func (s *Server) sessionScope(ctx context.Context) (workspaceID, projectID strin // too, but only for now. Caching the sentinel would leave the session // blind after the repository registers, so it is not cached: the next // call resolves again. - if rec, pending := s.multiIndexer.PendingRepoForPath(cwd); pending && rec.State.Loading() { + if _, classified := s.multiIndexer.RepoLoadAdmissionForPath(cwd); classified { ss.scopeResolved = false } ss.scopeWorkspaceID = unresolvedWorkspacePrefix + cwd diff --git a/internal/mcp/streamable/transport.go b/internal/mcp/streamable/transport.go index ba54f4cea..8a7eb3e3a 100644 --- a/internal/mcp/streamable/transport.go +++ b/internal/mcp/streamable/transport.go @@ -98,6 +98,13 @@ type Config struct { // unresolved or local route falls through to Dispatcher. Pass // nil for the single-server case (no servers.toml). Router *daemon.Router + // DispatcherRoutesTools hands Router's tools/call attempt to the + // Dispatcher instead of running it before dispatch: the same attempt + // (inputs, order and reply framing) rides the context, see + // ToolRouteFromContext. The daemon bridge sets it so its shared admission + // covers a remote call exactly like a local one; standalone transports + // leave it off and keep routing here. + DispatcherRoutesTools bool // AllowedOrigins, when non-empty, restricts which Origin // header values the transport accepts on POST. Defends against @@ -147,15 +154,16 @@ type InitializeHook func(ctx context.Context, state *SessionState) // the only shared mutable state and is itself goroutine-safe by // contract. type Transport struct { - dispatcher Dispatcher - store SessionStore - logger *zap.Logger - router *daemon.Router - allowedOrigins map[string]struct{} - protocolVersion string - maxRequestBytes int64 - sessionTTL time.Duration - initializeHook InitializeHook + dispatcher Dispatcher + store SessionStore + logger *zap.Logger + router *daemon.Router + dispatcherRoutesTools bool + allowedOrigins map[string]struct{} + protocolVersion string + maxRequestBytes int64 + sessionTTL time.Duration + initializeHook InitializeHook streamsMu sync.Mutex streams map[string]*serverStream @@ -199,16 +207,17 @@ func New(cfg Config) *Transport { allowed[strings.ToLower(o)] = struct{}{} } return &Transport{ - dispatcher: cfg.Dispatcher, - store: store, - logger: logger, - router: cfg.Router, - allowedOrigins: allowed, - protocolVersion: version, - maxRequestBytes: maxBytes, - sessionTTL: sessionTTL, - initializeHook: cfg.InitializeHook, - streams: make(map[string]*serverStream), + dispatcher: cfg.Dispatcher, + store: store, + logger: logger, + router: cfg.Router, + dispatcherRoutesTools: cfg.DispatcherRoutesTools, + allowedOrigins: allowed, + protocolVersion: version, + maxRequestBytes: maxBytes, + sessionTTL: sessionTTL, + initializeHook: cfg.InitializeHook, + streams: make(map[string]*serverStream), } } @@ -471,6 +480,9 @@ func (t *Transport) dispatchFrame(r *http.Request, state *SessionState, frame [] // to the in-process MCP server — federating them would change // semantics. if method == "tools/call" && t.router != nil { + if t.dispatcherRoutesTools { + return t.localDispatch(t.withToolRoute(r, *state, frame), *state, frame) + } if out, status, ok := t.tryRouteToolCall(r, *state, frame); ok { return out, status, nil } @@ -479,6 +491,31 @@ func (t *Transport) dispatchFrame(r *http.Request, state *SessionState, frame [] return t.localDispatch(r, *state, frame) } +// ToolRoute is the transport's routing attempt for one tools/call frame, run +// under the context the Dispatcher passes. It reports false when the call +// stays local, and the Dispatcher then handles the frame itself. +type ToolRoute func(ctx context.Context) ([]byte, bool) + +type toolRouteContextKey struct{} + +// ToolRouteFromContext returns the routing attempt the transport handed to its +// Dispatcher (Config.DispatcherRoutesTools), or nil when there is none. +func ToolRouteFromContext(ctx context.Context) ToolRoute { + route, _ := ctx.Value(toolRouteContextKey{}).(ToolRoute) + return route +} + +// withToolRoute defers tryRouteToolCall to the Dispatcher. The attempt keeps +// this request's inputs — the call's arguments, the session's workspace pin +// and cwd, the headers — so only where it runs changes, not what it routes on. +func (t *Transport) withToolRoute(r *http.Request, state SessionState, frame []byte) *http.Request { + route := ToolRoute(func(ctx context.Context) ([]byte, bool) { + out, _, ok := t.tryRouteToolCall(r.WithContext(ctx), state, frame) + return out, ok + }) + return r.WithContext(context.WithValue(r.Context(), toolRouteContextKey{}, route)) +} + // localDispatch attaches the session id / cwd / workspace to the // context and hands the frame to the configured Dispatcher. func (t *Transport) localDispatch(r *http.Request, state SessionState, frame []byte) ([]byte, int, error) { diff --git a/internal/mcp/tool_deadline.go b/internal/mcp/tool_deadline.go index ee531071f..bb8171f87 100644 --- a/internal/mcp/tool_deadline.go +++ b/internal/mcp/tool_deadline.go @@ -17,6 +17,7 @@ import ( mcpserver "github.com/mark3labs/mcp-go/server" "go.uber.org/zap" + "github.com/zzet/gortex/internal/daemon" "github.com/zzet/gortex/internal/graphview" "github.com/zzet/gortex/internal/viewmetrics" ) @@ -150,6 +151,7 @@ func boundHandler[Req, Res any]( // keep it pinned — and say what it is holding — if it ever stops // waiting for the handler that is reading it. callCtx, viewNote := withRetainedViewNote(callCtx) + callCtx = daemon.WithMCPDispatchCompletionOwner(callCtx) type outcome struct { res Res @@ -189,6 +191,10 @@ func boundHandler[Req, Res any]( case got := <-done: return got.res, got.err case <-callCtx.Done(): + // The outer dispatch may return before this inner handler. Carry + // its unfinished physical work through the daemon's bounded drain + // accounting, rather than releasing it when this firewall returns. + releaseDispatch := daemon.RetainMCPDispatchCompletion(callCtx) if ctx.Err() != nil { // The *caller* went away (client cancelled, or the transport's // own request lifetime fired first). Report that, not a @@ -221,10 +227,13 @@ func boundHandler[Req, Res any]( // refusing everyone else's calls; the lifetime hole is the // pin, and the pin is what this fixes. retained := viewNote.retain() - if retained != nil { + if retained != nil || releaseDispatch != nil { go func() { <-done retained.release() + if releaseDispatch != nil { + releaseDispatch() + } }() } var zero Res @@ -242,6 +251,9 @@ func boundHandler[Req, Res any]( go func() { <-done retained.release() + if releaseDispatch != nil { + releaseDispatch() + } abandonedToolCalls.Add(-1) }() if s != nil && s.logger != nil { @@ -515,19 +527,43 @@ func abandonedToolMessage(name string, timeout time.Duration, verdict mutationCo func (s *Server) boundToolHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHandlerFunc { return func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { - name := req.Params.Name - return boundHandler(s, "tool", name, (func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error))(h), - func(stuck int64, timeout time.Duration) (*mcp.CallToolResult, error) { - return mcp.NewToolResultError(busyMessage("tool calls", stuck, timeout)), nil - }, - func(timeout time.Duration, note *mutationCommitNote, retained *requestViewPin) (*mcp.CallToolResult, error) { - return mcp.NewToolResultError(abandonedToolMessage(name, timeout, note.verdict(), retained)), nil - }, - func(r any) (*mcp.CallToolResult, error) { - return mcp.NewToolResultError(fmt.Sprintf("tool %q internal error: %v", name, r)), nil - }, - )(ctx, req) + if s != nil && measuredSourceEdit(req) { + return s.runAcknowledgedEdit(ctx, req, h) + } + // The ordinary path has no mutation-only named returns, defer or + // closure captures. Reads do not create an observer or sample a clock. + return s.runBoundToolHandler(ctx, req, h) + } +} + +func (s *Server) runAcknowledgedEdit(ctx context.Context, req mcp.CallToolRequest, h mcpserver.ToolHandlerFunc) (result *mcp.CallToolResult, err error) { + ctx, ack, owner := s.withEditAcknowledgement(ctx) + if owner { + defer func() { ack.finishReply(result, err) }() + } + actual := func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + if owner { + defer ack.finishActual() + } + // Actual completion and its counters stay INSIDE the firewall. + return s.runMeasuredEditOperation(ctx, req, h) } + return s.runBoundToolHandler(ctx, req, actual) +} + +func (s *Server) runBoundToolHandler(ctx context.Context, req mcp.CallToolRequest, h mcpserver.ToolHandlerFunc) (*mcp.CallToolResult, error) { + name := req.Params.Name + return boundHandler(s, "tool", name, (func(context.Context, mcp.CallToolRequest) (*mcp.CallToolResult, error))(h), + func(stuck int64, timeout time.Duration) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(busyMessage("tool calls", stuck, timeout)), nil + }, + func(timeout time.Duration, note *mutationCommitNote, retained *requestViewPin) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(abandonedToolMessage(name, timeout, note.verdict(), retained)), nil + }, + func(r any) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf("tool %q internal error: %v", name, r)), nil + }, + )(ctx, req) } // addResource registers a resource with the same hang firewall tool calls get. diff --git a/internal/mcp/tools_cfg.go b/internal/mcp/tools_cfg.go index c0ae26eb0..30bc863a1 100644 --- a/internal/mcp/tools_cfg.go +++ b/internal/mcp/tools_cfg.go @@ -87,7 +87,7 @@ func (s *Server) handleGetCFG(ctx context.Context, req mcp.CallToolRequest) (*mc // Auto re-index stale file before querying. if parts := strings.SplitN(id, "::", 2); len(parts) == 2 { - s.ensureFresh([]string{parts[0]}) + s.ensureFreshContext(ctx, []string{parts[0]}) } sc, buildErr := s.buildSymbolCFG(ctx, id) diff --git a/internal/mcp/tools_coding.go b/internal/mcp/tools_coding.go index 8802e99db..273ff3fb3 100644 --- a/internal/mcp/tools_coding.go +++ b/internal/mcp/tools_coding.go @@ -366,7 +366,7 @@ func (s *Server) handleGetEditingContext(ctx context.Context, req mcp.CallToolRe fp = s.graphRelPath(ctx, fp) // Auto re-index stale file before querying. - s.ensureFresh([]string{fp}) + s.ensureFreshContext(ctx, []string{fp}) s.sessionFor(ctx).recordFile(fp) // Tool-call observer: credit the recent search for the symbols in @@ -949,7 +949,7 @@ func (s *Server) handleGetSymbolSource(ctx context.Context, req mcp.CallToolRequ // Auto re-index stale file before querying. if parts := strings.SplitN(id, "::", 2); len(parts) == 2 { - s.ensureFresh([]string{parts[0]}) + s.ensureFreshContext(ctx, []string{parts[0]}) } node := s.engineFor(ctx).GetSymbol(id) @@ -2982,6 +2982,7 @@ func (s *Server) handleRenameSymbol(ctx context.Context, req mcp.CallToolRequest } func (s *Server) handleEditSymbol(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + s.noteAcknowledgementParser(ctx, "not_applicable_text_splice") id, err := s.symbolIDArg(ctx, req) if err != nil { return mcp.NewToolResultError("id is required"), nil @@ -3194,6 +3195,9 @@ func (s *Server) handleEditSymbol(ctx context.Context, req mcp.CallToolRequest) } return mcp.NewToolResultError(fmt.Sprintf("could not write file: %v", writeErr)), nil } + observeEditSourceEffect(ctx, true, true) // identical symbol edits already refused above + observeEditSourceComplete(ctx) + sess := s.sessionFor(ctx) sess.recordModified(node.FilePath) sess.recordSymbol(id) diff --git a/internal/mcp/tools_contract_bridge_test.go b/internal/mcp/tools_contract_bridge_test.go index 7323dd1c7..fc37168fc 100644 --- a/internal/mcp/tools_contract_bridge_test.go +++ b/internal/mcp/tools_contract_bridge_test.go @@ -127,8 +127,7 @@ func fetchUsers() { g := graph.New() mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) return NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_contracts_filter_test.go b/internal/mcp/tools_contracts_filter_test.go index a61dd188e..b4e1836db 100644 --- a/internal/mcp/tools_contracts_filter_test.go +++ b/internal/mcp/tools_contracts_filter_test.go @@ -40,8 +40,8 @@ func TestHandleContracts_FiltersByProjectAndRef(t *testing.T) { orphanRepo := setupRepoWithHTTPConsumer(t, root, "orphan-svc") tmpCfg := filepath.Join(t.TempDir(), "config.yaml") - // Top-level Repos drives `mi.IndexAll()` so every repo actually - // gets walked; projects + refs are metadata used for filter + // Top-level Repos selects the startup registration fixtures; projects + // and refs are metadata used for filter // resolution only. Listing a repo in both places is valid — the // prefix resolves identically (it's derived from Name), so the // allow-set never double-counts. @@ -70,8 +70,7 @@ func TestHandleContracts_FiltersByProjectAndRef(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_contracts_test.go b/internal/mcp/tools_contracts_test.go index cffa49b20..fb1692a3b 100644 --- a/internal/mcp/tools_contracts_test.go +++ b/internal/mcp/tools_contracts_test.go @@ -92,8 +92,7 @@ func TestHandleContracts_ReflectsRuntimeTrackedRepos(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) singleton := indexer.New(g, preg, config.IndexConfig{}, zap.NewNop()) @@ -161,8 +160,7 @@ func TestHandleContracts_MatchesGraphContractCount(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_core.go b/internal/mcp/tools_core.go index 481b21f08..2a6e6b028 100644 --- a/internal/mcp/tools_core.go +++ b/internal/mcp/tools_core.go @@ -1719,7 +1719,7 @@ func (s *Server) handleGetSymbol(ctx context.Context, req mcp.CallToolRequest) ( // Auto re-index stale file before querying. if parts := strings.SplitN(id, "::", 2); len(parts) == 2 { - s.ensureFresh([]string{parts[0]}) + s.ensureFreshContext(ctx, []string{parts[0]}) } node := s.engineFor(ctx).GetSymbol(id) @@ -2748,7 +2748,7 @@ func (s *Server) handleGetFileSummary(ctx context.Context, req mcp.CallToolReque fp = s.graphRelPath(ctx, fp) // Auto re-index stale file before querying. - s.ensureFresh([]string{fp}) + s.ensureFreshContext(ctx, []string{fp}) // gcx is the high-volume agent format and only emits total_edges // in its meta header — never per-edge rows. Route gcx-only calls @@ -3771,6 +3771,10 @@ func (s *Server) buildGraphStatsPayloadFromStats(ctx context.Context, stats *gra } result["token_savings"] = s.tokenStatsFor(ctx).snapshot() + // Process CPU and this Server's logical edit outcomes span all views and + // clients. Keep that scope explicit beside the view-derived graph stats. + result["process_edit_window"] = s.processEditWindow() + result["wal_reclaim_observation"] = s.walReclaimObservation() if cs := s.cumulativeSavingsSnapshot(); cs != nil { result["cumulative_savings"] = cs diff --git a/internal/mcp/tools_core_index_test.go b/internal/mcp/tools_core_index_test.go index b996f5d6e..4086fed6b 100644 --- a/internal/mcp/tools_core_index_test.go +++ b/internal/mcp/tools_core_index_test.go @@ -60,8 +60,7 @@ func TestHandleIndexRepository_MultiRepoRoutesThroughMultiIndexer(t *testing.T) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) // Invariant before: every node has a non-empty RepoPrefix, and both repos @@ -196,8 +195,7 @@ func TestHandleIndexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_core_reindex_test.go b/internal/mcp/tools_core_reindex_test.go index e8ee12caf..168d5cc6a 100644 --- a/internal/mcp/tools_core_reindex_test.go +++ b/internal/mcp/tools_core_reindex_test.go @@ -276,8 +276,7 @@ func TestHandleReindexRepository_MultiRepoRoutesByPrefix(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) @@ -334,8 +333,7 @@ func TestHandleReindexRepository_MultiRepoPathScoped(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ @@ -388,8 +386,7 @@ func TestHandleReindexRepository_MultiRepoRejectsUntrackedPath(t *testing.T) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_enhancements.go b/internal/mcp/tools_enhancements.go index 6dc75a5ef..65586d88f 100644 --- a/internal/mcp/tools_enhancements.go +++ b/internal/mcp/tools_enhancements.go @@ -47,6 +47,10 @@ import ( // Routing each path to its owning per-repo indexer keeps the check accurate and // the work bounded to the file the caller is about to read. func (s *Server) ensureFresh(filePaths []string) []string { + return s.ensureFreshContext(context.Background(), filePaths) +} + +func (s *Server) ensureFreshContext(ctx context.Context, filePaths []string) []string { var refreshed []string const limit = 5 for _, fp := range filePaths { @@ -70,7 +74,7 @@ func (s *Server) ensureFresh(filePaths []string) []string { if !idx.IsTrackedStale(rel) { continue } - if !s.reindexFile(absPath) { + if !s.reindexFileForRead(ctx, absPath, idx) { s.logger.Warn("auto re-index failed", zap.String("file", fp), zap.String("resolved", absPath)) diff --git a/internal/mcp/tools_fileops.go b/internal/mcp/tools_fileops.go index 284a2a062..77b52c8ea 100644 --- a/internal/mcp/tools_fileops.go +++ b/internal/mcp/tools_fileops.go @@ -828,10 +828,12 @@ func incrementalReindexSucceeded(result *indexer.IndexResult, err error) bool { // reindexFile refreshes the graph for a single file after a write. Best-effort: // non-source files or files outside any indexed repo are silently skipped. -func (s *Server) reindexFile(absPath string) bool { +func (s *Server) reindexFile(absPath string) bool { return s.reindexFileObserved(absPath, nil) } + +func (s *Server) reindexFileObserved(absPath string, observer *indexer.ReadRepairObserver) bool { if s.multiIndexer != nil { if prefix := s.multiIndexer.RepoForFile(absPath); prefix != "" { - result, err := s.multiIndexer.IncrementalReindexRepo(prefix, []string{absPath}) + result, err := s.multiIndexer.IncrementalReindexRepoObserved(prefix, []string{absPath}, observer) if incrementalReindexSucceeded(result, err) { return true } @@ -1015,7 +1017,7 @@ func (s *Server) handleEditFile(ctx context.Context, req mcp.CallToolRequest) (* allowParseErrors := req.GetBool("allow_parse_errors", false) var gate parseGateResult if parseGateEnabled() { - gate = checkParseGate(relPath, content, newContentBytes) + gate = s.checkAcknowledgedParseGate(ctx, relPath, content, newContentBytes) if gate.Blocked && !allowParseErrors && !dryRun { return mcp.NewToolResultError(parseGateError(relPath, gate)), nil } @@ -1057,6 +1059,9 @@ func (s *Server) handleEditFile(ctx context.Context, req mcp.CallToolRequest) (* return mcp.NewToolResultError(fmt.Sprintf("could not write file: %v", writeErr)), nil } + observeEditSourceEffect(ctx, true, true) // identical edits already refused above + observeEditSourceComplete(ctx) + sess := s.sessionFor(ctx) sess.recordModified(relPath) @@ -1134,6 +1139,7 @@ func (s *Server) handleWriteFile(ctx context.Context, req mcp.CallToolRequest) ( status := "created" perm := os.FileMode(0o644) fileExists := false + fileExistenceKnown := false if info, err := os.Stat(absPath); err == nil { if info.IsDir() { return mcp.NewToolResultError(fmt.Sprintf("path %q is a directory", rawPath)), nil @@ -1141,6 +1147,9 @@ func (s *Server) handleWriteFile(ctx context.Context, req mcp.CallToolRequest) ( status = "overwritten" perm = info.Mode().Perm() fileExists = true + fileExistenceKnown = true + } else { + fileExistenceKnown = errors.Is(err, os.ErrNotExist) } // Drift guard: when the caller observed the file at base_sha, @@ -1166,8 +1175,11 @@ func (s *Server) handleWriteFile(ctx context.Context, req mcp.CallToolRequest) ( // Read the prior bytes once: the parse gate compares against them, and an // evidence receipt digests them as before_sha256. var priorContent []byte + priorContentKnown := fileExistenceKnown && !fileExists if fileExists { - priorContent, _ = os.ReadFile(absPath) + var readErr error + priorContent, readErr = os.ReadFile(absPath) + priorContentKnown = readErr == nil } // Before anything is written, so a withheld receipt never leaves an // overwritten file behind. @@ -1180,7 +1192,7 @@ func (s *Server) handleWriteFile(ctx context.Context, req mcp.CallToolRequest) ( allowParseErrors := req.GetBool("allow_parse_errors", false) var gate parseGateResult if parseGateEnabled() { - gate = checkParseGate(relPath, priorContent, contentBytes) + gate = s.checkAcknowledgedParseGate(ctx, relPath, priorContent, contentBytes) if gate.Blocked && !allowParseErrors && !dryRun { return mcp.NewToolResultError(parseGateError(relPath, gate)), nil } @@ -1220,6 +1232,9 @@ func (s *Server) handleWriteFile(ctx context.Context, req mcp.CallToolRequest) ( return mcp.NewToolResultError(fmt.Sprintf("could not write file: %v", writeErr)), nil } + observeEditSourceReplacement(ctx, priorContent, contentBytes, fileExists, priorContentKnown) + observeEditSourceComplete(ctx) + sess := s.sessionFor(ctx) sess.recordModified(relPath) diff --git a/internal/mcp/tools_fileops_singlerepo_test.go b/internal/mcp/tools_fileops_singlerepo_test.go index 5125bccac..2f80f3da6 100644 --- a/internal/mcp/tools_fileops_singlerepo_test.go +++ b/internal/mcp/tools_fileops_singlerepo_test.go @@ -39,8 +39,12 @@ func newSingleRepoServer(t *testing.T) (*Server, *graph.Graph, string) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // These fixtures test path ownership/freshness, not cross-repo batch + // resolution. Register their explicit roots through the production path. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, @@ -117,8 +121,12 @@ func TestResolveFilePath_MultiRepoBareRelativeStillAmbiguous(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + // These fixtures test path ownership/freshness, not cross-repo batch + // resolution. Register their explicit roots through the production path. + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/tools_fileops_worktree_test.go b/internal/mcp/tools_fileops_worktree_test.go index 35f09d94b..28e94ff96 100644 --- a/internal/mcp/tools_fileops_worktree_test.go +++ b/internal/mcp/tools_fileops_worktree_test.go @@ -87,8 +87,7 @@ func setupWorktreePair(t *testing.T) (mainRepo, worktree string, srv *Server) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) @@ -420,8 +419,7 @@ func TestEditTools_UnprefixedPathAmbiguousAcrossTwoWorktrees(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, MultiIndexer: mi, diff --git a/internal/mcp/tools_find_declaration_test.go b/internal/mcp/tools_find_declaration_test.go index 9cecd6569..86736e4be 100644 --- a/internal/mcp/tools_find_declaration_test.go +++ b/internal/mcp/tools_find_declaration_test.go @@ -251,8 +251,7 @@ func consumer() int { return uniqueDecl() + uniqueDecl() } reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) diff --git a/internal/mcp/tools_find_files_scoped_test.go b/internal/mcp/tools_find_files_scoped_test.go index 103ae148b..f7d1edcbb 100644 --- a/internal/mcp/tools_find_files_scoped_test.go +++ b/internal/mcp/tools_find_files_scoped_test.go @@ -18,6 +18,7 @@ import ( type findFilesProjectionStore struct { *store_sqlite.Store globalReads int + edgeReads int projectedRepos []string fullReads int summaryReads int @@ -42,6 +43,11 @@ func (s *findFilesProjectionStore) AllNodesLight() []*graph.Node { return s.Store.AllNodesLight() } +func (s *findFilesProjectionStore) AllEdges() []*graph.Edge { + s.edgeReads++ + return s.Store.AllEdges() +} + func (s *findFilesProjectionStore) NodesInScopeSeq(repos, files []string, kinds ...graph.NodeKind) iter.Seq[*graph.Node] { s.fullReads++ s.projectedRepos = append(s.projectedRepos, repos...) @@ -98,6 +104,7 @@ func TestFindFilesScopedNeverScansAndRowsStayFlat(t *testing.T) { require.NoError(t, err) require.False(t, res.IsError) require.Zero(t, tracked.globalReads, "a scoped find_files read the whole store") + require.Zero(t, tracked.edgeReads, "a scoped find_files read every edge") require.Zero(t, tracked.fullReads) require.Zero(t, tracked.summaryReads, "no project narrows the scope, so file locations suffice") require.Equal(t, 1, tracked.locationReads) diff --git a/internal/mcp/tools_move_inline.go b/internal/mcp/tools_move_inline.go index 17a150a72..c08e7ffce 100644 --- a/internal/mcp/tools_move_inline.go +++ b/internal/mcp/tools_move_inline.go @@ -84,6 +84,7 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) return mcp.NewToolResultError("symbol not found: " + id), nil } if !isGoSourcePath(node.FilePath) { + s.noteAcknowledgementParser(ctx, "unsupported_non_go_move") return mcp.NewToolResultError(fmt.Sprintf("unsupported language: move_symbol only supports Go files (got %q)", node.FilePath)), nil } if node.Name == "" { @@ -109,6 +110,7 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) return mcp.NewToolResultError("target_file is the same as the source file"), nil } if !isGoSourcePath(tgtAbs) { + s.noteAcknowledgementParser(ctx, "unsupported_non_go_move") return mcp.NewToolResultError(fmt.Sprintf("unsupported language: target_file must be a .go file (got %q)", targetFileArg)), nil } @@ -122,8 +124,10 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) } // Parse source file with comments so we keep doc-comments intact. + s.noteAcknowledgementParser(ctx, "explicit_go_parser_calls_formatting_unattributed") + ack := editAcknowledgementFrom(ctx, s) fset := token.NewFileSet() - srcFile, err := parser.ParseFile(fset, srcAbs, srcContent, parser.ParseComments) + srcFile, err := parseAcknowledgedGoFile(ack, fset, srcAbs, srcContent, parser.ParseComments) if err != nil { return mcp.NewToolResultError(fmt.Sprintf("could not parse source file: %v", err)), nil } @@ -143,13 +147,13 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) } targetPackage := "" if tgtExists { - tgtFileTmp, perr := parser.ParseFile(fset, tgtAbs, tgtContent, parser.ParseComments) + tgtFileTmp, perr := parseAcknowledgedGoFile(ack, fset, tgtAbs, tgtContent, parser.ParseComments) if perr != nil { return mcp.NewToolResultError(fmt.Sprintf("could not parse target file: %v", perr)), nil } targetPackage = tgtFileTmp.Name.Name } else { - targetPackage = inferGoPackageForDir(tgtDir, targetPackageOverride) + targetPackage = inferGoPackageForDirObserved(ack, tgtDir, targetPackageOverride) if targetPackage == "" { return mcp.NewToolResultError("could not determine target package; pass target_package"), nil } @@ -184,6 +188,7 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) // elsewhere; if so, the source file needs to import the target // package and rewrite bare references. newSrcContent, _, err = rewriteCallerFileContent(newSrcContent, srcAbs, callerRewriteOpts{ + acknowledgement: ack, Name: node.Name, Mode: callerSourcePkgRewrite, TargetImportPath: tgtImportPath, @@ -207,6 +212,7 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) // reference into a same-package call after relocation). if !samePackage { newTgtContent, _, err = rewriteCallerFileContent(newTgtContent, tgtAbs, callerRewriteOpts{ + acknowledgement: ack, Name: node.Name, Mode: callerTargetPkgRewrite, SourceImportPath: srcImportPath, @@ -287,6 +293,7 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) mode = callerTargetPkgRewrite } newContent, info, rerr := rewriteCallerFileContent(content, absPath, callerRewriteOpts{ + acknowledgement: ack, Name: node.Name, Mode: mode, SourceImportPath: srcImportPath, @@ -371,31 +378,47 @@ func (s *Server) handleMoveSymbol(ctx context.Context, req mcp.CallToolRequest) return s.respondJSONOrTOON(ctx, req, resp) } + measurement := s.beginMoveObservation(ctx, id, resp["target_id"].(string), srcAbs, tgtAbs, srcRel, tgtRelOut, samePackage && len(callerEdits) == 0) + defer measurement.finishWrites(false) // Apply writes atomically. Order: target first (so file exists for any // caller that triggers a follow-up open), then source, then callers. - if err := writeMoveFile(tgtAbs, newTgtContent); err != nil { + if err := measurement.write(editAcknowledgementFrom(ctx, s), tgtAbs, newTgtContent); err != nil { return mcp.NewToolResultError(err.Error()), nil } - if err := writeMoveFile(srcAbs, newSrcContent); err != nil { + observeEditSourceReplacement(ctx, tgtContent, newTgtContent, tgtExists, true) + if err := measurement.write(editAcknowledgementFrom(ctx, s), srcAbs, newSrcContent); err != nil { return mcp.NewToolResultError(err.Error()), nil } + observeEditSourceReplacement(ctx, srcContent, newSrcContent, true, true) for _, ce := range callerEdits { - if err := writeMoveFile(ce.absPath, ce.newContent); err != nil { + if err := measurement.write(editAcknowledgementFrom(ctx, s), ce.absPath, ce.newContent); err != nil { return mcp.NewToolResultError(err.Error()), nil } + observeEditSourceReplacement(ctx, ce.origContent, ce.newContent, true, true) } + measurement.finishWrites(true) + observeEditSourceComplete(ctx) // Re-index every touched file so the graph picks up the new location. sess := s.sessionFor(ctx) for _, t := range touched { sess.recordModified(t.Path) } - _ = s.reindexFile(srcAbs) - _ = s.reindexFile(tgtAbs) + if measurement != nil { + _ = measurement.reindex(ctx, srcAbs) + _ = measurement.reindex(ctx, tgtAbs) + } else { + _ = s.reindexFile(srcAbs) + _ = s.reindexFile(tgtAbs) + } for _, ce := range callerEdits { _ = s.reindexFile(ce.absPath) } + if measurement != nil { + resp["move_observation_id"] = measurement.id + resp["file_observations"] = measurement.references() + } return s.respondJSONOrTOON(ctx, req, resp) } @@ -442,6 +465,10 @@ func buildMovedSymbolID(s *Server, tgtAbs, name string) string { } func writeMoveFile(absPath string, content []byte) error { + return writeMoveFileObserved(nil, absPath, content) +} + +func writeMoveFileObserved(a *editAcknowledgement, absPath string, content []byte) error { perm := os.FileMode(0o644) if info, err := os.Stat(absPath); err == nil { perm = info.Mode().Perm() @@ -449,7 +476,10 @@ func writeMoveFile(absPath string, content []byte) error { if err := os.MkdirAll(filepath.Dir(absPath), 0o755); err != nil { return fmt.Errorf("could not create parent dir for %s: %w", absPath, err) } - if err := agents.AtomicWriteFile(absPath, content, perm); err != nil { + lap := beginEditAcknowledgementLap(a, editAckDiskWrite) + err := agents.AtomicWriteFile(absPath, content, perm) + lap.finish() + if err != nil { return fmt.Errorf("could not write %s: %w", absPath, err) } return nil @@ -581,7 +611,7 @@ func spliceOutGoDecl(src []byte, fset *token.FileSet, decl ast.Decl) ([]byte, er } // Pull start back one more newline if we now sit on a blank line so // the file doesn't grow blank lines on every move. - if start > 0 && src[start-1] == '\n' && end < len(src) { + if start > 1 && src[start-1] == '\n' && src[start-2] == '\n' && end < len(src) { start-- } out := make([]byte, 0, len(src)-(end-start)) @@ -612,10 +642,10 @@ func readGoFileIfExists(absPath string) ([]byte, bool, error) { return nil, false, fmt.Errorf("could not read target file: %w", err) } -// inferGoPackageForDir picks a package name for a new Go file in `dir`. -// Priority: explicit override, package of an existing .go sibling, then -// the directory's leaf name. -func inferGoPackageForDir(dir, override string) string { +// inferGoPackageForDirObserved picks a package name for a new Go file. +// Priority: explicit override, existing Go sibling, then directory leaf name. +// The optional owner observes only actual parser calls. +func inferGoPackageForDirObserved(ack *editAcknowledgement, dir, override string) string { if override != "" { return override } @@ -633,7 +663,7 @@ func inferGoPackageForDir(dir, override string) string { continue } fset := token.NewFileSet() - f, perr := parser.ParseFile(fset, "", content, parser.PackageClauseOnly) + f, perr := parseAcknowledgedGoFile(ack, fset, "", content, parser.PackageClauseOnly) if perr == nil && f.Name != nil { return f.Name.Name } @@ -759,6 +789,7 @@ const ( ) type callerRewriteOpts struct { + acknowledgement *editAcknowledgement Name string Mode callerRewriteMode SourceImportPath string @@ -783,7 +814,7 @@ func rewriteCallerFileContent(content []byte, absPath string, opts callerRewrite return content, callerRewriteResult{}, nil } fset := token.NewFileSet() - file, err := parser.ParseFile(fset, absPath, content, parser.ParseComments) + file, err := parseAcknowledgedGoFile(opts.acknowledgement, fset, absPath, content, parser.ParseComments) if err != nil { return nil, callerRewriteResult{}, fmt.Errorf("could not parse %s: %w", absPath, err) } diff --git a/internal/mcp/tools_multi_worktree_test.go b/internal/mcp/tools_multi_worktree_test.go index 147b36fa6..0002af3df 100644 --- a/internal/mcp/tools_multi_worktree_test.go +++ b/internal/mcp/tools_multi_worktree_test.go @@ -35,8 +35,7 @@ func newWorktreeMCPServer(t *testing.T, repos ...config.RepoEntry) (*Server, *in preg := testRegistry() g := graph.New() mi := indexer.NewMultiIndexer(g, preg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) singleton := indexer.New(g, preg, config.IndexConfig{}, zap.NewNop()) diff --git a/internal/mcp/tools_mutation_status.go b/internal/mcp/tools_mutation_status.go index 85c42be7c..c25cfdece 100644 --- a/internal/mcp/tools_mutation_status.go +++ b/internal/mcp/tools_mutation_status.go @@ -7,6 +7,8 @@ import ( "time" "github.com/mark3labs/mcp-go/mcp" + + "github.com/zzet/gortex/internal/indexer" ) var graphFreshnessReceiptID = regexp.MustCompile(`^mutation-\d+$`) @@ -27,6 +29,15 @@ func (s *Server) handleMutationStatus(ctx context.Context, req mcp.CallToolReque case receipt != "": record, ok := s.mutationCommits.byReceipt(receipt) if !ok { + if payload, handled := s.moveFileObservationPayload(ctx, receipt); handled { + return s.respondJSONOrTOON(ctx, req, payload) + } + if payload, handled := s.batchAcknowledgementReceiptPayload(ctx, receipt); handled { + return s.respondJSONOrTOON(ctx, req, payload) + } + if payload, handled := s.batchFileObservationPayload(ctx, receipt); handled { + return s.respondJSONOrTOON(ctx, req, payload) + } if payload, found := s.graphRefreshReceiptPayload(ctx, receipt); found { return s.respondJSONOrTOON(ctx, req, payload) } @@ -136,13 +147,29 @@ func (s *Server) graphRefreshReceiptPayload(ctx context.Context, id string) (map if outcome.Err != nil { payload["error"] = outcome.Err.Error() } - if phases := publicationPhasesPayload(receipt.id, time.Time{}); phases != nil { - payload["publication_phases"] = phases + var retained *indexer.PublicationPhaseRecord + var acknowledgement *editAcknowledgementSnapshot + acknowledgementState := "not_applicable" + if commit, found := s.mutationCommits.byFreshnessReceipt(receipt.id); found && mutationCommitInScope(ctx, commit) { + acknowledgementState = "unavailable_recording_or_evicted" + retained, _ = retainedMutationPublication(commit) + if ack := retainedMutationAcknowledgement(commit); ack != nil { + snapshot := ack.snapshot() + acknowledgement = &snapshot + } + if retained != nil && retained.Key() != receipt.id { + retained = nil + } } + attachMutationPhaseTimings(payload, receipt.id, time.Time{}, indexer.PublicationSourceFreshRequest, retained) + attachAcknowledgementPayload(payload, acknowledgement, acknowledgementState) return payload, true } func mutationCommitInScope(ctx context.Context, record *mutationCommitRecord) bool { + if !record.isPhysicalCommit() { + return false + } checkoutID, incarnation := mutationCheckoutScope(ctx) record.mu.RLock() defer record.mu.RUnlock() @@ -169,6 +196,9 @@ func (s *Server) recentMutationCommitInScope(ctx context.Context, path string) ( // pending receipt would read as pending forever, which is precisely the stale // answer this tool exists to avoid. func (s *Server) mutationStatusPayload(record *mutationCommitRecord) map[string]any { + if !record.isPhysicalCommit() { + return map[string]any{"found": false, "reason": "not_physical_commit"} + } s.refreshPendingSourceRecord(record) snap := record.snapshot() payload := map[string]any{ @@ -180,6 +210,9 @@ func (s *Server) mutationStatusPayload(record *mutationCommitRecord) map[string] "graph_status": snap.GraphStatus, } attachMutationRefreshSnapshot(payload, snap) + if snap.readRepair != nil { + payload["read_assisted_publication"] = snap.readRepair.payload() + } if snap.MutationID != "" { payload["mutation_id"] = snap.MutationID } @@ -207,14 +240,14 @@ func (s *Server) mutationStatusPayload(record *mutationCommitRecord) map[string] payload["graph_note"] = note } payload["guidance"] = mutationStatusGuidance(snap.DiskStatus) - if snap.ReindexReceipt != "" { - record.mu.RLock() - committedAt := record.committedAt - record.mu.RUnlock() - if phases := publicationPhasesPayload(snap.ReindexReceipt, committedAt); phases != nil { - payload["publication_phases"] = phases - } - } + retained, committedAt := retainedMutationPublication(record) + attachMutationPhaseTimings(payload, snap.ReindexReceipt, committedAt, indexer.PublicationSourceMCPEdit, retained) + var acknowledgement *editAcknowledgementSnapshot + if ack := retainedMutationAcknowledgement(record); ack != nil { + snapshot := ack.snapshot() + acknowledgement = &snapshot + } + attachAcknowledgementPayload(payload, acknowledgement, "unavailable_recording_or_evicted") return payload } diff --git a/internal/mcp/tools_review_rulepack_test.go b/internal/mcp/tools_review_rulepack_test.go index 2c7668063..d938323ca 100644 --- a/internal/mcp/tools_review_rulepack_test.go +++ b/internal/mcp/tools_review_rulepack_test.go @@ -51,8 +51,10 @@ func prefixedServerOver(t *testing.T, dir, name string) (*Server, string) { g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } srv := NewServer(query.NewEngine(g), g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ ConfigManager: cm, diff --git a/internal/mcp/tools_search_text_pathsep_test.go b/internal/mcp/tools_search_text_pathsep_test.go index abf0a31cb..7d9e5989b 100644 --- a/internal/mcp/tools_search_text_pathsep_test.go +++ b/internal/mcp/tools_search_text_pathsep_test.go @@ -62,8 +62,10 @@ func nestedRepoServer(t *testing.T, entries []config.RepoEntry) (*Server, *graph reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + for _, entry := range cm.Global().ReposSnapshot() { + _, err := mi.TrackRepoCtx(context.Background(), entry) + require.NoError(t, err) + } singleton := indexer.New(g, reg, config.IndexConfig{}, zap.NewNop()) srv := NewServer(query.NewEngine(g), g, singleton, nil, zap.NewNop(), nil, MultiRepoOptions{ diff --git a/internal/mcp/tools_search_text_test.go b/internal/mcp/tools_search_text_test.go index 4960e11f6..3cb886404 100644 --- a/internal/mcp/tools_search_text_test.go +++ b/internal/mcp/tools_search_text_test.go @@ -183,8 +183,7 @@ func TestSearchText_MultiRepoFanout(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) @@ -239,8 +238,7 @@ func TestSearchText_MultiRepoScopedGrepDoesNotLetOutOfScopeRepoConsumeLimit(t *t reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) flagOn := true @@ -297,8 +295,7 @@ func TestFilterTextMatchesByResolvedScope_FailsClosed(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) srv := NewServer(eng, g, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ @@ -367,8 +364,7 @@ func TestGetFileSummary_MultiRepoRelativePath(t *testing.T) { reg.Register(languages.NewGoExtractor()) g := graph.New() mi := indexer.NewMultiIndexer(g, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) require.True(t, mi.IsMultiRepo()) eng := query.NewEngine(g) @@ -514,8 +510,7 @@ func TestSearchTextScopedNeverScansAndRowsStayFlat(t *testing.T) { reg.Register(languages.NewGoExtractor()) counter := newScopedReadCounter(openContractCoreCapStore(t)) mi := indexer.NewMultiIndexer(counter, reg, search.NewNull(), cm, zap.NewNop()) - _, err = mi.IndexAll() - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) srv := NewServer(query.NewEngine(counter), counter, nil, nil, zap.NewNop(), nil, MultiRepoOptions{ConfigManager: cm, MultiIndexer: mi}) installContractCoreKindTestRuntime(t, srv) return srv, counter diff --git a/internal/mcp/view_freshness_proof.go b/internal/mcp/view_freshness_proof.go index 2c1ff4fad..88eb7efe0 100644 --- a/internal/mcp/view_freshness_proof.go +++ b/internal/mcp/view_freshness_proof.go @@ -316,7 +316,10 @@ var pendingEditRecordSequence atomic.Uint64 // admission through indexer.WithPublicationRecord, so the coordinator binds // the ticket to it (and marks ticket_enqueued) before it wakes the cycle that // serves it. Its origin is the instant the tool call entered the middleware. -func openMutationPhases(ctx context.Context, checkoutID, path string) (*indexer.PublicationPhaseRecord, string) { +func (s *Server) openMutationPhases(ctx context.Context, checkoutID, path string) (*indexer.PublicationPhaseRecord, string) { + if record := s.committedPublicationRecord(ctx, path); record != nil { + return record, record.Key() + } key := fmt.Sprintf("edit-pending-%d", pendingEditRecordSequence.Add(1)) record := indexer.DefaultPublicationPhases().Begin(checkoutID, key, indexer.PublicationSourceMCPEdit, toolReceivedAt(ctx)) // The disk commit happened before admission, on this request: its @@ -387,11 +390,53 @@ func committedAtFor(ctx context.Context, path string) time.Time { // publicationPhasesPayload renders the publication record kept under a // graph-refresh receipt, folding in the disk commit instant the commit ledger // holds (same process, same monotonic clock). Nil when no record exists. -func publicationPhasesPayload(receiptID string, committedAt time.Time) any { +func publicationPhasesPayload(receiptID string, committedAt time.Time) *indexer.PublicationPhaseSnapshot { record, ok := indexer.DefaultPublicationPhases().Lookup(receiptID) if !ok { return nil } record.MarkAt(indexer.PublicationReceiptCommitted, committedAt) - return record.Snapshot() + snapshot := record.Snapshot() + return &snapshot +} + +// attachMutationPhaseTimings reads one publication snapshot for both payloads, +// so a concurrently completing receipt cannot give them different endpoints. +// Older/canonical paths without a timeline expose unavailable durations, not +// zeroes. The legacy publication_phases value and vocabulary stay unchanged. +func attachMutationPhaseTimings(payload map[string]any, receiptID string, committedAt time.Time, source string, retained *indexer.PublicationPhaseRecord) { + snapshot := indexer.PublicationPhaseSnapshot{Source: source} + if retained != nil { + // Recovery may rebind the commit to a different freshness receipt. + // Its physical write record must not borrow the replacement's phases, + // or let an older completion appear to finish that newer request. + retained.MarkAt(indexer.PublicationReceiptCommitted, committedAt) + candidate := retained.Snapshot() + if receiptID == "" || candidate.Key == receiptID { + snapshot = candidate + payload["publication_phases"] = snapshot + winner, unavailable := retained.CoalescedPublicationEvidence() + if winner != nil { + payload["coalesced_publication"] = winner + if winner.Background != nil { + payload["requested_background"] = winner.Background + } + } + if winner == nil { + if background := retained.RequestedBackground(); background != nil { + payload["requested_background"] = background + } + } + if unavailable != nil { + payload["requested_background_unavailable"] = unavailable + } + } + } else if phases := publicationPhasesPayload(receiptID, committedAt); phases != nil { + snapshot = *phases + payload["publication_phases"] = snapshot + } + timings := snapshot.EditTimings() + payload["phases_ms"] = timings.Milliseconds + payload["phase_states"] = timings.States + payload["phase_timing"] = timings.Window } diff --git a/internal/mcp/view_mutation_state.go b/internal/mcp/view_mutation_state.go index c06af6026..f83d88d9d 100644 --- a/internal/mcp/view_mutation_state.go +++ b/internal/mcp/view_mutation_state.go @@ -158,8 +158,10 @@ func (s *Server) refreshCheckoutMutation(ctx context.Context, path string, state // the generations this request read stay pinned for exactly as long as // the detached publication is outstanding, cancellation tail included. pin := handoffRequestView(ctx, viewmetrics.HandoffCheckoutRefresh) - record, interimKey := openMutationPhases(ctx, state.checkoutID, path) + record, interimKey := s.openMutationPhases(ctx, state.checkoutID, path) + admissionLap := s.beginAcknowledgementLap(ctx, editAckTicketAdmission) ticket, err := scheduler.EnqueueRefresh(indexer.WithPublicationRecord(withCommittedWrite(context.WithoutCancel(ctx), state, path), record), path) + admissionLap.finish() if err != nil { failMutationPhases(record) pin.release() diff --git a/internal/mcp/view_ref_test.go b/internal/mcp/view_ref_test.go index 12736d83c..027c699af 100644 --- a/internal/mcp/view_ref_test.go +++ b/internal/mcp/view_ref_test.go @@ -168,9 +168,7 @@ func newUnadmittedRefStack(t *testing.T) *refStack { languages.RegisterAll(reg) bm := search.NewNull() mi := indexer.NewMultiIndexer(store, reg, bm, cm, zap.NewNop()) - if _, err := mi.IndexScoped("", ""); err != nil { - t.Fatalf("index the fixture repository: %v", err) - } + registerMCPFixtureRepos(t, mi, cm) graphID := indexer.GraphIDFor(refTestPrefix) seedRefCatalog(t, store, graphID, repo, headTree) diff --git a/internal/mcp/view_request_test.go b/internal/mcp/view_request_test.go index e82421293..5b674ab22 100644 --- a/internal/mcp/view_request_test.go +++ b/internal/mcp/view_request_test.go @@ -154,9 +154,7 @@ func newViewStackWithRepos(t *testing.T, includeOther bool) *viewStack { languages.RegisterAll(reg) bm := search.NewNull() mi := indexer.NewMultiIndexer(store, reg, bm, cm, zap.NewNop()) - if _, err := mi.IndexScoped("", ""); err != nil { - t.Fatalf("index fixture repos: %v", err) - } + registerMCPFixtureRepos(t, mi, cm) graphID := indexer.GraphIDFor("repo") stack := &viewStack{ diff --git a/internal/mcp/wal_observations.go b/internal/mcp/wal_observations.go new file mode 100644 index 000000000..4ad6a20bf --- /dev/null +++ b/internal/mcp/wal_observations.go @@ -0,0 +1,64 @@ +package mcp + +import ( + "reflect" + + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +// walReclaimSnapshotReader is intentionally smaller than daemon storage status: +// WALReclaimStats copies counters under a short mutex plus atomic loads. It +// does not stat files, read the wal-index, query SQL or wait for the writer. +type walReclaimSnapshotReader interface { + WALReclaimStats() store_sqlite.WALReclaimStats +} + +type walReclaimObservation struct { + Scope string `json:"scope"` + ObservationEpoch string `json:"observation_epoch"` + EpochScope string `json:"epoch_scope"` + SnapshotAvailable bool `json:"snapshot_available"` + Reason string `json:"reason,omitempty"` + CancelledWriterParked *int64 `json:"cancelled_writer_parked"` + CancelledWriteIntent *int64 `json:"cancelled_write_intent"` + CancelledOther *int64 `json:"cancelled_other"` + ObservedPeakWALBytes *int64 `json:"observed_peak_wal_bytes"` + WALPeakObserved bool `json:"wal_peak_observed"` + PeakScope string `json:"peak_scope"` +} + +// walReclaimObservation uses the Server's constructor-bound physical store, +// never a selected/composed reader or an unwrapped base. The Server epoch is +// only an observation-binding correlation, not a durable store-open identity. +func (s *Server) walReclaimObservation() walReclaimObservation { + result := walReclaimObservation{ + Scope: "physical_store_since_open", EpochScope: "server_binding", + PeakScope: "sampled_since_store_open_lower_bound", + } + if s == nil || s.graph == nil { + result.Reason = "store_unavailable" + return result + } + if s.editCPU != nil { + result.ObservationEpoch = s.editCPU.epoch + } + reader, ok := s.graph.(walReclaimSnapshotReader) + if !ok { + result.Reason = "unsupported_store" + return result + } + if value := reflect.ValueOf(reader); value.Kind() == reflect.Pointer && value.IsNil() { + result.Reason = "store_unavailable" + return result + } + stats := reader.WALReclaimStats() // exactly one existing, I/O-free snapshot + parked, intent, other := stats.CancelledWriterParked, stats.CancelledWriteIntent, stats.CancelledOther + result.SnapshotAvailable = true + result.CancelledWriterParked, result.CancelledWriteIntent, result.CancelledOther = &parked, &intent, &other + result.WALPeakObserved = stats.WALPeakObserved + if stats.WALPeakObserved { + peak := stats.ObservedPeakWALBytes + result.ObservedPeakWALBytes = &peak + } + return result +} diff --git a/internal/mcp/wal_observations_test.go b/internal/mcp/wal_observations_test.go new file mode 100644 index 000000000..56df5b081 --- /dev/null +++ b/internal/mcp/wal_observations_test.go @@ -0,0 +1,159 @@ +package mcp + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +type walObservationSpy struct { + graph.Store + stats store_sqlite.WALReclaimStats + reads atomic.Int64 +} + +func (s *walObservationSpy) WALReclaimStats() store_sqlite.WALReclaimStats { + s.reads.Add(1) + return s.stats +} + +func (*walObservationSpy) DBStats() (int64, int64) { + panic("MCP WAL observations must not stat store files") +} + +func (*walObservationSpy) CloseCheckpointEstimate() (time.Duration, int64) { + panic("MCP WAL observations must not inspect the wal-index") +} + +func TestWALObservationSnapshotOnlyNullablePeakAndOwnedValues(t *testing.T) { + spy := &walObservationSpy{Store: graph.New()} + s := &Server{graph: spy} + first := s.walReclaimObservation() + require.True(t, first.SnapshotAvailable) + require.EqualValues(t, 1, spy.reads.Load()) + require.EqualValues(t, 0, *first.CancelledWriterParked) + require.EqualValues(t, 0, *first.CancelledWriteIntent) + require.EqualValues(t, 0, *first.CancelledOther) + require.Nil(t, first.ObservedPeakWALBytes, "no observation must not become zero") + require.False(t, first.WALPeakObserved) + spy.stats = store_sqlite.WALReclaimStats{ + CancelledWriterParked: 3, CancelledWriteIntent: 2, CancelledOther: 1, WALPeakObserved: true, + } + second := s.walReclaimObservation() + require.EqualValues(t, 2, spy.reads.Load()) + require.True(t, second.WALPeakObserved) + require.NotNil(t, second.ObservedPeakWALBytes) + require.EqualValues(t, 0, *second.ObservedPeakWALBytes, "an actually observed empty WAL is valid zero") + require.EqualValues(t, 3, *second.CancelledWriterParked) + require.EqualValues(t, 2, *second.CancelledWriteIntent) + require.EqualValues(t, 1, *second.CancelledOther) + *second.CancelledWriterParked = 999 + *second.ObservedPeakWALBytes = 999 + third := s.walReclaimObservation() + require.EqualValues(t, 3, *third.CancelledWriterParked) + require.Zero(t, *third.ObservedPeakWALBytes, "wire pointers must not mutate retained store data") + encoded, err := json.Marshal(first) + require.NoError(t, err) + var wire map[string]any + require.NoError(t, json.Unmarshal(encoded, &wire)) + require.Contains(t, wire, "observed_peak_wal_bytes") + require.Nil(t, wire["observed_peak_wal_bytes"]) + require.Equal(t, float64(0), wire["cancelled_other"], "supported zero counter remains numeric") +} + +func TestWALObservationUnsupportedAndMaskedReadersStayUnavailable(t *testing.T) { + spy := &walObservationSpy{Store: graph.New()} + var nilSpy *walObservationSpy + for _, source := range []graph.Store{nil, graph.New(), struct{ graph.Store }{spy}, nilSpy} { + got := (&Server{graph: source}).walReclaimObservation() + require.False(t, got.SnapshotAvailable) + require.NotEmpty(t, got.Reason) + require.Nil(t, got.CancelledWriterParked) + require.Nil(t, got.CancelledWriteIntent) + require.Nil(t, got.CancelledOther) + require.Nil(t, got.ObservedPeakWALBytes) + } + require.Zero(t, spy.reads.Load(), "masked capabilities must not unwrap to a physical store") +} + +func TestWALObservationPhysicalScopeAndCommonStatsGetterCensus(t *testing.T) { + stack := newViewStack(t) + spy := &walObservationSpy{Store: stack.store, stats: store_sqlite.WALReclaimStats{ + CancelledWriterParked: 5, CancelledWriteIntent: 4, CancelledOther: 3, + WALPeakObserved: true, ObservedPeakWALBytes: 123456, + }} + stack.srv.graph = spy // same physical store, snapshot spy only + var resource, tool walReclaimObservation + _, err := stack.readBoundResource(t, stack.worktreeRoot, "gortex://stats", func(ctx context.Context) ([]mcp.ResourceContents, error) { + resource = stack.srv.buildGraphStatsPayload(ctx)["wal_reclaim_observation"].(walReclaimObservation) + return nil, nil + }) + require.NoError(t, err) + require.EqualValues(t, 1, spy.reads.Load()) + _, err = stack.callWithView(t, "", "graph_stats", nil, func(ctx context.Context) (*mcp.CallToolResult, error) { + tool = stack.srv.buildGraphStatsPayload(ctx)["wal_reclaim_observation"].(walReclaimObservation) + return mcp.NewToolResultText(`{"ok":true}`), nil + }) + require.NoError(t, err) + require.EqualValues(t, 2, spy.reads.Load(), "one snapshot per common payload, no duplicate collection") + require.Equal(t, resource, tool, "a view does not scope physical WAL counters") + require.Equal(t, "physical_store_since_open", tool.Scope) + require.Equal(t, "sampled_since_store_open_lower_bound", tool.PeakScope) + require.Equal(t, "server_binding", tool.EpochScope) + require.Equal(t, stack.srv.editCPU.epoch, tool.ObservationEpoch) + require.EqualValues(t, 5, *tool.CancelledWriterParked) +} + +func TestWALObservationSharedGenerationAndResetNeverClearPeak(t *testing.T) { + t.Setenv("GORTEX_SQLITE_WAL_RECLAIM_MB", "0") + path := filepath.Join(t.TempDir(), "observations.sqlite") + store, err := store_sqlite.Open(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + store.AddBatch([]*graph.Node{{ID: "r/a.go::A", Name: "A", FilePath: "r/a.go", Kind: graph.KindFunction}}, nil) + _, walBytes := store.DBStats() // test setup primes an EXISTING successful observation + require.Positive(t, walBytes) + base := (&Server{graph: store}).walReclaimObservation() + generation := (&Server{graph: store.AtGeneration(7)}).walReclaimObservation() + require.Equal(t, base, generation, "generation handles share the same physical core observations") + require.GreaterOrEqual(t, *base.ObservedPeakWALBytes, walBytes) + // Test-only independent SQL resets this idle owned WAL. The transport itself + // must not issue SQL, observe a new size or reset the stored high-water mark. + db, err := sql.Open("sqlite", path) + require.NoError(t, err) + defer db.Close() + db.SetMaxOpenConns(1) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + _, err = db.ExecContext(ctx, "PRAGMA busy_timeout=1000") // setup allowance for an existing background reader + require.NoError(t, err) + var busy, frames, copied int + require.NoError(t, db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &frames, &copied)) + require.Zero(t, busy, "the actual idle WAL must be reset, not a refused query") + file, err := os.Stat(path + "-wal") + require.NoError(t, err) + require.Zero(t, file.Size()) + require.NotNil(t, store.GetNode("r/a.go::A"), "reset must retain the actual stored payload") + for range 3 { + after := (&Server{graph: store}).walReclaimObservation() + require.Equal(t, base, after, "TRUNCATE and stats reads cannot reset sampled peak/cause counters") + } + memory, err := store_sqlite.Open(":memory:") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, memory.Close()) }) + unobserved := (&Server{graph: memory}).walReclaimObservation() + require.True(t, unobserved.SnapshotAvailable, "counter snapshot support is distinct from WAL size observation") + require.False(t, unobserved.WALPeakObserved) + require.Nil(t, unobserved.ObservedPeakWALBytes) +} diff --git a/internal/mcp/workspace_isolation_test.go b/internal/mcp/workspace_isolation_test.go index 6801f1000..13d272a10 100644 --- a/internal/mcp/workspace_isolation_test.go +++ b/internal/mcp/workspace_isolation_test.go @@ -59,8 +59,7 @@ func newIsolationServer(t *testing.T) (srv *Server, repoA, repoB string) { reg := testRegistry() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, reg, bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") // index every configured repo - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) // index every configured repo eng := query.NewEngine(g) eng.SetSearch(bm) diff --git a/internal/mcp/workspace_root_scope_test.go b/internal/mcp/workspace_root_scope_test.go index 386e9df04..4c478f239 100644 --- a/internal/mcp/workspace_root_scope_test.go +++ b/internal/mcp/workspace_root_scope_test.go @@ -67,8 +67,7 @@ func newContainedFixture(t *testing.T) containedFixture { g := graph.New() bm := search.NewNull() mi := indexer.NewMultiIndexer(g, testRegistry(), bm, cm, zap.NewNop()) - _, err = mi.IndexScoped("", "") - require.NoError(t, err) + registerMCPFixtureRepos(t, mi, cm) eng := query.NewEngine(g) eng.SetSearch(bm) diff --git a/internal/mcp/worktree_batch_integration_test.go b/internal/mcp/worktree_batch_integration_test.go index 13cc3bea9..0c998795e 100644 --- a/internal/mcp/worktree_batch_integration_test.go +++ b/internal/mcp/worktree_batch_integration_test.go @@ -159,6 +159,16 @@ func TestWorktreeBatchFacadeGuardedTwoFileEdit(t *testing.T) { payload := lifecycleResultPayload(t, written) require.Equal(t, "committed", payload["status"]) require.Equal(t, "committed", payload["disk_status"]) + state, retained := f.srv.batchTransactions.Load("guarded-two-file") + require.True(t, retained) + ack := f.srv.batchAcknowledgement(state.(*batchTransactionState)) + require.Equal(t, "observed", ack.State) + require.NotNil(t, ack.Snapshot) + require.EqualValues(t, 3, ack.Snapshot.DiskWrite.Completed, "two physical files and one forward directory-sync envelope") + require.Equal(t, "not_applicable", ack.Snapshot.ParseGate.State) + if root == f.worktree { + require.EqualValues(t, 1, ack.Snapshot.TicketAdmission.Completed, "one actual checkout batch admission, not one per file") + } if root == f.worktree { require.Equal(t, f.checkoutID, payload["checkout_id"]) require.Equal(t, f.worktree, payload["checkout_root"]) diff --git a/internal/platform/process_cpu.go b/internal/platform/process_cpu.go new file mode 100644 index 000000000..49dbd22b0 --- /dev/null +++ b/internal/platform/process_cpu.go @@ -0,0 +1,9 @@ +package platform + +import "errors" + +// ErrProcessCPUUnavailable means this platform has no supported process CPU +// endpoint. A failed/unavailable sample must not become a measured zero. +var ErrProcessCPUUnavailable = errors.New("process CPU time unavailable on this platform") + +var errInvalidProcessCPUTime = errors.New("invalid process CPU time") diff --git a/internal/platform/process_cpu_other.go b/internal/platform/process_cpu_other.go new file mode 100644 index 000000000..d40f96aae --- /dev/null +++ b/internal/platform/process_cpu_other.go @@ -0,0 +1,10 @@ +//go:build !unix && !windows + +package platform + +import "time" + +// ProcessCPUTime reports unavailable when no process CPU endpoint is supported. +func ProcessCPUTime() (time.Duration, error) { + return 0, ErrProcessCPUUnavailable +} diff --git a/internal/platform/process_cpu_other_test.go b/internal/platform/process_cpu_other_test.go new file mode 100644 index 000000000..c0a37ff38 --- /dev/null +++ b/internal/platform/process_cpu_other_test.go @@ -0,0 +1,15 @@ +//go:build !unix && !windows + +package platform + +import ( + "errors" + "testing" +) + +func TestProcessCPUTimeUnsupportedIsUnavailable(t *testing.T) { + cpu, err := ProcessCPUTime() + if cpu != 0 || !errors.Is(err, ErrProcessCPUUnavailable) { + t.Fatalf("unsupported CPU must be unavailable: %s,%v", cpu, err) + } +} diff --git a/internal/platform/process_cpu_test.go b/internal/platform/process_cpu_test.go new file mode 100644 index 000000000..c21fed03e --- /dev/null +++ b/internal/platform/process_cpu_test.go @@ -0,0 +1,19 @@ +//go:build unix || windows + +package platform + +import "testing" + +func TestProcessCPUTimeCurrentProcessMonotonic(t *testing.T) { + before, err := ProcessCPUTime() + if err != nil { + t.Fatalf("first current-process sample: %v", err) + } + after, err := ProcessCPUTime() + if err != nil { + t.Fatalf("second current-process sample: %v", err) + } + if before < 0 || after < before { + t.Fatalf("invalid CPU progression: %s -> %s", before, after) + } +} diff --git a/internal/platform/process_cpu_unix.go b/internal/platform/process_cpu_unix.go new file mode 100644 index 000000000..2fb468ceb --- /dev/null +++ b/internal/platform/process_cpu_unix.go @@ -0,0 +1,22 @@ +//go:build unix + +package platform + +import ( + "syscall" + "time" +) + +// ProcessCPUTime returns the current process's cumulative user plus system CPU. +// It samples only the calling process, without reading files or child usage. +func ProcessCPUTime() (time.Duration, error) { + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + return 0, err + } + user, system := usage.Utime.Nano(), usage.Stime.Nano() + if user < 0 || system < 0 || user > (1<<63-1)-system { + return 0, errInvalidProcessCPUTime + } + return time.Duration(user + system), nil +} diff --git a/internal/platform/process_cpu_windows.go b/internal/platform/process_cpu_windows.go new file mode 100644 index 000000000..26f603f63 --- /dev/null +++ b/internal/platform/process_cpu_windows.go @@ -0,0 +1,29 @@ +//go:build windows + +package platform + +import ( + "time" + + "golang.org/x/sys/windows" +) + +// ProcessCPUTime returns the current process's cumulative user plus system CPU. +// Kernel/user FILETIMEs are durations in 100ns ticks, not absolute epoch dates. +func ProcessCPUTime() (time.Duration, error) { + var creation, exit, kernel, user windows.Filetime + if err := windows.GetProcessTimes(windows.CurrentProcess(), &creation, &exit, &kernel, &user); err != nil { + return 0, err + } + return processCPUFiletimes(kernel, user) +} + +func processCPUFiletimes(kernel, user windows.Filetime) (time.Duration, error) { + kernelTicks := uint64(kernel.HighDateTime)<<32 | uint64(kernel.LowDateTime) + userTicks := uint64(user.HighDateTime)<<32 | uint64(user.LowDateTime) + const maxTicks = uint64((1<<63 - 1) / 100) + if kernelTicks > maxTicks || userTicks > maxTicks-kernelTicks { + return 0, errInvalidProcessCPUTime + } + return time.Duration(kernelTicks+userTicks) * 100 * time.Nanosecond, nil +} diff --git a/internal/platform/process_cpu_windows_test.go b/internal/platform/process_cpu_windows_test.go new file mode 100644 index 000000000..07eb3c87d --- /dev/null +++ b/internal/platform/process_cpu_windows_test.go @@ -0,0 +1,26 @@ +//go:build windows + +package platform + +import ( + "errors" + "testing" + "time" + + "golang.org/x/sys/windows" +) + +func TestProcessCPUFiletimesAreDurations(t *testing.T) { + cpu, err := processCPUFiletimes(windows.Filetime{LowDateTime: 15_000_000}, windows.Filetime{LowDateTime: 25_000_000}) + if err != nil || cpu != 4*time.Second { + t.Fatalf("CPU durations: got %s, %v; want 4s", cpu, err) + } + cpu, err = processCPUFiletimes(windows.Filetime{}, windows.Filetime{}) + if err != nil || cpu != 0 { + t.Fatalf("known zero CPU: got %s, %v", cpu, err) + } + _, err = processCPUFiletimes(windows.Filetime{HighDateTime: ^uint32(0)}, windows.Filetime{}) + if !errors.Is(err, errInvalidProcessCPUTime) { + t.Fatalf("overflow must be unavailable: %v", err) + } +} diff --git a/internal/serverstack/backend.go b/internal/serverstack/backend.go index 6d4b60cc5..c728b09bb 100644 --- a/internal/serverstack/backend.go +++ b/internal/serverstack/backend.go @@ -93,6 +93,13 @@ func resolveBackendPath(in, filename string) (string, error) { return abs, nil } +// ResolveStorePath applies the same path/default/parent rules as the daemon +// stack without opening SQLite. Detached startup needs this before exec so +// SQLITE_TMPDIR is present in SQLite's process-start environment. +func ResolveStorePath(path string) (string, error) { + return resolveBackendPath(path, "store.sqlite") +} + // openSqliteBackend opens (or creates) the SQLite store at path. The // pure-Go modernc.org/sqlite driver keeps the binary CGo-free while still // getting a real query planner that drives the graph's secondary indexes. diff --git a/internal/viewmetrics/catalog.go b/internal/viewmetrics/catalog.go index 68ba98522..c57f49ea8 100644 --- a/internal/viewmetrics/catalog.go +++ b/internal/viewmetrics/catalog.go @@ -17,6 +17,18 @@ import "strings" // Series names. The prefix is the subsystem, the suffix is the unit: _total // for a counter, _seconds for a duration, and a bare noun for a gauge. const ( + // Dispatcher admission series have a fixed three-lane vocabulary. Queue + // bucket counters are disjoint upper bounds in milliseconds, not cumulative. + MCPDispatchInFlight = "views_mcp_dispatch_in_flight" + MCPDispatchQueueDepth = "views_mcp_dispatch_queue_depth" + MCPDispatchDraining = "views_mcp_dispatch_draining" + MCPDispatchCancelledHeld = "views_mcp_dispatch_cancelled_held" + MCPDispatchRefusalsTotal = "views_mcp_dispatch_refusals_total" + MCPDispatchReservedSlotsUsed = "views_mcp_dispatch_reserved_slots_used" + MCPDispatchReservedAdmissionsTotal = "views_mcp_dispatch_reserved_admissions_total" + MCPDispatchQueueWaitBucketsTotal = "views_mcp_dispatch_queue_wait_buckets_total" + MCPDispatchQueueWaitSeconds = "views_mcp_dispatch_queue_wait_seconds" + // SearchRerankAnalysisSkippedBusyTotal counts searches that omitted all analysis // ranking signals because an optional read found the store writer busy. SearchRerankAnalysisSkippedBusyTotal = "views_search_rerank_analysis_skipped_busy_total" @@ -632,7 +644,20 @@ func FallbackReasonCode(reason string) string { // catalog declares every series. It is the allow-list: a name absent from it // records nothing. +var dispatchLaneLabel = labelSpec{name: "lane", values: []string{"mutating", "fast", "slow"}} +var dispatchWaitOutcomeLabel = labelSpec{name: "outcome", values: []string{"admitted", "refused", "cancelled"}} + var catalog = map[string]spec{ + MCPDispatchInFlight: {kind: kindGauge, labels: []labelSpec{dispatchLaneLabel}}, + MCPDispatchQueueDepth: {kind: kindGauge, labels: []labelSpec{dispatchLaneLabel}}, + MCPDispatchDraining: {kind: kindGauge, labels: []labelSpec{dispatchLaneLabel}}, + MCPDispatchCancelledHeld: {kind: kindGauge, labels: []labelSpec{dispatchLaneLabel}}, + MCPDispatchRefusalsTotal: {kind: kindCounter, labels: []labelSpec{dispatchLaneLabel}}, + MCPDispatchReservedSlotsUsed: {kind: kindGauge}, + MCPDispatchReservedAdmissionsTotal: {kind: kindCounter}, + MCPDispatchQueueWaitSeconds: {kind: kindDuration, labels: []labelSpec{dispatchLaneLabel, dispatchWaitOutcomeLabel}}, + MCPDispatchQueueWaitBucketsTotal: {kind: kindCounter, labels: []labelSpec{dispatchLaneLabel, dispatchWaitOutcomeLabel, {name: "bucket_ms", values: []string{"1ms", "5ms", "10ms", "25ms", "50ms", "100ms", "250ms", "500ms", "1000ms", "2500ms", "5000ms", "10000ms", "30000ms", "inf"}}}}, + SearchRerankAnalysisSkippedBusyTotal: {kind: kindCounter}, FamilyInventorySeconds: {kind: kindDuration, labels: []labelSpec{ {name: LabelOutcome, values: []string{OutcomeOK, OutcomeError}},