diff --git a/bench/load/README.md b/bench/load/README.md index 976ad529a..484219b55 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -242,7 +242,7 @@ probes and immediate same-editing-session file/source readbacks. Session setup is outside the edit window. Future arrivals do not wait for earlier ACKs or probe replies. Canceled/unstarted edits remain in the offered denominator. -`--edit-agents N` (1 to 32) runs N fixed editors against the same single +`--edit-agents N` (1 to 32) defaults to N fixed editors against one repository. Each editor offers its own open-loop Poisson stream with the same 15 second mean; editor 0 keeps the one-editor seeding, so `--edit-agents 1` offers exactly the original plan. The streams merge by due time (ties by editor @@ -250,16 +250,38 @@ index) and the merge order is the global ordinal, so markers and mutation IDs stay keyed by ordinal and each ordinal is owned by one editor for the whole run. Plan steps and edit records name that editor. An edit and its immediate file/source readbacks go through its own editor's session; probes and receipt -status stay on the edit's separate prober session, and run-level status, stats -and the restore batch use editor 0. A blocked editor ACK does not delay another -editor's arrival. Editors share one file. `edit.write` carries whole-file +status stay on the edit's separate prober session, and run-level status and +stats use editor 0. The singleton restore batch also uses editor 0. A blocked editor ACK does not delay another +editor's arrival. In a singleton cell, editors share one file. `edit.write` carries whole-file content precomputed in ordinal order, so an out-of-order write would silently drop another editor's marker until the next write restored it; it is refused with more than one editor. An out-of-order `edit.batch` fails its `old_string` precondition and stays a failed record, and `edit.file` anchor insertions do not depend on order. The 1-edit diagnostic and -the paired no-regression cell stay one editor. Routing across several -repositories or linked worktrees is not implemented. +the paired no-regression cell stay one editor. + +`--edit-repos 6 --edit-agents 6 --agents 0 --worktrees 0` selects exactly +six distinct pinned repositories. Source checkouts that share one Git common +directory count as one repository. For linked views of one pinned primary, +use `--edit-repos 1 --worktrees 3 --edit-agents 4 --agents 0`; linked counts +3, 7 and 15 give 4, 8 and 16 total checkouts. These routed cells require 100 +global offered edits, at least one fixed editor per checkout, and ordinary Go +`add_intrafile_call` / `edit.file` / JSON fixtures. Pure readers, native and +lifecycle senders, moves and other output/tool/language cells are explicitly +unsupported for routed runs; their existing singleton cells remain separate. + +The plan pins every route's logical repository, checkout identity, commit, +file and callee. A fixed editor, its immediate readbacks, and each edit's +independent prober and receipt query stay on that checkout's cwd. Linked +checkouts require their own exact public view admission even though their +repository prefix is shared. Linked edit records retain that admitted checkout +ID; a missing or different receipt checkout ID cannot supply terminal graph +or accounting evidence. Each route restores its admitted original bytes +through its own editor and transaction after mutation outcomes settle. The +source repositories stay read-only. Route steps preserve the original merged +editor arrival stream and global marker/mutation IDs; a route with no offered +ordinal is not silently given an extra edit. Implementation and fixture tests +do not establish a completed live or reference campaign. `edit_sender` selects a native or lifecycle traffic cell; without it every plan is unchanged, and paired runs refuse it. A native write goes around MCP, so its @@ -749,3 +771,59 @@ Inspect a profile with `go tool pprof -sample_index=inuse_space -top worktrees.step3.heap-load-peak.pprof`, and what one more worktree retains with `go tool pprof -sample_index=inuse_space -top -base worktrees.step2.heap-idle.pprof worktrees.step3.heap-idle.pprof`. + +### Live checkout discovery containment + +`--scenario worktree-discovery` starts with the first two pinned repositories +in the manifest. It keeps their reading/editing sessions open while adding +`--worktrees` detached checkouts alternating the two repositories during the offered +window. Each added checkout gets separate root and subdirectory query sessions. +Use `--agents 0 --edit-agents 8` for eight combined edit/readback agents, or +`--agents 8 --edit-agents 0` for a read-only control. Editing requires the same +validated edit fixtures as the ordinary edit scenarios. + +`phase.gate.checkout_discovery` records every added root, first request, first +observed exact worktree view, and the resulting visibility lag. That lag is an +upper bound on catalog identity allocation; it does not claim an unobserved +earlier allocation time. Each cwd has its class, call/error/refusal counts, +complete-call read latency distribution and separately counted censored reads. +Attempts retain the actual session cwd and structured RPC `error_data`, including +the refused root, family, reason and retryability when supplied by the daemon. +A refusal or completed error outside a currently pending checkout fails the run. + +Run an otherwise identical `--worktrees 0` control to compare primary read p95. +Keep both reports and all refused/censored attempts. Zero refusals without any +`pending_refusals` is not evidence of fault containment under a stuck discovery. +A hung-discovery fixture, upgraded-store fixture and the reference-host runs +remain separate acceptance cells; this scenario does not manufacture them. +All measurements require an isolated host window and fresh owned clones/stores. + + +Discovery additions alternate the primary and unrelated control families. Each +new root keeps its own family query and prefix for the exact-view witness; +foreign-family capacity refusals fail the containment gate even while that +new root is pending. `--discovery-stall-first` installs a daemon-only Git shim +for the first new primary-family checkout. Its child ignores TERM and retains +stdout until explicit release or a 60-second bound. The harness releases it +before daemon shutdown and retains child/root/operation events. A missing +stall or pending-refusal observation fails this cell. + +For upgraded fixtures, `--discovery-prepare ` prepares owned +roots without opening a store or launching a daemon. Add +`--discovery-existing-worktrees 20 --discovery-existing-nested 5` to create five +in-repository and fifteen sibling checkouts of the first family. The unrelated +control is retained, so descriptor counts are two primaries plus twenty linked +checkouts; selected-family counts are one plus twenty. The descriptor records +actual paths, Git common directories, commits, source and configuration hashes, +and a random owner token. Preparation requires a separate exclusive fixture +producer to populate and upgrade the store at those exact paths. + +`--discovery-resume ` consumes that prepared root once. Lineage +must bind the descriptor, old and upgrade binary/source pins, schema-44 main +and WAL hashes, and structured normal-stop/catalog-identity receipts. Unknown, +live, changed or relocated identities are refused. The stopped immutable +snapshot remains inside the owned root; the destination store must be absent. +The existing seed copier copies only main/WAL files, then both source and +copied hashes are checked again. There is no SQL or catalog path rewriting. +These discovery-only flags preserve the startup scenario's `--seed-store` +contract. A copied fixture or synthetic parser control is not upgrade acceptance. diff --git a/bench/load/call.go b/bench/load/call.go index a9be86946..dc5a6bda3 100644 --- a/bench/load/call.go +++ b/bench/load/call.go @@ -112,18 +112,33 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[ } } } - if err == nil && callCtx.Err() != nil { - err = callCtx.Err() + // A socket deadline can fire before the context timer goroutine runs. + // Classify against the actual call deadline as well, so a clipped + // offered window does not invent a product/checkout failure. + expired := callCtx.Err() != nil + if deadline, ok := callCtx.Deadline(); ok && !returned.Before(deadline) { + expired = true + } + expiry := callCtx.Err() + if expired && expiry == nil { + expiry = context.DeadlineExceeded + } + if err == nil && expired { + err = expiry } if err == nil { err = checkToolResult(data) } a := Attempt{Tool: r.name(), Role: role, Request: hash, Agent: agent, Step: step, Slot: slot, Retry: retry, Started: start, DurationMS: float64(returned.Sub(start)) / float64(time.Millisecond), LogicalMS: float64(returned.Sub(logicalStart)) / float64(time.Millisecond), Outcome: "success"} + if session, ok := client.(*socketClient); ok { + a.CWD = session.cwd + } var remote *rpcError if err != nil { a.Outcome, a.Error = "error", err.Error() if errors.As(err, &remote) { a.Code = remote.Code + a.ErrorData = copyRPCErrorData(remote.Data) } if remote != nil && remote.Code == -32002 && (!retryTyped || remote.retryEditProbe()) { a.Outcome = "busy" @@ -135,9 +150,9 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[ a.Code = -32000 a.ExpectedOutcome = "symbol_absent" } - if callCtx.Err() != nil { + if expired { a.Outcome = terminalOutcome() - err = errors.Join(err, callCtx.Err()) + err = errors.Join(err, expiry) } if retryTyped && errors.Is(err, errEditProbeEvidenceIncomplete) { a.Outcome = "harness_incomplete" diff --git a/bench/load/client.go b/bench/load/client.go index 66f5371a5..0e5b45046 100644 --- a/bench/load/client.go +++ b/bench/load/client.go @@ -64,6 +64,7 @@ type socketClient struct { notifies []notificationObservation notifyDropped int version string + cwd string // immutable checkout supplied by the successful handshake } // Per-call values returned to its caller, never a shared last-ID cache. Times @@ -142,7 +143,7 @@ func openMCP(ctx context.Context, socket, cwd string) (*socketClient, error) { if err != nil { return nil, err } - c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, writeGate: make(chan struct{}, 1)} + c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: cwd, writeGate: make(chan struct{}, 1)} go c.readLoop() if _, err := c.call(ctx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "gortex-load", "version": "1"}}); err != nil { _ = c.Close() diff --git a/bench/load/client_test.go b/bench/load/client_test.go index 2f3c9c701..5aea18254 100644 --- a/bench/load/client_test.go +++ b/bench/load/client_test.go @@ -22,6 +22,7 @@ type stubFrame struct { Params map[string]any `json:"params"` Kind string `json:"kind"` Connection int `json:"-"` + Cwd string `json:"-"` } // A real newline-framed daemon stub exercises the public CLI's transport; @@ -78,6 +79,7 @@ func startStub(t *testing.T, handle func(stubFrame) (any, *rpcError)) string { return } frame.Connection = connectionID + frame.Cwd, _ = handshake["cwd"].(string) if frame.Kind != "" { result, _ := handle(frame) encode(map[string]any{"ok": true, "result": result}) diff --git a/bench/load/discovery_catalog.go b/bench/load/discovery_catalog.go new file mode 100644 index 000000000..c1651c1a3 --- /dev/null +++ b/bench/load/discovery_catalog.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "reflect" + "time" + + "github.com/zzet/gortex/internal/indexer" +) + +// These fields come from actual list_checkouts observations, never prepared +// topology declarations. View riders are intentionally not catalog identity. +type discoveryCatalogIdentity struct { + Root string `json:"root"` + CheckoutID string `json:"checkout_id"` + Incarnation string `json:"incarnation"` + FamilyID string `json:"family_id"` + RepoPrefix string `json:"repo_prefix"` + CommonDir string `json:"common_dir"` +} +type discoveryCatalogScope struct { + CWD string `json:"cwd"` + Overview indexer.FamiliesOverview `json:"overview"` +} +type discoveryCatalogReceipt struct { + Before []discoveryCatalogScope `json:"before"` + After []discoveryCatalogScope `json:"after"` +} + +func discoveryCatalogIdentities(scopes []discoveryCatalogScope, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) { + expected := make(map[string]discoverySeedRepo, len(repos)) + for _, repo := range repos { + expected[repo.Repo.Path] = repo + } + result := make(map[string]discoveryCatalogIdentity, len(repos)) + ids := make(map[string]string) + owners := make(map[string]string) + for _, scope := range scopes { + for _, family := range scope.Overview.Families { + if family.FamilyID == "" || family.CommonDir == "" || family.PrimaryRepoPrefix == "" { + return nil, errors.New("discovery catalog has incomplete family identity") + } + graphs := make(map[string]indexer.GraphOverview) + for _, graph := range family.Graphs { + if graph.GraphID == "" || graph.RepoPrefix == "" || graph.OwnerCheckoutID == "" { + return nil, errors.New("discovery catalog has incomplete graph identity") + } + if old, ok := graphs[graph.GraphID]; ok && old != graph { + return nil, errors.New("discovery catalog graph has conflicting identities") + } + graphs[graph.GraphID] = graph + owners[family.FamilyID+"\x00"+graph.RepoPrefix] = graph.OwnerCheckoutID + } + for _, checkout := range family.Checkouts { + repo, ok := expected[checkout.RootPath] + prefix := family.PrimaryRepoPrefix + if checkout.GraphID != "" { + prefix = graphs[checkout.GraphID].RepoPrefix + } + if !ok || checkout.CheckoutID == "" || checkout.Incarnation == "" || prefix != repo.Repo.Prefix || family.CommonDir != repo.CommonDir { + return nil, errors.New("discovery catalog checkout does not bind prepared root and family") + } + row := discoveryCatalogIdentity{Root: checkout.RootPath, CheckoutID: checkout.CheckoutID, Incarnation: checkout.Incarnation, FamilyID: family.FamilyID, RepoPrefix: prefix, CommonDir: family.CommonDir} + if old, ok := result[row.Root]; ok && old != row { + return nil, errors.New("discovery catalog root has conflicting identity") + } + if root, ok := ids[row.CheckoutID]; ok && root != row.Root { + return nil, errors.New("discovery catalog ID names multiple roots") + } + result[row.Root] = row + ids[row.CheckoutID] = row.Root + } + } + } + if len(result) != len(expected) { + return nil, errors.New("discovery catalog does not cover every prepared root") + } + for _, repo := range repos[:2] { + row := result[repo.Repo.Path] + if owners[row.FamilyID+"\x00"+row.RepoPrefix] != row.CheckoutID { + return nil, errors.New("discovery catalog primary graph has wrong checkout owner") + } + } + return result, nil +} + +func discoveryReadCatalogReceipt(root string, proof discoverySeedProof, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) { + if err := discoveryOwnedFile(root, proof.CatalogReceipt); err != nil { + return nil, err + } + hash, err := discoveryFileSHA(proof.CatalogReceipt) + if err != nil || !discoveryDigestValid(proof.CatalogReceiptSHA256, 32) || hash != proof.CatalogReceiptSHA256 { + return nil, errors.New("discovery raw catalog receipt digest mismatch") + } + var raw discoveryCatalogReceipt + if err := discoveryReadJSON(proof.CatalogReceipt, &raw); err != nil { + return nil, err + } + before, err := discoveryCatalogIdentities(raw.Before, repos) + if err != nil { + return nil, err + } + after, err := discoveryCatalogIdentities(raw.After, repos) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(before, after) { + return nil, errors.New("discovery old-to-upgraded catalog identity mapping changed") + } + return after, nil +} + +func (p *privateRun) verifyDiscoveryCatalog(ctx context.Context) (json.RawMessage, error) { + if p.discoveryCatalog == nil { + return nil, nil + } + var scopes []discoveryCatalogScope + for _, repo := range p.Repos[:2] { + callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + conn, reader, version, err := dialSocketSurface(callCtx, p.Socket, repo.Path, "mcp", "full", "hide") + if err != nil { + cancel() + return nil, err + } + client := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: repo.Path, writeGate: make(chan struct{}, 1)} + go client.readLoop() + _, err = client.call(callCtx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "discovery-lineage", "version": "1"}}) + if err == nil { + err = client.send(callCtx, map[string]any{"jsonrpc": "2.0", "method": "notifications/initialized"}, false) + } + var raw json.RawMessage + if err == nil { + raw, err = client.call(callCtx, "tools/call", map[string]any{"name": "list_checkouts", "arguments": map[string]any{"format": "json", "max_bytes": 0}}) + } + _ = client.Close() + cancel() + if err != nil { + return nil, err + } + payload, err := textPayload(raw, "content") + if err != nil { + return nil, err + } + var overview indexer.FamiliesOverview + if err = json.Unmarshal(payload, &overview); err != nil { + return nil, err + } + scopes = append(scopes, discoveryCatalogScope{CWD: repo.Path, Overview: overview}) + } + repos := make([]discoverySeedRepo, len(p.Repos)) + for i, repo := range p.Repos { + identity := p.discoveryCatalog[repo.Path] + repos[i] = discoverySeedRepo{Repo: repo, CommonDir: identity.CommonDir} + } + actual, err := discoveryCatalogIdentities(scopes, repos) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(actual, p.discoveryCatalog) { + return nil, errors.New("resumed discovery changed observed catalog checkout identity") + } + raw, err := json.Marshal(scopes) + if err == nil { + err = discoveryWriteJSON(filepath.Join(p.Root, "resumed-catalog-observed.json"), scopes) + } + return raw, err +} diff --git a/bench/load/discovery_fault.go b/bench/load/discovery_fault.go new file mode 100644 index 000000000..20b1585f6 --- /dev/null +++ b/bench/load/discovery_fault.go @@ -0,0 +1,124 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type discoveryOptions struct { + StallFirst bool + Prepare string + Resume string + ExistingWorktrees int + ExistingNested int +} + +func bindDiscoveryFlags(f *flag.FlagSet, o *discoveryOptions) { + f.StringVar(&o.Prepare, "discovery-prepare", "", "worktree-discovery: prepare owned roots and write a descriptor without launching a daemon") + f.StringVar(&o.Resume, "discovery-resume", "", "worktree-discovery: consume exact owned upgraded-store lineage at unchanged prepared roots") + f.IntVar(&o.ExistingNested, "discovery-existing-nested", 0, "discovery preparation: existing linked checkouts placed inside the primary repository") + f.IntVar(&o.ExistingWorktrees, "discovery-existing-worktrees", 0, "discovery preparation: existing linked checkouts of the first family") + f.BoolVar(&o.StallFirst, "discovery-stall-first", false, "worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound") +} +func (o discoveryOptions) validate(scenario string, count int) error { + if o.ExistingNested < 0 || o.ExistingNested > o.ExistingWorktrees { + return errors.New("discovery existing nested count must fit the existing topology") + } + if o.ExistingWorktrees < 0 { + return errors.New("discovery existing-worktree count must be nonnegative") + } + if o.Prepare != "" || o.Resume != "" || o.ExistingWorktrees != 0 || o.ExistingNested != 0 { + if scenario != checkoutDiscoveryScenario || o.Prepare != "" && o.Resume != "" || o.ExistingWorktrees != 0 && o.Prepare == "" { + return errors.New("discovery prepare/resume flags require that scenario, exclusive modes and preparation for existing topology") + } + if runtime.GOOS == "windows" { + return errors.New("owned discovery seed resume currently requires Unix owner checks") + } + } + if !o.StallFirst { + return nil + } + if scenario != checkoutDiscoveryScenario || count < 1 { + return errors.New("--discovery-stall-first requires worktree-discovery and at least one new worktree") + } + if runtime.GOOS == "windows" { + return errors.New("discovery Git stall requires Unix process ownership") + } + return nil +} + +func discoveryShellLiteral(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } + +// No shell runs while preparing this adapter. Only the owned daemon receives +// its PATH; worktree creation and all harness Git retain the real executable. +func (p *privateRun) installDiscoveryFault() (func() error, map[string]string, error) { + if runtime.GOOS == "windows" { + return nil, nil, errors.New("discovery Git stall requires Unix") + } + root, err := canonicalPath(p.Root) + if err != nil || root != p.Root { + return nil, nil, errors.New("discovery fault requires its canonical private root") + } + realGit, err := exec.LookPath("git") + if err != nil { + return nil, nil, err + } + realGit, err = filepath.Abs(realGit) + if err != nil { + return nil, nil, err + } + dir := filepath.Join(p.Root, "discovery-git-fault") + if err := os.Mkdir(dir, 0o700); err != nil { + return nil, nil, err + } + target := filepath.Join(p.Root, "repos", "discovery-linked-0-0") + release, events := filepath.Join(dir, "release"), filepath.Join(dir, "events.tsv") + script := fmt.Sprintf(`#!/bin/sh +cwd=$(pwd -P) +want_cwd=0 +operation= +for arg in "$@"; do + if [ "$want_cwd" = 1 ]; then cwd=$arg; want_cwd=0; continue; fi + case "$arg" in -C) want_cwd=1;; rev-parse|worktree) operation=$arg;; esac +done +case "$cwd" in %s|%s/*) + if [ -n "$operation" ]; then + ready=%s.ready.$$ + (trap '' TERM HUP INT + : > "$ready" + count=0 + while [ ! -f %s ] && [ "$count" -lt 60 ]; do sleep 1; count=$((count+1)); done + ) & + child=$! + while [ ! -f "$ready" ] && kill -0 "$child" 2>/dev/null; do sleep 0.01; done + printf '%%s\t%%s\t%%s\n' "$child" "$cwd" "$operation" >> %s + wait "$child" + exit 124 + fi;; +esac +exec %s "$@" +`, discoveryShellLiteral(target), discoveryShellLiteral(target), discoveryShellLiteral(events), discoveryShellLiteral(release), discoveryShellLiteral(events), discoveryShellLiteral(realGit)) + shim := filepath.Join(dir, "git") + if err := os.WriteFile(shim, []byte(script), 0o700); err != nil { + return nil, nil, err + } + env := append([]string(nil), p.Env...) + found := false + for i, entry := range env { + if strings.HasPrefix(entry, "PATH=") { + env[i] = "PATH=" + dir + string(os.PathListSeparator) + strings.TrimPrefix(entry, "PATH=") + found = true + } + } + if !found { + env = append(env, "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH")) + } + p.daemonEnv = env + return func() error { return os.WriteFile(release, []byte("release owned Git descendants\n"), 0o600) }, map[string]string{"discovery_git_shim": shim, "discovery_git_events": events, "discovery_git_release": release}, nil +} diff --git a/bench/load/discovery_fault_test.go b/bench/load/discovery_fault_test.go new file mode 100644 index 000000000..468576607 --- /dev/null +++ b/bench/load/discovery_fault_test.go @@ -0,0 +1,96 @@ +//go:build linux || darwin + +package main + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDiscoveryFaultRetainsOwnedPipeAndDoesNotAffectHarnessGit(t *testing.T) { + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + originalEnv := append([]string(nil), p.Env...) + release, files, err := p.installDiscoveryFault() + require.NoError(t, err) + t.Cleanup(func() { _ = release() }) + require.Equal(t, originalEnv, p.Env, "harness environment remains unchanged") + require.NotEqual(t, p.Env, p.daemonEnv) + require.NoError(t, p.addDiscoveryWorktree(t.Context(), 0), "harness worktree creation must never enter its daemon shim") + linked := p.Repos[len(p.Repos)-1] + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, files["discovery_git_shim"], "-C", linked.Path, "rev-parse", "--show-toplevel") + configureOwnedProcess(cmd) + cmd.Cancel = func() error { return killOwnedProcess(cmd) } + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + cmd.WaitDelay = 3 * time.Second + require.NoError(t, cmd.Start()) + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + joined := false + t.Cleanup(func() { + _ = release() + if !joined { + _ = killOwnedProcess(cmd) + <-done + } + }) + var child int + require.Eventually(t, func() bool { + raw, err := os.ReadFile(files["discovery_git_events"]) + if err != nil { + return false + } + fields := strings.Fields(string(raw)) + if len(fields) != 3 { + return false + } + child, err = strconv.Atoi(fields[0]) + return err == nil && fields[1] == linked.Path && fields[2] == "rev-parse" + }, 3*time.Second, 5*time.Millisecond) + require.NoError(t, cmd.Process.Kill(), "cancelled Git parent exits while descendant retains its stdout pipe") + require.NoError(t, syscall.Kill(child, syscall.SIGTERM), "TERM goes only to recorded owned descendant") + select { + case err := <-done: + joined = true + t.Fatalf("stall unexpectedly exited after TERM: %v", err) + case <-time.After(30 * time.Millisecond): + } + require.NoError(t, release()) + select { + case err := <-done: + joined = true + require.Error(t, err) + case <-ctx.Done(): + t.Fatal("released owned shim did not join") + } + _, err = p.git(t.Context(), p.Repos[1].Path, "rev-parse", "--show-toplevel") + require.NoError(t, err) + _, err = os.Stat(filepath.Join(p.Root, "discovery-git-fault", "release")) + require.NoError(t, err) +} + +func TestDiscoveryStallOptionsRequireActualAdditions(t *testing.T) { + base := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--agents", "2", "--edit-agents", "0", "--discovery-stall-first"} + _, err := parseRun(append(base, "--scenario", checkoutDiscoveryScenario, "--worktrees", "2")) + require.NoError(t, err) + _, err = parseRun(append(base, "--scenario", checkoutDiscoveryScenario, "--worktrees", "0")) + require.Error(t, err) + _, err = parseRun(append(base, "--scenario", "read")) + require.Error(t, err) +} diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go new file mode 100644 index 000000000..f42021c4b --- /dev/null +++ b/bench/load/discovery_scope.go @@ -0,0 +1,231 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const checkoutDiscoveryScenario = "worktree-discovery" + +type checkoutDiscoveryObservation struct { + Root string `json:"root"` + FamilyRole string `json:"family_role"` + Prefix string `json:"repo_prefix"` + Query string `json:"query"` + Added time.Time `json:"added"` + FirstRequest time.Time `json:"first_request"` + ExactAt time.Time `json:"exact_at"` + ExactViewLag Metric `json:"exact_view_lag_ms"` + View json.RawMessage `json:"view,omitempty"` + Error string `json:"error,omitempty"` +} + +type checkoutCWDCounts struct { + Class string `json:"class"` + ReadLatency Distribution `json:"read_latency_ms"` + ReadCensored int `json:"read_censored"` + Calls int `json:"calls"` + Refusals int `json:"refusals"` + Errors int `json:"errors"` + Latency Distribution `json:"latency_ms"` +} + +type checkoutDiscoveryReport struct { + ForeignCapacityRefusals int `json:"foreign_capacity_refusals"` + PendingRefusals int `json:"pending_refusals"` + Checkouts []checkoutDiscoveryObservation `json:"checkouts"` + ByCWD map[string]checkoutCWDCounts `json:"by_cwd"` + OutsidePendingRefusals int `json:"outside_pending_refusals"` + OutsidePendingErrors int `json:"outside_pending_errors"` +} + +// A refusal belongs to a pending checkout only while its discovery is +// unresolved. Component boundaries keep a sibling path out of that scope. +func pendingCheckoutForAttempt(a Attempt, checkouts []checkoutDiscoveryObservation) bool { + for _, checkout := range checkouts { + if inside(checkout.Root, a.CWD) && !a.Started.Before(checkout.Added) && + (checkout.ExactAt.IsZero() || a.Started.Before(checkout.ExactAt)) { + return true + } + } + return false +} + +func summarizeCheckoutDiscovery(attempts []Attempt, checkouts []checkoutDiscoveryObservation, primary, other string) checkoutDiscoveryReport { + out := checkoutDiscoveryReport{Checkouts: checkouts, ByCWD: make(map[string]checkoutCWDCounts)} + values := make(map[string][]float64) + reads := make(map[string][]float64) + for _, a := range attempts { + row := out.ByCWD[a.CWD] + row.Class = "outside_known_checkouts" + if inside(primary, a.CWD) { + row.Class = "primary" + } + if inside(other, a.CWD) { + row.Class = "other_repository" + } + for _, checkout := range checkouts { + if inside(checkout.Root, a.CWD) { + row.Class = "new_worktree" + if checkout.FamilyRole == "foreign" { + row.Class = "foreign_new_worktree" + if a.Code != 0 && (a.ErrorData["reason"] == "family_capacity" || a.ErrorData["reason"] == "stuck_capacity") { + out.ForeignCapacityRefusals++ + } + } + } + } + if a.Role == "workload" && !strings.HasPrefix(a.Tool, "edit.") { + if a.Outcome == "phase_censored" { + row.ReadCensored++ + } else { + reads[a.CWD] = append(reads[a.CWD], a.DurationMS) + } + } + row.Calls++ + values[a.CWD] = append(values[a.CWD], a.DurationMS) + refused := a.Code != 0 + failed := a.Outcome != "success" && a.Outcome != "phase_censored" + if refused { + row.Refusals++ + } + if failed { + row.Errors++ + } + if pendingCheckoutForAttempt(a, checkouts) && refused { + out.PendingRefusals++ + } + if !pendingCheckoutForAttempt(a, checkouts) { + if refused { + out.OutsidePendingRefusals++ + } + if failed { + out.OutsidePendingErrors++ + } + } + out.ByCWD[a.CWD] = row + } + for cwd, row := range out.ByCWD { + row.Latency = distribution(values[cwd], "ms") + row.ReadLatency = distribution(reads[cwd], "ms") + out.ByCWD[cwd] = row + } + return out +} + +// The existing readers keep their original sessions and repository assignment +// throughout the additions. Discovery probes own separate root/subdirectory +// sessions; every refusal remains an attempt instead of being retried away. +func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRun, clients []*socketClient, phase *Phase) (resultErr error) { + phase.SessionModel = "per_agent_plus_new_worktree_root_and_subdirectory_probes" + ctx, cancel := context.WithTimeout(parent, o.Duration) + defer cancel() + frozen := *p + frozen.Repos = append([]privateRepo(nil), p.Repos...) + load := Phase{} + done := make(chan error, 1) + go func() { done <- runAgents(ctx, o, &frozen, clients, o.Editors > 0, &load) }() + first := len(phase.Attempts) + var checkouts []checkoutDiscoveryObservation + var probes []*socketClient + defer func() { + cancel() + loadErr := <-done + closeClients(probes) + phase.Attempts = append(phase.Attempts, load.Attempts...) + phase.Edits = append(phase.Edits, load.Edits...) + finishEditMetrics(phase) + if len(checkouts) != o.Worktrees { + resultErr = errors.Join(resultErr, errors.New("offered window ended before all planned worktree additions")) + } + report := summarizeCheckoutDiscovery(phase.Attempts[first:], checkouts, frozen.Repos[0].Path, frozen.Repos[1].Path) + phase.Gate["checkout_discovery"], _ = json.Marshal(report) + if o.Discovery.StallFirst && report.PendingRefusals == 0 { + resultErr = errors.Join(resultErr, errors.New("git stall produced no pending refusal exposure")) + } + if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 || report.ForeignCapacityRefusals > 0 { + resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors, %d foreign capacity refusals", report.OutsidePendingRefusals, report.OutsidePendingErrors, report.ForeignCapacityRefusals)) + } + if parent.Err() != nil { + resultErr = errors.Join(resultErr, parent.Err(), loadErr) + } + }() + // Additions are spread over the offered window, including a baseline with + // --worktrees=0. Timing is recorded as observed, never inferred from count. + interval := o.Duration / time.Duration(o.Worktrees+1) + nextAdd := time.Now().Add(interval) + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return resultErr + case <-ticker.C: + } + if len(checkouts) < o.Worktrees && !time.Now().Before(nextAdd) { + family := len(checkouts) % 2 + if err := p.addDiscoveryWorktree(ctx, family); err != nil { + return err + } + repo := p.Repos[len(p.Repos)-1] + role := "primary" + if family == 1 { + role = "foreign" + } + entry := checkoutDiscoveryObservation{Root: repo.Path, FamilyRole: role, Prefix: repo.Prefix, Query: repo.Corpus.Query, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} + checkouts = append(checkouts, entry) + nextAdd = nextAdd.Add(interval) + subdir := filepath.Join(repo.Path, ".discovery-probe") + if err := os.Mkdir(subdir, 0o700); err != nil { + return err + } + for _, cwd := range []string{repo.Path, subdir} { + client, err := openMCP(ctx, p.Socket, cwd) + if err != nil { + return fmt.Errorf("open discovery session in %s: %w", cwd, err) + } + probes = append(probes, client) + } + } + for i, client := range probes { + entry := &checkouts[i/2] + if entry.FirstRequest.IsZero() { + entry.FirstRequest = time.Now() + } + r := request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": entry.Query}} + _, rows, _ := callTool(ctx, client, r, r.Arguments, "probe", i, len(checkouts), 0, o.Deadline, waitContext) + phase.Attempts = append(phase.Attempts, rows...) + if i%2 != 0 || !entry.ExactAt.IsZero() || ctx.Err() != nil { + continue + } + raw, err := control(ctx, p.Socket, "search_symbols", map[string]any{"path": entry.Root, "query": entry.Query, "limit": 1}) + entry.View = append(json.RawMessage(nil), raw...) + if err != nil { + entry.Error = err.Error() + continue + } + var answer struct { + View struct { + Exact bool `json:"exact"` + Kind string `json:"kind"` + Prefix string `json:"repo_prefix"` + } `json:"view"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + entry.Error = err.Error() + continue + } + if answer.View.Exact && answer.View.Kind == "worktree" && answer.View.Prefix == entry.Prefix { + entry.ExactAt = time.Now() + entry.ExactViewLag = measured(float64(entry.ExactAt.Sub(entry.FirstRequest))/float64(time.Millisecond), "ms") + entry.Error = "" + } + } + } +} diff --git a/bench/load/discovery_scope_test.go b/bench/load/discovery_scope_test.go new file mode 100644 index 000000000..2aee33958 --- /dev/null +++ b/bench/load/discovery_scope_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDiscoveryAttemptsRetainSessionCWDAndRefusalTarget(t *testing.T) { + cwd := filepath.Join(t.TempDir(), "pending", "subdir") + root := filepath.Dir(cwd) + details := map[string]any{"error_code": "view_building", "path": root, + "family": "family-a", "reason": "family_capacity", "state": "discovery_pending", "retryable": true} + socket := startStub(t, func(_ stubFrame) (any, *rpcError) { + return nil, &rpcError{Code: -32000, Message: "checkout discovery unavailable", Data: details} + }) + client, err := openMCP(t.Context(), socket, cwd) + require.NoError(t, err) + defer client.Close() + _, attempts, err := callTool(t.Context(), client, request{Tool: "search", Operation: "symbols"}, + map[string]any{"operation": "symbols", "query": "Target"}, "workload", 1, 0, 0, time.Second, + func(context.Context, time.Duration) error { return nil }) + require.Error(t, err) + require.Len(t, attempts, 1) + encoded, err := json.Marshal(attempts[0]) + require.NoError(t, err) + var observation map[string]any + require.NoError(t, json.Unmarshal(encoded, &observation)) + require.Equal(t, cwd, observation["cwd"], "request cwd must remain distinct from its pending checkout root") + require.Equal(t, details, observation["error_data"], "refusal scope must survive report serialization") +} + +func TestDiscoveryRefusalAccountingUsesCWDAndPendingWindow(t *testing.T) { + start := time.Unix(100, 0) + root := filepath.Join(t.TempDir(), "pending") + entries := []checkoutDiscoveryObservation{{Root: root, Added: start, ExactAt: start.Add(time.Second)}} + rows := []Attempt{ + {CWD: filepath.Join(root, "subdir"), Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", DurationMS: 2}, + {CWD: root + "-sibling", Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", DurationMS: 3}, + {CWD: root, Started: start.Add(2 * time.Second), Code: -32000, Outcome: "error", DurationMS: 4}, + {CWD: filepath.Dir(root), Started: start.Add(time.Millisecond), Outcome: "success", DurationMS: 1}, + } + report := summarizeCheckoutDiscovery(rows, entries, filepath.Dir(root), root+"-sibling") + require.Equal(t, 2, report.OutsidePendingRefusals, "a sibling and an already-published checkout are outside the pending scope") + require.Equal(t, 2, report.OutsidePendingErrors) + require.Equal(t, 1, report.ByCWD[root].Refusals) + require.Equal(t, 1, report.ByCWD[filepath.Dir(root)].Latency.Count) +} + +func TestDiscoveryScenarioOptionsHaveAnExplicitBaseline(t *testing.T) { + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", checkoutDiscoveryScenario, "--agents", "2", "--edit-agents", "0", "--worktrees", "0"} + options, err := parseRun(args) + require.NoError(t, err) + require.Equal(t, 0, options.Worktrees) + require.Equal(t, checkoutDiscoveryScenario, options.Scenario) + _, err = parseRun(append(args, "--agents", "1")) + require.Error(t, err, "both initial repositories must retain a reading session") +} + +func TestDiscoveryAddsWorktreesWhileOriginalSessionsKeepReading(t *testing.T) { + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + p.Repos[0].Prefix, p.Repos[1].Prefix = "sample", "other" + var mu sync.Mutex + starts, ends := make(map[int]time.Time), make(map[int]time.Time) + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Kind != "" { + return map[string]any{"view": map[string]any{"exact": true, "kind": "worktree", "repo_prefix": "sample"}}, nil + } + mu.Lock() + if starts[frame.Connection].IsZero() { + starts[frame.Connection] = time.Now() + } + ends[frame.Connection] = time.Now() + mu.Unlock() + return stubContent(map[string]any{"results": []any{map[string]any{"id": "sample/sample.go::Example", "name": "Example"}}}), nil + }) + var clients []*socketClient + for _, repo := range p.Repos { + c, err := openMCP(t.Context(), p.Socket, repo.Path) + require.NoError(t, err) + clients = append(clients, c) + } + defer closeClients(clients) + phase := Phase{Gate: make(map[string]json.RawMessage)} + err = measureCheckoutDiscovery(t.Context(), runOptions{Agents: 2, Worktrees: 1, Duration: 1200 * time.Millisecond, Deadline: time.Second, Profile: "saturation", Seed: 1}, p, clients, &phase) + failures := make(map[string]int) + for _, a := range phase.Attempts { + if a.Outcome != "success" && a.Outcome != "phase_censored" { + failures[a.Error]++ + } + } + require.NoError(t, err, "attempt failures: %v", failures) + var report checkoutDiscoveryReport + require.NoError(t, json.Unmarshal(phase.Gate["checkout_discovery"], &report)) + require.Len(t, report.Checkouts, 1) + added := report.Checkouts[0].Added + require.NotNil(t, report.Checkouts[0].ExactViewLag.Value) + mu.Lock() + defer mu.Unlock() + for _, conn := range []int{1, 2} { + require.True(t, starts[conn].Before(added), "original session must run before worktree add") + require.True(t, ends[conn].After(added), "original session must remain active during discovery") + } + root := report.Checkouts[0].Root + require.Positive(t, report.ByCWD[root].Calls) + require.Positive(t, report.ByCWD[filepath.Join(root, ".discovery-probe")].Calls) + require.Zero(t, report.OutsidePendingRefusals) +} + +func TestDiscoveryForeignCapacityIsNotExcusedAsPending(t *testing.T) { + start := time.Unix(100, 0) + root := filepath.Join(t.TempDir(), "foreign-new") + entries := []checkoutDiscoveryObservation{{Root: root, FamilyRole: "foreign", Added: start}} + rows := []Attempt{ + {CWD: root, Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "in_progress"}}, + {CWD: root, Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "family_capacity"}}, + {CWD: filepath.Join(root, "subdir"), Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "stuck_capacity"}}, + } + report := summarizeCheckoutDiscovery(rows, entries, root+"-primary", root+"-other") + require.Equal(t, 3, report.PendingRefusals) + require.Equal(t, 2, report.ForeignCapacityRefusals, "foreign family capacity cannot be hidden by the new-root pending window") + require.Equal(t, "foreign_new_worktree", report.ByCWD[root].Class) +} + +func TestDiscoveryNewRootsRetainEachUnrelatedFamily(t *testing.T) { + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + second.Query = "ForeignOnly" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + p.Repos[0].Prefix, p.Repos[1].Prefix = "primary", "foreign" + for _, family := range []int{0, 1, 0, 1} { + require.NoError(t, p.addDiscoveryWorktree(t.Context(), family)) + linked := p.Repos[len(p.Repos)-1] + primary := p.Repos[family] + expected, err := p.git(t.Context(), primary.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + require.NoError(t, err) + actual, err := p.git(t.Context(), linked.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + require.NoError(t, err) + require.Equal(t, string(expected), string(actual), "linked checkout must share only its selected family") + require.Equal(t, primary.Prefix, linked.Prefix) + require.Equal(t, primary.Corpus.Query, linked.Corpus.Query) + } + require.Error(t, p.addDiscoveryWorktree(t.Context(), 2)) +} diff --git a/bench/load/discovery_seed.go b/bench/load/discovery_seed.go new file mode 100644 index 000000000..cb1c6d807 --- /dev/null +++ b/bench/load/discovery_seed.go @@ -0,0 +1,381 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" +) + +type discoverySeedRepo struct { + Repo privateRepo `json:"repo"` + Role string `json:"role"` + FileSHA256 string `json:"file_sha256"` + CommonDir string `json:"git_common_dir"` +} +type discoverySeedDescriptor struct { + State string `json:"state"` + Root string `json:"owned_root"` + Nonce string `json:"owner_nonce"` + ConfigSHA256 string `json:"config_sha256"` + BootConfigSHA256 string `json:"boot_config_sha256"` + GitConfigSHA256 string `json:"git_config_sha256"` + Repos []discoverySeedRepo `json:"repositories"` + PrimaryCount int `json:"primary_checkout_count"` + LinkedCount int `json:"linked_checkout_count"` + NestedCount int `json:"nested_linked_checkout_count"` + TotalCount int `json:"total_checkout_count"` +} +type discoverySeedLineage struct { + State string `json:"state"` + Descriptor string `json:"descriptor"` + DescriptorSHA256 string `json:"descriptor_sha256"` + OwnedRoot string `json:"owned_root"` + SourceGeneration string `json:"source_generation"` + OldVersion string `json:"old_version"` + OldBinarySHA256 string `json:"old_binary_sha256"` + UpgradeSourceCommit string `json:"upgrade_source_commit"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Snapshot string `json:"snapshot"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + OwnerPID int `json:"owner_pid"` + StopReceipt string `json:"stop_receipt"` + StopReceiptSHA256 string `json:"stop_receipt_sha256"` + IdentityReceipt string `json:"identity_receipt"` + IdentityReceiptSHA256 string `json:"identity_receipt_sha256"` + Repos []discoverySeedRepo `json:"repositories"` +} + +func discoveryFileSHA(path string) (string, error) { + info, err := os.Lstat(path) + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", errors.New("discovery lineage requires regular files, not symlinks") + } + f, err := os.Open(path) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, f) + if err := errors.Join(copyErr, f.Close()); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} +func discoveryReadJSON(path string, out any) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + d := json.NewDecoder(io.LimitReader(f, 4<<20)) + d.DisallowUnknownFields() + if err := d.Decode(out); err != nil { + return err + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("discovery descriptor must contain exactly one JSON object") + } + return nil +} +func discoveryWriteJSON(path string, value any) error { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + _, err = f.Write(append(b, '\n')) + return errors.Join(err, f.Close()) +} + +// Preparation only makes owned clones/topology and metadata. The exclusive +// producer must populate and upgrade this exact root before resume can pass. +func prepareDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) error { + if len(corpora) < 2 { + return errors.New("discovery preparation requires two unrelated pinned corpora") + } + if err := validateOutputRoots(corpora, o.Discovery.Prepare); err != nil { + return err + } + if _, err := os.Lstat(o.Discovery.Prepare); !errors.Is(err, os.ErrNotExist) { + return errors.New("discovery descriptor destination must be absent") + } + p, err := newPrivateRun(ctx, o.Scratch, o.Binary, corpora[:2]) + if err != nil { + return err + } + defer os.RemoveAll(p.SocketDir) + complete := false + defer func() { + if !complete { + _ = p.removeState() + } + }() + for i := range o.Discovery.ExistingWorktrees { + if err := p.addExistingDiscoveryWorktree(ctx, i, i < o.Discovery.ExistingNested); err != nil { + return err + } + } + if _, err := p.writeDiscoveryBootRepos(); err != nil { + return err + } + token := make([]byte, 32) + if _, err := rand.Read(token); err != nil { + return err + } + d := discoverySeedDescriptor{State: "PREPARED_ROOTS_NO_STORE", Root: p.Root, Nonce: hex.EncodeToString(token), PrimaryCount: 2, LinkedCount: len(p.Repos) - 2, NestedCount: o.Discovery.ExistingNested, TotalCount: len(p.Repos)} + d.ConfigSHA256, err = discoveryFileSHA(p.Config) + if err != nil { + return err + } + d.BootConfigSHA256, err = discoveryFileSHA(filepath.Join(p.Root, "config", "gortex", "config.yaml")) + if err != nil { + return err + } + d.GitConfigSHA256, err = discoveryFileSHA(filepath.Join(p.Root, "gitconfig")) + if err != nil { + return err + } + for i, repo := range p.Repos { + role := "existing_primary_family" + repo.Prefix = p.Repos[0].Corpus.Label + if i < 2 { + role = "primary" + repo.Prefix = repo.Corpus.Label + } + sha, err := discoveryFileSHA(repo.File) + if err != nil { + return err + } + common, err := p.git(ctx, repo.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + if err != nil { + return err + } + d.Repos = append(d.Repos, discoverySeedRepo{Repo: repo, Role: role, FileSHA256: sha, CommonDir: strings.TrimSpace(string(common))}) + } + if err := discoveryWriteJSON(filepath.Join(p.Root, "discovery-owner.json"), d); err != nil { + return err + } + if err := discoveryWriteJSON(o.Discovery.Prepare, d); err != nil { + return err + } + complete = true + fmt.Printf("prepared owned discovery descriptor %s; no daemon or store was opened\n", o.Discovery.Prepare) + return nil +} + +func discoveryDigestValid(s string, n int) bool { + b, err := hex.DecodeString(s) + return err == nil && len(b) == n +} +func discoveryOwnedFile(root, path string) error { + canonical, err := canonicalPath(path) + if err != nil || canonical != path || !inside(root, path) || root == path { + return errors.New("discovery lineage path is not canonical inside its owned root") + } + _, err = discoveryFileSHA(path) + return err +} + +// No catalog/path rewriting is allowed: the snapshot was upgraded at the same +// roots that this one-shot consumer will use. All source files stay unchanged. +func resumeDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) (*privateRun, error) { + var l discoverySeedLineage + if err := discoveryReadJSON(o.Discovery.Resume, &l); err != nil { + return nil, err + } + if l.State != "VALIDATED_OWNED_UPGRADE" || l.SourceGeneration == "" || l.OldVersion == "" || l.SchemaVersion != 44 || !l.NormalStop || !l.OwnersExited || l.OwnerPID < 1 || !discoveryDigestValid(l.OldBinarySHA256, 32) || !discoveryDigestValid(l.UpgradeBinarySHA256, 32) || !discoveryDigestValid(l.UpgradeSourceCommit, 20) { + return nil, errors.New("discovery seed lacks complete validated upgrade lineage") + } + sha, err := discoveryFileSHA(l.Descriptor) + if err != nil || sha != l.DescriptorSHA256 { + return nil, errors.New("discovery descriptor digest mismatch") + } + var d discoverySeedDescriptor + if err := discoveryReadJSON(l.Descriptor, &d); err != nil { + return nil, err + } + scratch, err := canonicalPath(o.Scratch) + if err != nil { + return nil, err + } + root, err := canonicalPath(d.Root) + if err != nil || root != d.Root || root != l.OwnedRoot || !inside(scratch, root) || root == scratch || !strings.HasPrefix(filepath.Base(root), "load-") { + return nil, errors.New("discovery seed is outside its exact owned scratch root") + } + var owner discoverySeedDescriptor + if err := discoveryReadJSON(filepath.Join(root, "discovery-owner.json"), &owner); err != nil || !reflect.DeepEqual(owner, d) { + return nil, errors.New("discovery root ownership does not match descriptor") + } + if d.State != "PREPARED_ROOTS_NO_STORE" || !discoveryDigestValid(d.Nonce, 32) || d.PrimaryCount != 2 || d.LinkedCount < 0 || d.NestedCount < 0 || d.NestedCount > d.LinkedCount || d.TotalCount != 2+d.LinkedCount || len(d.Repos) != d.TotalCount || !reflect.DeepEqual(l.Repos, d.Repos) { + return nil, errors.New("discovery lineage topology/identity differs from preparation") + } + if len(corpora) != 2 || !reflect.DeepEqual(d.Repos[0].Repo.Corpus, corpora[0]) || !reflect.DeepEqual(d.Repos[1].Repo.Corpus, corpora[1]) { + return nil, errors.New("discovery seed corpus pins mismatch") + } + for path, want := range map[string]string{filepath.Join(root, "config.yaml"): d.ConfigSHA256, filepath.Join(root, "config", "gortex", "config.yaml"): d.BootConfigSHA256, filepath.Join(root, "gitconfig"): d.GitConfigSHA256, l.StopReceipt: l.StopReceiptSHA256, l.IdentityReceipt: l.IdentityReceiptSHA256} { + if err := discoveryOwnedFile(root, path); err != nil { + return nil, err + } + actual, err := discoveryFileSHA(path) + if err != nil || !discoveryDigestValid(want, 32) || actual != want { + return nil, errors.New("discovery configuration/producer proof digest mismatch") + } + } + var catalog map[string]discoveryCatalogIdentity + for _, proofPath := range []string{l.StopReceipt, l.IdentityReceipt} { + var proof discoverySeedProof + if err := discoveryReadJSON(proofPath, &proof); err != nil { + return nil, err + } + if proof.State != "OBSERVED_OWNED_UPGRADE" || proof.ObservedAt.IsZero() || proof.OwnedRoot != root || proof.OwnerPID != l.OwnerPID || !proof.NormalStop || !proof.OwnersExited || proof.MainSHA256 != l.MainSHA256 || proof.WALSHA256 != l.WALSHA256 || proof.SchemaVersion != 44 || proof.UpgradeBinarySHA256 != l.UpgradeBinarySHA256 || !reflect.DeepEqual(proof.Repos, d.Repos) { + return nil, errors.New("discovery producer proof is unknown or does not bind this stopped snapshot and identity topology") + } + observed, err := discoveryReadCatalogReceipt(root, proof, d.Repos) + if err != nil { + return nil, err + } + if catalog != nil && !reflect.DeepEqual(catalog, observed) { + return nil, errors.New("discovery producer proofs disagree on catalog identities") + } + catalog = observed + } + process, err := os.FindProcess(l.OwnerPID) + if err != nil { + return nil, err + } + err = process.Signal(syscall.Signal(0)) + _ = process.Release() + if !errors.Is(err, syscall.ESRCH) && !errors.Is(err, os.ErrProcessDone) { + return nil, errors.New("discovery upgrade owner is live or cannot be proven exited") + } + binaryPath, err := filepath.Abs(o.Binary) + if err != nil { + return nil, err + } + p := &privateRun{Root: root, Store: filepath.Join(root, "store.db"), Config: filepath.Join(root, "config.yaml"), Binary: binaryPath} + for _, entry := range d.Repos { + repo := entry.Repo + if err := discoveryOwnedFile(root, repo.File); err != nil { + return nil, err + } + if !inside(root, repo.Path) || repo.File != filepath.Join(repo.Path, repo.Corpus.File) { + return nil, errors.New("discovery repository path mismatch") + } + actual, err := discoveryFileSHA(repo.File) + if err != nil || actual != entry.FileSHA256 { + return nil, errors.New("discovery prepared source content changed") + } + p.Repos = append(p.Repos, repo) + } + p.Env = privateEnv(root, "", filepath.Join(root, "gitconfig")) + for i, repo := range p.Repos { + head, err := p.git(ctx, repo.Path, "rev-parse", "HEAD") + if err != nil || strings.TrimSpace(string(head)) != repo.Corpus.Commit { + return nil, errors.New("discovery prepared checkout HEAD changed") + } + common, err := p.git(ctx, repo.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + if err != nil || strings.TrimSpace(string(common)) != d.Repos[i].CommonDir { + return nil, errors.New("discovery prepared Git family changed") + } + } + if err := discoveryOwnedFile(root, l.Snapshot); err != nil { + return nil, err + } + for _, suffix := range []string{"", "-wal", "-shm"} { + if _, err := os.Lstat(p.Store + suffix); !errors.Is(err, os.ErrNotExist) { + return nil, errors.New("discovery destination store must be absent before one-shot resume") + } + } + before, err := discoverySeedPair(l.Snapshot) + if err != nil || before[0] != l.MainSHA256 || before[1] != l.WALSHA256 { + return nil, errors.New("discovery quiescent snapshot digest mismatch") + } + f, err := os.Open(l.Snapshot) + if err != nil { + return nil, err + } + header := make([]byte, 100) + _, readErr := io.ReadFull(f, header) + err = errors.Join(readErr, f.Close()) + if err != nil || string(header[:16]) != "SQLite format 3\x00" || binary.BigEndian.Uint32(header[60:64]) != 44 { + return nil, errors.New("discovery snapshot is not a checkpointed schema44 SQLite main") + } + if err := discoveryWriteJSON(filepath.Join(root, "discovery-consumed.json"), map[string]string{"descriptor_sha256": sha, "upgrade_binary_sha256": l.UpgradeBinarySHA256}); err != nil { + return nil, err + } + if err := copySeedStore(l.Snapshot, p.Store); err != nil { + return nil, err + } + after, err := discoverySeedPair(l.Snapshot) + if err != nil || before != after { + return nil, errors.New("discovery snapshot changed during copy; partial destination retained") + } + copied, err := discoverySeedPair(p.Store) + if err != nil || copied != before { + return nil, errors.New("discovery seed copy differs; partial destination retained") + } + p.discoveryCatalog = catalog + p.SocketDir, err = os.MkdirTemp("/tmp", "gl-") + if err != nil { + return nil, err + } + p.Socket = filepath.Join(p.SocketDir, "d.sock") + p.Env = privateEnv(root, p.Socket, filepath.Join(root, "gitconfig")) + return p, nil +} +func discoverySeedPair(path string) ([2]string, error) { + var out [2]string + var err error + if _, err := os.Lstat(path + "-shm"); !errors.Is(err, os.ErrNotExist) { + return out, errors.New("discovery snapshot has shared-memory state or unknown ownership") + } + out[0], err = discoveryFileSHA(path) + if err != nil { + return out, err + } + out[1], err = discoveryFileSHA(path + "-wal") + if errors.Is(err, os.ErrNotExist) { + err = nil + } + return out, err +} + +// Both retained producer receipts must bind observations to this exact stopped +// output; state assertions without snapshot and identity evidence are refused. +type discoverySeedProof struct { + CatalogReceipt string `json:"catalog_receipt"` + CatalogReceiptSHA256 string `json:"catalog_receipt_sha256"` + State string `json:"state"` + ObservedAt time.Time `json:"observed_at"` + OwnedRoot string `json:"owned_root"` + OwnerPID int `json:"owner_pid"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Repos []discoverySeedRepo `json:"repositories"` +} diff --git a/bench/load/discovery_seed_test.go b/bench/load/discovery_seed_test.go new file mode 100644 index 000000000..b638bcc87 --- /dev/null +++ b/bench/load/discovery_seed_test.go @@ -0,0 +1,197 @@ +//go:build linux || darwin + +package main + +import ( + "encoding/binary" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" +) + +// This is only a byte-copy/parser fixture, deliberately not a real SQLite +// database or upgrade receipt. It never establishes upgrade acceptance. +func syntheticDiscoverySeed(t *testing.T) (runOptions, []Corpus, discoverySeedLineage) { + t.Helper() + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + scratch := t.TempDir() + descriptor := filepath.Join(scratch, "prepared.json") + o := runOptions{Binary: "unused", Scratch: scratch, Discovery: discoveryOptions{Prepare: descriptor, ExistingWorktrees: 2, ExistingNested: 1}} + corpora := []Corpus{first, second} + require.NoError(t, prepareDiscoveryState(t.Context(), o, corpora)) + var d discoverySeedDescriptor + require.NoError(t, discoveryReadJSON(descriptor, &d)) + snapshot := filepath.Join(d.Root, "synthetic-byte-snapshot.db") + header := make([]byte, 100) + copy(header, "SQLite format 3\x00") + binary.BigEndian.PutUint32(header[60:64], 44) + require.NoError(t, os.WriteFile(snapshot, header, 0o600)) + mainSHA, err := discoveryFileSHA(snapshot) + require.NoError(t, err) + child := exec.Command("sh", "-c", "exit 0") + require.NoError(t, child.Run()) + l := discoverySeedLineage{State: "VALIDATED_OWNED_UPGRADE", Descriptor: descriptor, OwnedRoot: d.Root, SourceGeneration: "synthetic-test-only", OldVersion: "synthetic", OldBinarySHA256: strings.Repeat("1", 64), UpgradeSourceCommit: strings.Repeat("2", 40), UpgradeBinarySHA256: strings.Repeat("3", 64), Snapshot: snapshot, MainSHA256: mainSHA, SchemaVersion: 44, NormalStop: true, OwnersExited: true, OwnerPID: child.Process.Pid, StopReceipt: filepath.Join(d.Root, "synthetic-stop.json"), IdentityReceipt: filepath.Join(d.Root, "synthetic-identity.json"), Repos: d.Repos} + l.DescriptorSHA256, err = discoveryFileSHA(descriptor) + require.NoError(t, err) + rewriteSyntheticDiscoveryProofs(t, &l) + o.Discovery = discoveryOptions{Resume: filepath.Join(scratch, "lineage.json")} + return o, corpora, l +} +func rewriteSyntheticDiscoveryProofs(t *testing.T, l *discoverySeedLineage) { + t.Helper() + catalogPath := filepath.Join(l.OwnedRoot, "synthetic-catalog.json") + var scopes []discoveryCatalogScope + for i := 0; i < 2; i++ { + repo := l.Repos[i] + family := indexer.FamilyOverview{FamilyID: fmt.Sprintf("synthetic-family-%d", i), CommonDir: repo.CommonDir, PrimaryRepoPrefix: repo.Repo.Prefix} + family.Graphs = []indexer.GraphOverview{{GraphID: fmt.Sprintf("synthetic-graph-%d", i), RepoPrefix: repo.Repo.Prefix, OwnerCheckoutID: fmt.Sprintf("synthetic-checkout-%d", i)}} + for j, row := range l.Repos { + if row.CommonDir != repo.CommonDir { + continue + } + checkout := indexer.CheckoutOverview{CheckoutID: fmt.Sprintf("synthetic-checkout-%d", j), Incarnation: fmt.Sprintf("synthetic-incarnation-%d", j), RootPath: row.Repo.Path} + if j == i { + checkout.GraphID = family.Graphs[0].GraphID + } + family.Checkouts = append(family.Checkouts, checkout) + } + scopes = append(scopes, discoveryCatalogScope{CWD: repo.Repo.Path, Overview: indexer.FamiliesOverview{Families: []indexer.FamilyOverview{family}}}) + } + _ = os.Remove(catalogPath) + require.NoError(t, discoveryWriteJSON(catalogPath, discoveryCatalogReceipt{Before: scopes, After: scopes})) + catalogSHA, err := discoveryFileSHA(catalogPath) + require.NoError(t, err) + proof := discoverySeedProof{CatalogReceipt: catalogPath, CatalogReceiptSHA256: catalogSHA, State: "OBSERVED_OWNED_UPGRADE", ObservedAt: time.Now(), OwnedRoot: l.OwnedRoot, OwnerPID: l.OwnerPID, NormalStop: l.NormalStop, OwnersExited: l.OwnersExited, MainSHA256: l.MainSHA256, WALSHA256: l.WALSHA256, SchemaVersion: l.SchemaVersion, UpgradeBinarySHA256: l.UpgradeBinarySHA256, Repos: l.Repos} + for _, path := range []string{l.StopReceipt, l.IdentityReceipt} { + _ = os.Remove(path) + require.NoError(t, discoveryWriteJSON(path, proof)) + } + l.StopReceiptSHA256, err = discoveryFileSHA(l.StopReceipt) + require.NoError(t, err) + l.IdentityReceiptSHA256, err = discoveryFileSHA(l.IdentityReceipt) + require.NoError(t, err) +} +func TestDiscoverySeedResumeRefusesUnknownLiveAndMismatchedLineage(t *testing.T) { + for _, name := range []string{"unknown", "topology", "snapshot", "live", "shared_memory", "changed_git_family"} { + t.Run(name, func(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + switch name { + case "unknown": + l.State = "TEMPLATE" + case "topology": + l.Repos = append([]discoverySeedRepo(nil), l.Repos...) + l.Repos[2].Repo.Prefix = "foreign" + case "snapshot": + l.MainSHA256 = strings.Repeat("0", 64) + rewriteSyntheticDiscoveryProofs(t, &l) + case "live": + l.OwnerPID = os.Getpid() + rewriteSyntheticDiscoveryProofs(t, &l) + case "shared_memory": + require.NoError(t, os.WriteFile(l.Snapshot+"-shm", nil, 0o600)) + case "changed_git_family": + require.NoError(t, os.WriteFile(filepath.Join(l.Repos[2].Repo.Path, ".git"), []byte("gitdir: "+filepath.Join(l.Repos[1].Repo.Path, ".git")+"\n"), 0o600)) + } + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + _, err := resumeDiscoveryState(t.Context(), o, corpora) + require.Error(t, err) + require.NoFileExists(t, filepath.Join(l.OwnedRoot, "store.db"), "refused lineage must not create a destination store") + }) + } +} +func TestDiscoverySeedCopiesSnapshotWithoutChangingExistingIdentities(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + p, err := resumeDiscoveryState(t.Context(), o, corpora) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + require.Equal(t, l.OwnedRoot, p.Root) + require.Len(t, p.Repos, 4) + require.True(t, inside(p.Repos[0].Path, p.Repos[2].Path)) + require.False(t, inside(p.Repos[0].Path, p.Repos[3].Path)) + for i, repo := range p.Repos { + require.Equal(t, l.Repos[i].Repo.Path, repo.Path) + require.Equal(t, l.Repos[i].Repo.Prefix, repo.Prefix) + } + source, err := discoveryFileSHA(l.Snapshot) + require.NoError(t, err) + dest, err := discoveryFileSHA(p.Store) + require.NoError(t, err) + require.Equal(t, l.MainSHA256, source) + require.Equal(t, source, dest) + _, err = resumeDiscoveryState(t.Context(), o, corpora) + require.Error(t, err, "a prepared run is consumed once") +} + +func TestDiscoverySeedRefusesRekeyedCatalogBeforeCopy(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + var proof discoverySeedProof + require.NoError(t, discoveryReadJSON(l.IdentityReceipt, &proof)) + var raw discoveryCatalogReceipt + require.NoError(t, discoveryReadJSON(proof.CatalogReceipt, &raw)) + raw.After[0].Overview.Families[0].Checkouts[1].Incarnation = "synthetic-rekeyed-incarnation" + require.NoError(t, os.Remove(proof.CatalogReceipt)) + require.NoError(t, discoveryWriteJSON(proof.CatalogReceipt, raw)) + var err error + proof.CatalogReceiptSHA256, err = discoveryFileSHA(proof.CatalogReceipt) + require.NoError(t, err) + for _, path := range []string{l.StopReceipt, l.IdentityReceipt} { + require.NoError(t, os.Remove(path)) + require.NoError(t, discoveryWriteJSON(path, proof)) + } + l.StopReceiptSHA256, err = discoveryFileSHA(l.StopReceipt) + require.NoError(t, err) + l.IdentityReceiptSHA256, err = discoveryFileSHA(l.IdentityReceipt) + require.NoError(t, err) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + _, err = resumeDiscoveryState(t.Context(), o, corpora) + require.ErrorContains(t, err, "catalog identity mapping changed") + require.NoFileExists(t, filepath.Join(l.OwnedRoot, "store.db")) +} + +func TestDiscoverySeedVerifiesLiveCatalogAfterResume(t *testing.T) { + for _, rekey := range []bool{false, true} { + t.Run(fmt.Sprintf("rekey_%t", rekey), func(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + var proof discoverySeedProof + require.NoError(t, discoveryReadJSON(l.IdentityReceipt, &proof)) + var raw discoveryCatalogReceipt + require.NoError(t, discoveryReadJSON(proof.CatalogReceipt, &raw)) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + p, err := resumeDiscoveryState(t.Context(), o, corpora) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + var calls atomic.Int32 + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + calls.Add(1) + if frame.Method != "tools/call" || frame.Params["name"] != "list_checkouts" { + return nil, &rpcError{Code: -32601, Message: "unexpected synthetic call"} + } + overview := raw.After[frame.Connection-1].Overview + if rekey && frame.Connection == 1 { + overview.Families[0].Checkouts[1].CheckoutID = "unique-live-rekey" + } + return stubContent(overview), nil + }) + observed, err := p.verifyDiscoveryCatalog(t.Context()) + require.Equal(t, int32(2), calls.Load(), "both primary family scopes must be observed") + if rekey { + require.ErrorContains(t, err, "changed observed catalog checkout identity") + require.NoFileExists(t, filepath.Join(p.Root, "resumed-catalog-observed.json")) + } else { + require.NoError(t, err) + require.NotEmpty(t, observed) + require.FileExists(t, filepath.Join(p.Root, "resumed-catalog-observed.json")) + } + }) + } +} diff --git a/bench/load/discovery_worktree.go b/bench/load/discovery_worktree.go new file mode 100644 index 000000000..872fb3174 --- /dev/null +++ b/bench/load/discovery_worktree.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Only this scenario alternates the two unrelated families. Existing worktree +// scenarios keep their primary-only topology and labels. +func (p *privateRun) addDiscoveryWorktree(ctx context.Context, family int) error { + if family < 0 || family > 1 || len(p.Repos) < 2 { + return errors.New("discovery requires one of its two initial families") + } + primary := p.Repos[family] + ordinal := 0 + for _, repo := range p.Repos { + if strings.HasPrefix(repo.Corpus.Label, "discovery-linked-") { + ordinal++ + } + } + label := fmt.Sprintf("discovery-linked-%d-%d", family, ordinal) + path := filepath.Join(p.Root, "repos", label) + if _, err := p.git(ctx, primary.Path, "worktree", "add", "--detach", path, primary.Corpus.Commit); err != nil { + return err + } + file, err := ownedFile(path, primary.Corpus.File) + if err != nil { + return err + } + corpus := primary.Corpus + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Corpus: corpus, Path: path, File: file, Prefix: primary.Prefix}) + return nil +} + +func (p *privateRun) writeDiscoveryBootRepos() ([]privateRepo, error) { + if len(p.Repos) < 2 { + return nil, errors.New("discovery requires two configured primaries") + } + configured := *p + configured.Repos = p.Repos[:2] + return configured.writeBootRepos(false) +} + +func (p *privateRun) addExistingDiscoveryWorktree(ctx context.Context, ordinal int, nested bool) error { + if len(p.Repos) < 2 { + return errors.New("existing discovery topology requires two initial primaries") + } + primary := p.Repos[0] + label := fmt.Sprintf("existing-linked-%d", ordinal) + path := filepath.Join(p.Root, "repos", label) + if nested { + path = filepath.Join(primary.Path, ".load-existing", label) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + } + if _, err := p.git(ctx, primary.Path, "worktree", "add", "--detach", path, primary.Corpus.Commit); err != nil { + return err + } + file, err := ownedFile(path, primary.Corpus.File) + if err != nil { + return err + } + corpus := primary.Corpus + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Corpus: corpus, Path: path, File: file, Prefix: primary.Prefix}) + return nil +} diff --git a/bench/load/edit_probe_absence.go b/bench/load/edit_probe_absence.go index c1de43bad..946a992da 100644 --- a/bench/load/edit_probe_absence.go +++ b/bench/load/edit_probe_absence.go @@ -63,7 +63,11 @@ func (c *editProbeCaller) expectedAbsence(data json.RawMessage, identity wireCal if !proven || editStarted.IsZero() || identity.WriteStarted.IsZero() || identity.WriteStarted.Before(editStarted) { return nil } - scoped := c.runtime.p.Repos[0].Prefix + "/" + logical + repo, err := c.runtime.repoFor(c.state.step) + if err != nil { + return err + } + scoped := repo.Prefix + "/" + logical if message, matched := editProbeExactNotFound(data, scoped); matched { return &editProbeExpectedAbsence{symbol: scoped, message: message} } diff --git a/bench/load/edit_probe_checkout_receipt_test.go b/bench/load/edit_probe_checkout_receipt_test.go new file mode 100644 index 000000000..b7e2569d4 --- /dev/null +++ b/bench/load/edit_probe_checkout_receipt_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// A same-prefix view must not borrow another checkout's terminal status even +// when the mutation ID, source bytes and all graph answers otherwise match. +func TestEditProbeLinkedReceiptCannotBorrowAnotherCheckoutIdentity(t *testing.T) { + for _, cell := range []string{"matching", "foreign", "missing", "anonymous_admission", "duplicate_admission", "missing_checkout"} { + t.Run(cell, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c := editProbeCorpus() + c.Label = "fixture" + p := &privateRun{Root: t.TempDir()} + labels := []string{"fixture", "linked-1"} + if cell == "missing_checkout" { + labels = labels[:1] + } + if cell == "duplicate_admission" { + labels = append(labels, "linked-2") + } + for i, label := range labels { + root := filepath.Join(p.Root, label) + if err := os.Mkdir(root, 0700); err != nil { + t.Fatal(err) + } + content := fmt.Sprintf("package sample\n// checkout%d\nfunc Example() int { return 1 }\n%s\n", i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(content), 0600); err != nil { + t.Fatal(err) + } + corpus := c + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: "fixture", Corpus: corpus}) + } + editors, linkedCount := len(labels), len(labels)-1 + if cell == "missing_checkout" { + editors, linkedCount = 2, 1 + } + full, err := makeEditProbeTrafficPlan(c, 42, 100, editors) + if err != nil { + t.Fatal(err) + } + var selected editProbeStep + for _, step := range full.Steps { + if step.Editor == 1 { + selected = step + break + } + } + disk := editProbeDiskStub(t, c, p.Repos[len(p.Repos)-1].Path) + viewID := "expected-linked-checkout" + if cell == "anonymous_admission" { + viewID = "" + } + statusID := "expected-linked-checkout" + if cell == "foreign" { + statusID = "foreign-checkout" + } + if cell == "missing" { + statusID = "" + } + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Kind == "search_symbols" { + return map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": viewID}}, nil + } + if frame.Method != "tools/call" { + return stubContent(map[string]any{}), nil + } + args, _ := frame.Params["arguments"].(map[string]any) + if frame.Params["name"] == "edit" && args["operation"] == "file" { + data, err := os.ReadFile(p.Repos[1].File) + if err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "read failed"} + } + match, _ := args["match"].(string) + replacement, _ := args["replacement"].(string) + if match == "" || strings.Count(string(data), match) != 1 { + return nil, &rpcError{Code: -32000, Message: "exact match missing"} + } + if err := os.WriteFile(p.Repos[1].File, []byte(strings.Replace(string(data), match, replacement, 1)), 0600); err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "write failed"} + } + return stubContent(map[string]any{"mutation_receipt": "own-receipt"}), nil + } + if frame.Params["name"] == "change" && args["operation"] == "receipt" { + status := map[string]any{"found": true, "receipt": "own-receipt", "mutation_id": selected.MutationID, "graph_status_terminal": true, "graph_status": "fresh"} + if statusID != "" { + status["checkout_id"] = statusID + } + return stubContent(status), nil + } + return disk(frame) + }) + gate, err := json.Marshal(map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": viewID}}) + if err != nil { + t.Fatal(err) + } + phase := &Phase{Metrics: map[string]Metric{}, Gate: map[string]json.RawMessage{}} + for _, label := range labels[1:] { + phase.Gate[label+".view"] = gate + } + var clients []*socketClient + t.Cleanup(func() { closeClients(clients) }) + for i := range editors { + repo := p.Repos[i%len(p.Repos)] + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + clients = append(clients, client) + } + run, probers, err := prepareEditProbe(ctx, runOptions{Seed: 42, EditCount: 100, Editors: editors, Worktrees: linkedCount, Deadline: 2 * time.Second, editProbeBaseline: true}, p, nil, clients, phase) + clients = append(clients, probers...) + if run != nil { + t.Cleanup(func() { _ = run.close() }) + } + if cell == "anonymous_admission" || cell == "duplicate_admission" || cell == "missing_checkout" { + if err == nil { + t.Fatal("anonymous exact/shared-prefix view admitted without a checkout identity") + } + return + } + if err != nil { + t.Fatal(err) + } + run.origin = time.Now() + step := run.plan.Steps[selected.Ordinal] + state := run.states[step.Ordinal] + run.activate(ctx, step) + awaitEditProbeFirstProbe(t, state) + run.doEdit(ctx, step) + if state.record.CheckoutID != "expected-linked-checkout" { + t.Fatalf("record lost its admitted checkout identity: %+v", state.record) + } + if cell == "matching" { + if state.record.Outcome != "complete" || !state.record.GraphTerminal { + t.Fatalf("own checkout receipt refused: %+v", state.record) + } + } else if state.record.Outcome == "complete" || state.record.GraphTerminal || !strings.Contains(strings.Join(state.record.Errors, ";"), "checkout") { + t.Fatalf("%s checkout identity borrowed a terminal mutation: %+v", cell, state.record) + } + }) + } +} diff --git a/bench/load/edit_probe_lifecycle_test.go b/bench/load/edit_probe_lifecycle_test.go index 3a45131e5..052ae6dd0 100644 --- a/bench/load/edit_probe_lifecycle_test.go +++ b/bench/load/edit_probe_lifecycle_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "encoding/json" + "errors" "os" "path/filepath" "slices" @@ -121,6 +123,116 @@ func TestEditProbeDeleteActivatedBeforeItsCreateCompletes(t *testing.T) { } } +// A probe can enumerate the created file just before the native delete +// removes it. Its later read must observe absence, not invent a fixture IO +// failure. Prove presence first and force that exact ordering without sleeps. +func TestEditProbeDiskSnapshotFollowsCreateThenConcurrentDelete(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + created := filepath.Join(dir, plan.Steps[0].File) + removeOnRead := false + var reported []error + handle := editProbeDiskRepliesWithReadFile(c, dir, func(err error) { reported = append(reported, err) }, func(path string) ([]byte, error) { + if path == created && removeOnRead { + removeOnRead = false + if err := os.Remove(path); err != nil { + return nil, err + } + } + return os.ReadFile(path) + }) + invoke := func(oracle editProbeOracle) json.RawMessage { + t.Helper() + answer, rpcErr := handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": oracle.call.Tool, "arguments": oracle.call.Arguments}}) + if rpcErr != nil { + t.Fatal(rpcErr) + } + raw, err := json.Marshal(answer) + if err != nil { + t.Fatal(err) + } + return raw + } + if _, found := editProbeExactNotFound(invoke(editProbeOracles(c, plan.Steps[0])[0]), "fixture/"+plan.Steps[1].BeforeSymbol); !found { + t.Fatal("not-yet-created file was not exactly absent") + } + content := plan.Steps[0].Request.Arguments["content"].(string) + if err := os.WriteFile(created, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + callee := c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] + for _, oracle := range editProbeOracles(c, plan.Steps[0]) { + body, err := decodeEditProbeResult(invoke(oracle)) + if err != nil || !editProbeCorrect(oracle.name, body, c, plan.Steps[0], callee) { + t.Fatalf("create presence was not proved by %s: %v %+v", oracle.name, err, body) + } + } + removeOnRead = true + for _, oracle := range editProbeOracles(c, plan.Steps[1]) { + raw := invoke(oracle) + if oracle.name == "source" { + if _, found := editProbeExactNotFound(raw, "fixture/"+plan.Steps[1].BeforeSymbol); !found { + t.Fatal("deleted file lacks its exact not-found identity") + } + continue + } + body, err := decodeEditProbeResult(raw) + if err != nil || !editProbeCorrect(oracle.name, body, c, plan.Steps[1], callee) { + t.Fatalf("delete absence was not proved by %s: %v %+v", oracle.name, err, body) + } + } + if removeOnRead { + t.Fatal("delete did not occur between enumeration and read") + } + if _, err := os.Stat(created); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("created file remains after causal delete: %v", err) + } + if len(reported) != 0 { + t.Fatalf("causal disappearance was reported as a fixture failure: %v", reported) + } +} + +func TestEditProbeDiskSnapshotKeepsOtherReadFailures(t *testing.T) { + for _, tc := range []struct { + name, operation string + err error + }{ + {"snapshot_permission", "summary", os.ErrPermission}, + {"direct_file_missing", "file", os.ErrNotExist}, + } { + t.Run(tc.name, func(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + var reported []error + handle := editProbeDiskRepliesWithReadFile(c, dir, func(err error) { reported = append(reported, err) }, func(path string) ([]byte, error) { + return nil, &os.PathError{Op: "open", Path: path, Err: tc.err} + }) + _, rpcErr := handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": "read", "arguments": map[string]any{"operation": tc.operation, "target": map[string]any{"file": c.File}}}}) + if rpcErr != nil { + t.Fatal(rpcErr) + } + if len(reported) == 0 { + t.Fatal("fixture read failure was silently accepted") + } + for _, err := range reported { + if !errors.Is(err, tc.err) { + t.Fatalf("wrong retained read failure: %v", err) + } + } + }) + } +} + func TestEditProbeCreatedFilesRestoredByOneTimedBatch(t *testing.T) { c := editProbeCreateDeleteCorpus(t) plan, err := makeEditProbePlan(c, 42, 100) diff --git a/bench/load/edit_probe_native_test.go b/bench/load/edit_probe_native_test.go index cac71e48d..980d1aac8 100644 --- a/bench/load/edit_probe_native_test.go +++ b/bench/load/edit_probe_native_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -32,6 +33,10 @@ func editProbeDiskStub(t *testing.T, c Corpus, dir string) func(stubFrame) (any, } func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFrame) (any, *rpcError) { + return editProbeDiskRepliesWithReadFile(c, dir, report, os.ReadFile) +} + +func editProbeDiskRepliesWithReadFile(c Corpus, dir string, report func(error), readFile func(string) ([]byte, error)) func(stubFrame) (any, *rpcError) { declared := regexp.MustCompile(`func (GortexEditProbe\w+)\(`) callee := c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] return func(frame stubFrame) (any, *rpcError) { @@ -60,9 +65,16 @@ func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFra graph, edges := []any{map[string]any{"id": "fixture/" + c.Symbol}}, []any{} paths, _ := filepath.Glob(filepath.Join(dir, "*.go")) for _, path := range paths { - data, err := os.ReadFile(path) + data, err := readFile(path) + // Native delete can remove an enumerated file before this read. + // That file is absent in this reply's snapshot. Direct read.file + // calls and every other IO error still fail the fixture. + if errors.Is(err, os.ErrNotExist) { + continue + } if err != nil { report(err) + continue } file := filepath.Base(path) for _, match := range declared.FindAllStringSubmatch(string(data), -1) { @@ -82,7 +94,7 @@ func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFra rows := []any{} switch frame.Params["name"].(string) + "." + args["operation"].(string) { case "read.file": - data, err := os.ReadFile(filepath.Join(dir, file)) + data, err := readFile(filepath.Join(dir, file)) if err != nil { report(err) } diff --git a/bench/load/edit_probe_plan.go b/bench/load/edit_probe_plan.go index d10cd7249..88950a2c9 100644 --- a/bench/load/edit_probe_plan.go +++ b/bench/load/edit_probe_plan.go @@ -28,6 +28,7 @@ type editProbeStep struct { ActivateNS int64 `json:"activate_ns"` DueNS int64 `json:"due_ns"` Editor int `json:"editor,omitempty"` + Route string `json:"route,omitempty"` Marker string `json:"marker"` MutationID string `json:"mutation_id"` Request request `json:"request"` @@ -46,28 +47,42 @@ type editProbeStep struct { File string `json:"file,omitempty"` } +type editProbeRoute struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` +} + type editProbePlan struct { - Version int `json:"version"` - Arrival string `json:"arrival"` - MeanNS int64 `json:"mean_interarrival_ns"` - ProbePeriodNS int64 `json:"probe_period_ns"` - Count int `json:"count"` - Editors int `json:"editors,omitempty"` - Readers int `json:"readers,omitempty"` - NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` - Seed int64 `json:"seed"` - Kind string `json:"kind"` - Tool string `json:"tool"` - Language string `json:"language"` - Mode string `json:"mode"` - Sender string `json:"sender,omitempty"` - Repo string `json:"repo"` - Commit string `json:"commit"` - File string `json:"file"` - Callee string `json:"callee"` - SessionModel string `json:"session_model"` - NotSelected []string `json:"not_selected_not_tested"` - Steps []editProbeStep `json:"steps"` + Version int `json:"version"` + Arrival string `json:"arrival"` + MeanNS int64 `json:"mean_interarrival_ns"` + ProbePeriodNS int64 `json:"probe_period_ns"` + Count int `json:"count"` + Editors int `json:"editors,omitempty"` + Readers int `json:"readers,omitempty"` + NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` + Seed int64 `json:"seed"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` + Sender string `json:"sender,omitempty"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + SessionModel string `json:"session_model"` + NotSelected []string `json:"not_selected_not_tested"` + Steps []editProbeStep `json:"steps"` + Routes []editProbeRoute `json:"routes,omitempty"` + LinkedWorktrees int `json:"linked_worktrees,omitempty"` } type editProbeArrival struct { @@ -127,6 +142,150 @@ func planEditProbe(c Corpus, o runOptions) (editProbePlan, error) { return p, nil } +// A linked plan has one logical repository and separately identified checkouts. +// The runtime supplies the expanded private corpora; reporting starts from the +// one pinned primary. Both must describe the same immutable source fixture. +func editProbeLinkedCorpora(corpora []Corpus, linked int) ([]Corpus, error) { + if linked < 1 || linked > 15 || len(corpora) != 1 && len(corpora) != linked+1 { + return nil, errors.New("linked edit-probe routes require one primary and at most 15 linked checkouts") + } + primary := corpora[0] + out := []Corpus{primary} + for i := 1; i <= linked; i++ { + label := fmt.Sprintf("linked-%d", i) + if primary.Label == label { + return nil, errors.New("primary label collides with a linked checkout identity") + } + c := primary + c.Label = label + if len(corpora) > 1 && corpora[i] != c { + return nil, errors.New("linked checkout differs from the pinned primary fixture") + } + out = append(out, c) + } + return out, nil +} + +// Routed plans reuse each corpus's existing finite plan. Every corpus sees +// the same global ordinals/arrivals; each fixed editor takes only its own +// corpus's request at that ordinal, so no secondary stream is reseeded. +func planEditProbeRoutes(corpora []Corpus, o runOptions) (editProbePlan, error) { + if len(corpora) == 0 { + return editProbePlan{}, errors.New("edit-and-probe requires a pinned corpus") + } + if o.EditRepos < 0 { + return editProbePlan{}, errors.New("invalid edit repository count") + } + linked := o.Worktrees > 0 + if !linked && o.EditRepos <= 1 { + return planEditProbe(corpora[0], o) + } + if linked { + if o.EditRepos > 1 { + return editProbePlan{}, errors.New("six repositories and linked checkouts are separate edit-probe cells") + } + var err error + corpora, err = editProbeLinkedCorpora(corpora, o.Worktrees) + if err != nil { + return editProbePlan{}, err + } + } else if o.EditRepos != 6 || len(corpora) != 6 { + return editProbePlan{}, errors.New("six-repository planning requires exactly six distinct pinned corpora") + } + if o.Editors < len(corpora) || o.EditCount != 100 || o.Agents != 0 { + return editProbePlan{}, errors.New("routed planning requires at least one fixed editor per checkout, 100 edits and no pure readers") + } + plans := make([]editProbePlan, len(corpora)) + routes := make([]editProbeRoute, len(corpora)) + labels, sources := map[string]bool{}, map[string]bool{} + for i, c := range corpora { + if c.Label == "" || labels[c.Label] || !linked && sources[filepath.Clean(c.Source)] { + return editProbePlan{}, errors.New("six-repository routes require distinct corpus labels and source repositories") + } + labels[c.Label], sources[filepath.Clean(c.Source)] = true, true + p, err := planEditProbe(c, o) + if err != nil { + return editProbePlan{}, fmt.Errorf("route %s: %w", c.Label, err) + } + if p.Kind != "add_intrafile_call" || p.Tool != "edit.file" || p.Language != "go" || p.Mode != "json" || p.Sender != "" { + return editProbePlan{}, errors.New("six-repository planning currently supports ordinary Go add_intrafile_call/edit.file JSON fixtures only; native, lifecycle, move and other output cells are unsupported") + } + plans[i] = p + logicalRepo := c.Label + if linked { + logicalRepo = corpora[0].Label + } + routes[i] = editProbeRoute{ID: c.Label, Repo: logicalRepo, Commit: c.Commit, File: p.File, Callee: p.Callee, Kind: p.Kind, Tool: p.Tool, Language: p.Language, Mode: p.Mode} + } + out := plans[0] + out.Routes = routes + if linked { + out.LinkedWorktrees = o.Worktrees + } + out.Steps = append([]editProbeStep(nil), out.Steps...) + for i, step := range out.Steps { + route := step.Editor % len(plans) + out.Steps[i] = plans[route].Steps[i] + out.Steps[i].Route = routes[route].ID + } + out.NotSelected = nil + for _, limitation := range plans[0].NotSelected { + if limitation != "several editors share one repository; 6-repository and linked-worktree routing are not implemented" { + out.NotSelected = append(out.NotSelected, limitation) + } + } + out.NotSelected = append(out.NotSelected, "routed cells support ordinary Go add_intrafile_call/edit.file JSON only; reference and live campaign acceptance remains separate") + return out, nil +} + +func editProbeRouteOperations(plan editProbePlan, corpora []Corpus) ([]request, error) { + if len(corpora) == 0 { + return nil, errors.New("edit-and-probe operations require a pinned corpus") + } + if len(plan.Routes) == 0 { + for _, step := range plan.Steps { + if step.Route != "" { + return nil, errors.New("edit route has no pinned route metadata") + } + } + return editProbeOperations(plan, corpora[0]), nil + } + if plan.LinkedWorktrees > 0 { + var err error + corpora, err = editProbeLinkedCorpora(corpora, plan.LinkedWorktrees) + if err != nil { + return nil, err + } + } + if len(plan.Routes) != len(corpora) { + return nil, errors.New("edit route corpus denominator differs from pinned metadata") + } + byRoute := make(map[string]Corpus, len(corpora)) + for i, c := range corpora { + route := plan.Routes[i] + logicalRepo := c.Label + if plan.LinkedWorktrees > 0 { + logicalRepo = corpora[0].Label + } + if _, duplicate := byRoute[route.ID]; duplicate || route.ID != c.Label || route.Repo != logicalRepo || route.Commit != c.Commit || route.File != c.File { + return nil, errors.New("edit route metadata differs from its pinned corpus") + } + byRoute[route.ID] = c + } + var operations []request + for _, step := range plan.Steps { + c, found := byRoute[step.Route] + if !found || step.Route == "" || step.Editor < 0 || step.Editor >= plan.Editors || step.Route != plan.Routes[step.Editor%len(plan.Routes)].ID { + return nil, errors.New("offered edit has an unknown or foreign repository route") + } + operations = append(operations, step.Request) + for _, oracle := range editProbeOracles(c, step) { + operations = append(operations, oracle.call) + } + } + return operations, nil +} + func makeEditProbeTrafficPlan(c Corpus, seed int64, count, editors int) (editProbePlan, error) { if err := validateCorpus(c); err != nil { return editProbePlan{}, err diff --git a/bench/load/edit_probe_route_phase_test.go b/bench/load/edit_probe_route_phase_test.go new file mode 100644 index 000000000..320303e0d --- /dev/null +++ b/bench/load/edit_probe_route_phase_test.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// This is startup/layout admission followed by an intentional pre-edit stop. +// Every offered edit stays censored; no live mutation/capacity claim is made. +func TestEditProbeMultiRootPhaseAdmitsOnlyItsPlannedOwnedCheckouts(t *testing.T) { + for _, cell := range []struct { + name string + repos, linked, editors int + }{ + {"six_repositories", 6, 0, 6}, {"four_total_worktrees", 1, 3, 4}, + } { + t.Run(cell.name, func(t *testing.T) { + t.Setenv("LOAD_DAEMON_FIXTURE", "serve") + var corpora []Corpus + var originals []string + for i := range cell.repos { + c := gitFixture(t) + c.Label = fmt.Sprintf("route-%d", i) + c.EditAnchor = "func Caller() int { return Example() }" + c.EditTemplate = "\nfunc {{marker}}() int { return Example() }\n" + content := fmt.Sprintf("package sample\n// owner-%d\nfunc Example() int { return 1 }\n%s\n", i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(c.Source, c.File), []byte(content), 0600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", c.File}, {"commit", "--quiet", "-m", "routing fixture"}, {"rev-parse", "HEAD"}} { + cmd := exec.Command("git", append([]string{"-C", c.Source, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "core.hooksPath=" + os.DevNull}, args...)...) + data, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v %s", args, err, data) + } + if args[0] == "rev-parse" { + c.Commit = strings.TrimSpace(string(data)) + } + } + corpora = append(corpora, c) + originals = append(originals, content) + } + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + o := runOptions{Binary: binary, Profile: "agent", Host: "native", Editors: cell.editors, EditCount: 100, EditRepos: cell.repos, Worktrees: cell.linked, Seed: 42, Duration: time.Second, Deadline: time.Second, WarmTimeout: 10 * time.Second, StopTimeout: 5 * time.Second} + stop := errors.New("layout admitted; intentional diagnostic stop before edits") + called := false + o.pairedDriver = func(_ context.Context, p *privateRun, _, _ []*socketClient, phase *Phase) error { + called = true + run := phase.editProbeRuntime + total := cell.repos + cell.linked + if run == nil || len(p.Repos) != total || len(run.plan.Routes) != total || len(run.originals) != total { + return fmt.Errorf("planned checkout denominator lost: private=%d runtime=%v", len(p.Repos), run != nil) + } + data, err := os.ReadFile(filepath.Join(p.Root, "config", "gortex", "config.yaml")) + if err != nil { + return err + } + var config struct { + Repos []json.RawMessage `json:"repos"` + } + if err := json.Unmarshal(data, &config); err != nil { + return err + } + expectedBoot := cell.repos + if len(config.Repos) != expectedBoot { + return fmt.Errorf("boot configured %d roots, want%d", len(config.Repos), expectedBoot) + } + for _, step := range run.plan.Steps { + repo, err := run.repoFor(step) + if err != nil { + return err + } + expected := p.Repos[step.Editor%len(p.Repos)] + if repo.Path != expected.Path || run.editorFor(step).cwd != expected.Path || run.states[step.Ordinal].client.cwd != expected.Path { + return fmt.Errorf("ordinal%d fell back from its owned checkout", step.Ordinal) + } + } + for i, repo := range p.Repos { + current, err := os.ReadFile(repo.File) + if err != nil || string(run.originals[repo.Path]) != string(current) { + return fmt.Errorf("checkout%d original capture missing: %v", i, err) + } + if cell.linked > 0 && i > 0 { + var receipt struct { + View struct { + Kind string `json:"kind"` + Exact bool `json:"exact"` + Prefix string `json:"repo_prefix"` + } `json:"view"` + } + if err := json.Unmarshal(phase.Gate[repo.Corpus.Label+".view"], &receipt); err != nil || !receipt.View.Exact || receipt.View.Kind != "worktree" || receipt.View.Prefix != repo.Prefix { + return fmt.Errorf("checkout%d has no exact public worktree admission", i) + } + } + } + return stop + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + phase, _, artifacts, _, _, err := runPhase(ctx, o, editProbeScenario, corpora, "", t.TempDir()) + if !called || !errors.Is(err, stop) { + t.Fatalf("owned routed phase was not admitted: called=%v err=%v", called, err) + } + if phase.EditProbe == nil || len(phase.EditProbe.Records) != 100 { + t.Fatal("diagnostic stop lost offered denominator") + } + for _, record := range phase.EditProbe.Records { + if record.Outcome != "phase_censored" || record.Ack.Value != nil { + t.Fatal("pre-edit layout diagnostic fabricated work", record) + } + } + forced := phase.Metrics["force_killed"] + if forced.Value == nil || *forced.Value != 0 { + t.Fatal("layout fixture failed normal owned stop", phase.Metrics) + } + if _, err := os.Stat(filepath.Join(artifacts["private_root"], "repos")); !os.IsNotExist(err) { + t.Fatal("private checkout handles/state retained", err) + } + for i, c := range corpora { + actual, err := os.ReadFile(filepath.Join(c.Source, c.File)) + if err != nil || string(actual) != originals[i] { + t.Fatal("source fixture was mutated", i, err) + } + } + }) + } +} + +func TestEditProbeSourceRepositoryIdentityDoesNotCountLinkedViewsTwice(t *testing.T) { + primary := gitFixture(t) + before, err := os.ReadFile(filepath.Join(primary.Source, primary.File)) + if err != nil { + t.Fatal(err) + } + linkedPath := filepath.Join(t.TempDir(), "linked-source") + cmd := exec.CommandContext(t.Context(), "git", "-C", primary.Source, "-c", "core.hooksPath="+os.DevNull, "-c", "gc.auto=0", "-c", "maintenance.auto=false", "-c", "core.autocrlf=true", "-c", "core.eol=crlf", "worktree", "add", "--detach", linkedPath, primary.Commit) + if data, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("own source worktree: %v %s", err, data) + } + // Git can materialize the same tree with different line endings. Capture + // each checkout before admission; equality to the primary's bytes would + // falsely blame a read-only query for Git's checkout normalization. + linkedBefore, err := os.ReadFile(filepath.Join(linkedPath, primary.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(linkedBefore, []byte("\r\n")) || bytes.Equal(linkedBefore, before) { + t.Fatal("fixture must distinguish the linked CRLF checkout from the primary LF source") + } + originals := map[string][]byte{primary.Source: before, linkedPath: linkedBefore} + linked := primary + linked.Source, linked.Label = linkedPath, "linked-source" + if err := validateEditProbeSourceRepos(t.Context(), []Corpus{primary, linked}, t.TempDir()); err == nil || !strings.Contains(err.Error(), "distinct source repositories") { + t.Fatalf("two source views masqueraded as independent repositories: %v", err) + } + independent := gitFixture(t) + if err := validateEditProbeSourceRepos(t.Context(), []Corpus{primary, independent}, t.TempDir()); err != nil { + t.Fatalf("independent source repositories refused: %v", err) + } + for _, c := range []Corpus{primary, linked} { + after, err := os.ReadFile(filepath.Join(c.Source, c.File)) + if err != nil || !bytes.Equal(after, originals[c.Source]) { + t.Fatalf("read-only identity admission mutated %s source bytes: %v", c.Label, err) + } + } +} diff --git a/bench/load/edit_probe_route_readback_test.go b/bench/load/edit_probe_route_readback_test.go new file mode 100644 index 000000000..b63ca0037 --- /dev/null +++ b/bench/load/edit_probe_route_readback_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// Two selected ordinals are a unit diagnostic, never a hundred-edit campaign. +// The stub performs actual owned writes and binds each receipt to its cwd. +func TestEditProbeRoutedReadbacksAndRestoreKeepEachCheckoutOriginal(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + p := &privateRun{Root: t.TempDir()} + originals := make([]string, 2) + steps := make([]editProbeStep, 2) + replies := make(map[string]func(stubFrame) (any, *rpcError)) + for i := range 2 { + c := editProbeCorpus() + c.Label = fmt.Sprintf("checkout-%d", i) + callee := fmt.Sprintf("Example%d", i) + c.Symbol = c.File + "::" + callee + c.EditAnchor = "func Caller() int { return " + callee + "() }" + c.EditTemplate = "\nfunc {{marker}}() int { return " + callee + "() }\n" + root := filepath.Join(p.Root, c.Label) + if err := os.Mkdir(root, 0700); err != nil { + t.Fatal(err) + } + originals[i] = fmt.Sprintf("package sample\n// sentinel-%d\nfunc %s() int { return %d }\n%s\n", i, callee, i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(originals[i]), 0600); err != nil { + t.Fatal(err) + } + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: "fixture", Corpus: c}) + plan, err := makeEditProbeTrafficPlan(c, 42, 100, 2) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + if step.Editor == i { + steps[i] = step + break + } + } + replies[root] = editProbeDiskStub(t, c, root) + } + var mu sync.Mutex + receipts := make(map[string]string) + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + repo := -1 + for i, r := range p.Repos { + if frame.Cwd == r.Path { + repo = i + break + } + } + if repo < 0 { + return nil, &rpcError{Code: -32000, Message: "unknown checkout"} + } + args, _ := frame.Params["arguments"].(map[string]any) + options, _ := args["options"].(map[string]any) + if frame.Params["name"] == "edit" && args["operation"] == "file" { + target, _ := args["target"].(map[string]any) + if target["file"] != "fixture/"+p.Repos[repo].Corpus.File { + return nil, &rpcError{Code: -32000, Message: "foreign file"} + } + data, err := os.ReadFile(p.Repos[repo].File) + if err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "read failed"} + } + match, _ := args["match"].(string) + replacement, _ := args["replacement"].(string) + if match == "" || strings.Count(string(data), match) != 1 { + return nil, &rpcError{Code: -32000, Message: "exact match missing"} + } + if err := os.WriteFile(p.Repos[repo].File, []byte(strings.Replace(string(data), match, replacement, 1)), 0600); err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "write failed"} + } + mutation, _ := options["mutation_id"].(string) + receipt := fmt.Sprintf("checkout-%d-%s", repo, mutation) + mu.Lock() + receipts[frame.Cwd+"|"+receipt] = mutation + mu.Unlock() + return stubContent(map[string]any{"mutation_receipt": receipt}), nil + } + if frame.Params["name"] == "change" && args["operation"] == "receipt" { + receipt, _ := options["receipt"].(string) + mu.Lock() + mutation := receipts[frame.Cwd+"|"+receipt] + mu.Unlock() + if mutation == "" { + return nil, &rpcError{Code: -32000, Message: "receipt belongs to another checkout"} + } + return stubContent(map[string]any{"graph_status_terminal": true, "graph_status": "fresh", "mutation_id": mutation}), nil + } + return replies[frame.Cwd](frame) + }) + phase := &Phase{Metrics: map[string]Metric{}} + var clients []*socketClient + t.Cleanup(func() { closeClients(clients) }) + for _, repo := range p.Repos { + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + clients = append(clients, client) + } + run, probers, err := prepareEditProbe(ctx, runOptions{Seed: 42, EditCount: 100, Editors: 2, Deadline: time.Second, editProbeBaseline: true}, p, nil, clients, phase) + clients = append(clients, probers...) + if run != nil { + t.Cleanup(func() { _ = run.close() }) + } + if err != nil { + t.Fatal(err) + } + run.origin = time.Now() + selected := make([]*editProbeState, 2) + for i, step := range steps { + state := run.states[step.Ordinal] + // Corpus-specific requests are independently pinned by the planner + // test. This diagnostic selects one real ordinal per checkout. + state.step, state.record = step, newEditProbeRecord(p.Repos[i].Corpus, step) + state.record.Errors = nil + selected[i] = state + } + phase.EditProbe.Records = nil + for i, state := range selected { + raw, _, err := run.perform(ctx, state, run.editorFor(state.step), state.step.Request, "edit", "workload", 0) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil { + t.Fatal(err) + } + receipt, _ := body["mutation_receipt"].(string) + status := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": receipt}}} + raw, _, err = run.perform(ctx, state, state.client, status, "status", "probe", 1) + if err != nil { + t.Fatal(err) + } + body, err = decodeEditProbeResult(raw) + if err != nil || body["mutation_id"] != state.step.MutationID || body["graph_status_terminal"] != true { + t.Fatalf("own receipt missing: %v %v", body, err) + } + state.record.Receipt, state.record.GraphTerminal = receipt, true + _, correct := run.immediateReadbacks(ctx, state, state.step, []string{"file", "source"}, nil) + if !correct { + t.Errorf("checkout%d same-session readbacks used another fixture/callee: %+v", i, state.record.Readbacks) + } + foreign := selected[1-i].client + if _, _, err := run.perform(ctx, state, foreign, status, "status", "probe", 2); err == nil { + t.Error("foreign checkout session borrowed a receipt") + } + phase.EditProbe.Records = append(phase.EditProbe.Records, state.record) + } + foreignReceipt := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": selected[1].record.Receipt}}} + if _, _, err := run.perform(ctx, selected[0], selected[0].client, foreignReceipt, "status", "probe", 3); err == nil { + t.Fatal("same-prefix foreign receipt borrowed another checkout's mutation") + } + // A completed sibling cannot justify rewriting an unknown/pending route. + beforeRestore := make([][]byte, len(p.Repos)) + for i, repo := range p.Repos { + beforeRestore[i], err = os.ReadFile(repo.File) + if err != nil { + t.Fatal(err) + } + } + phase.EditProbe.Records[1].GraphTerminal = false + if err := run.restore(ctx); err == nil { + t.Fatal("pending route permitted unsafe cleanup") + } + for i, repo := range p.Repos { + actual, err := os.ReadFile(repo.File) + if err != nil || string(actual) != string(beforeRestore[i]) { + t.Fatalf("pending cleanup changed checkout%d: %v", i, err) + } + } + phase.EditProbe.Records[1].GraphTerminal = true + if err := run.restore(ctx); err != nil { + t.Fatal(err) + } + var restores []struct { + Route string `json:"route"` + Editor int `json:"editor"` + } + if err := json.Unmarshal(phase.EditProbe.Restore, &restores); err != nil || len(restores) != 2 { + t.Fatalf("restore evidence lost a checkout: %s %v", phase.EditProbe.Restore, err) + } + for i, row := range restores { + if row.Route != p.Repos[i].Corpus.Label || row.Editor != i { + t.Fatalf("restore borrowed another checkout's owner: %+v", row) + } + } + for i, repo := range p.Repos { + actual, err := os.ReadFile(repo.File) + if err != nil { + t.Fatal(err) + } + if string(actual) != originals[i] { + t.Errorf("checkout%d original not restored through its own view: %q", i, actual) + } + } +} diff --git a/bench/load/edit_probe_routes.go b/bench/load/edit_probe_routes.go new file mode 100644 index 000000000..91e780830 --- /dev/null +++ b/bench/load/edit_probe_routes.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// Multi-checkout cleanup never combines common-prefix paths into editor 0's +// view. Each admitted original has its own editor session and transaction. +func (r *editProbeRuntime) restoreRoutes(ctx context.Context) (resultErr error) { + if r.plan.Kind != "add_intrafile_call" || r.plan.Tool != "edit.file" || r.plan.Mode != "json" || r.plan.Sender != "" { + return errors.New("routed restore supports ordinary add_intrafile_call/edit.file JSON only") + } + type restoredRoute struct { + Route string `json:"route"` + Editor int `json:"editor"` + File string `json:"file"` + Response json.RawMessage `json:"response"` + Error string `json:"error,omitempty"` + } + var rows []restoredRoute + started := time.Now() + defer func() { + r.phase.EditProbe.RestoreMS = measured(float64(time.Since(started))/float64(time.Millisecond), "ms") + data, err := json.Marshal(rows) + r.phase.EditProbe.Restore = data + resultErr = errors.Join(resultErr, err) + }() + for ordinal, repo := range r.p.Repos { + original, found := r.originals[repo.Path] + if !found { + return errors.New("routed restore has no admitted original for checkout") + } + editor := -1 + for i, client := range r.editors { + if client != nil && filepath.Clean(client.cwd) == filepath.Clean(repo.Path) { + editor = i + break + } + } + if editor < 0 { + return errors.New("routed restore has no editor for checkout") + } + file := filepath.Join(repo.Path, filepath.FromSlash(repo.Corpus.File)) + current, err := os.ReadFile(file) + if err != nil { + return err + } + changes := []any{map[string]any{"op": "edit_file", "path": repo.Prefix + "/" + repo.Corpus.File, "old_string": string(current), "new_string": string(original)}} + req := request{Tool: "edit", Operation: "batch", Arguments: map[string]any{"operation": "batch", "changes": changes, "options": map[string]any{"transaction_id": fmt.Sprintf("load-restore-%016x-route-%d", uint64(r.options.Seed), ordinal)}, "output": map[string]any{"format": "json", "max_bytes": 0}}} + data, _, err := callEditProbeTool(ctx, r.editors[editor], req, req.Arguments, "cleanup", ordinal, len(r.states), 0, r.options.Deadline, editProbeRetrySleep(r.options.Seed, len(r.states), ordinal)) + row := restoredRoute{Route: repo.Corpus.Label, Editor: editor, File: repo.Corpus.File, Response: data} + if err == nil { + var restored []byte + restored, err = os.ReadFile(file) + if err == nil && string(restored) != string(original) { + err = errors.New("public routed restore did not restore its checkout's original bytes") + } + } + if err != nil { + row.Error = err.Error() + } + rows = append(rows, row) + if err != nil { + return err + } + } + return nil +} + +// Distinct source repositories are a physical admission check, not a path in +// the workload hash. Read-only git metadata cannot turn linked source views +// into six independent repositories or touch any source working bytes. +func validateEditProbeSourceRepos(ctx context.Context, corpora []Corpus, outputRoot string) error { + reader := &privateRun{Root: outputRoot, Env: append(os.Environ(), "GIT_OPTIONAL_LOCKS=0")} + var seen []os.FileInfo + for _, c := range corpora { + data, err := reader.git(ctx, c.Source, "rev-parse", "--git-common-dir") + if err != nil { + return err + } + common := strings.TrimSpace(string(data)) + if !filepath.IsAbs(common) { + common = filepath.Join(c.Source, common) + } + common, err = canonicalPath(common) + if err != nil { + return err + } + identity, err := os.Stat(common) + if err != nil { + return err + } + for _, prior := range seen { + if os.SameFile(prior, identity) { + return errors.New("six-repository execution requires six distinct source repositories; linked checkouts share one repository") + } + } + seen = append(seen, identity) + } + return nil +} diff --git a/bench/load/edit_probe_routes_test.go b/bench/load/edit_probe_routes_test.go new file mode 100644 index 000000000..022aef95f --- /dev/null +++ b/bench/load/edit_probe_routes_test.go @@ -0,0 +1,309 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestEditProbeSixRepositoryRoutesAreExplicitAndAffectPlanIdentity(t *testing.T) { + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", "6", "--worktrees", "0", "--edit-count", "100", "--seed", "42", "--edit-repos", "6"} + o, err := parseRun(args) + if err != nil { + t.Fatalf("explicit six-repository edit cell refused: %v", err) + } + corpora := editProbeRoutingCorpora() + report, err := newRunReport(o, corpora, strings.Repeat("a", 40), Host{}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(report.Workload.EditProbe) + if err != nil { + t.Fatal(err) + } + var plan struct { + Routes []struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + } `json:"routes"` + Steps []struct { + Ordinal int `json:"ordinal"` + Editor int `json:"editor"` + Route string `json:"route"` + DueNS int64 `json:"due_ns"` + Marker string `json:"marker"` + MutationID string `json:"mutation_id"` + Request request `json:"request"` + } `json:"steps"` + } + if err := json.Unmarshal(data, &plan); err != nil { + t.Fatal(err) + } + if len(plan.Routes) != 6 || len(plan.Steps) != 100 { + t.Fatalf("selected route/offer denominator lost: routes=%d steps=%d", len(plan.Routes), len(plan.Steps)) + } + for i, route := range plan.Routes { + c := corpora[i] + if route.ID != c.Label || route.Repo != c.Label || route.Commit != c.Commit || route.File != c.File || route.Callee != strings.TrimPrefix(c.Symbol, c.File+"::") { + t.Fatalf("route%d lost its pinned fixture identity: %+v", i, route) + } + } + original, err := makeEditProbeTrafficPlan(corpora[0], 42, 100, 6) + if err != nil { + t.Fatal(err) + } + seenRoutes, seenMutations := map[string]bool{}, map[string]bool{} + for ordinal, step := range plan.Steps { + if step.Editor < 0 || step.Editor >= 6 { + t.Fatalf("invalid fixed editor: %+v", step) + } + c := corpora[step.Editor] + replacement, _ := step.Request.Arguments["replacement"].(string) + if step.Ordinal != ordinal || step.Route != c.Label || step.DueNS != original.Steps[ordinal].DueNS || !strings.Contains(replacement, strings.TrimPrefix(c.Symbol, c.File+"::")+"()") || step.Marker == "" || step.MutationID == "" || seenMutations[step.MutationID] { + t.Fatalf("offer was rerouted, reshaped or lost ownership: %+v", step) + } + seenRoutes[step.Route], seenMutations[step.MutationID] = true, true + } + if len(seenRoutes) != 6 { + t.Fatal("selected repository has no offered edits", seenRoutes) + } + // A secondary repository's pin belongs to the offered workload identity; + // it cannot be ignored merely because the primary fixture is unchanged. + changed := append([]Corpus(nil), corpora...) + changed[3].Commit = strings.Repeat("b", 40) + other, err := newRunReport(o, changed, report.GitSHA, Host{}, t.TempDir()) + if err != nil || other.RequestPlan == report.RequestPlan { + t.Fatalf("secondary route pin absent from plan identity: %v", err) + } +} + +func TestEditProbeSelectedCheckoutsKeepProberSessionsOnTheirEditorRoute(t *testing.T) { + for _, sharedPrefix := range []bool{false, true} { + name := "distinct_repositories" + if sharedPrefix { + name = "shared_prefix_checkouts" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + p := &privateRun{Root: t.TempDir()} + for i := 0; i < 2; i++ { + c := editProbeCorpus() + c.Label = fmt.Sprintf("route-%d", i) + root := filepath.Join(p.Root, "repos", fmt.Sprintf("route-%d", i)) + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + content := fmt.Sprintf("package sample\n// route%d\nfunc Example() int { return %d }\n%s\n", i, i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + prefix := c.Label + if sharedPrefix { + prefix, c.Label = "fixture", "fixture" + if i > 0 { + c.Label = fmt.Sprintf("linked-%d", i) + } + } + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: prefix, Corpus: c}) + } + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + target, _ := args["target"].(map[string]any) + return stubContent(map[string]any{"cwd": frame.Cwd, "symbol": target["symbol"]}), nil + }) + var editors []*socketClient + for _, repo := range p.Repos { + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + editors = append(editors, client) + } + t.Cleanup(func() { closeClients(editors) }) + phase := &Phase{Metrics: map[string]Metric{}, Gate: map[string]json.RawMessage{}} + if sharedPrefix { + view, err := json.Marshal(map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": "socket-linked-checkout"}}) + if err != nil { + t.Fatal(err) + } + phase.Gate["linked-1.view"] = view + } + options := runOptions{Seed: 42, EditCount: 100, Editors: 2, Deadline: time.Second} + if sharedPrefix { + options.Worktrees = 1 // one linked checkout in addition to primary + } + runtime, probers, err := prepareEditProbe(ctx, options, p, nil, editors, phase) + if runtime != nil { + t.Cleanup(func() { _ = runtime.close() }) // prepared edits intentionally never run + } + t.Cleanup(func() { closeClients(probers) }) + if err != nil || runtime == nil || len(probers) != 100 { + t.Fatalf("selected route setup unavailable: probers=%d err=%v", len(probers), err) + } + cwd := func(client *socketClient) string { + t.Helper() + raw, err := client.call(ctx, "tools/call", map[string]any{"name": "read", "arguments": map[string]any{"operation": "source"}}) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil { + t.Fatal(err) + } + value, _ := body["cwd"].(string) + return value + } + for _, state := range runtime.states { + want := p.Repos[state.step.Editor].Path + if actual := cwd(editors[state.step.Editor]); actual != want { + t.Fatalf("editor control is on the wrong checkout: got=%s want=%s", actual, want) + } + if actual := cwd(state.client); actual != want { + t.Fatalf("edit%d prober fell back to the primary checkout: editor=%d got=%s want=%s", state.step.Ordinal, state.step.Editor, actual, want) + } + repo := p.Repos[state.step.Editor] + req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": repo.Corpus.Symbol}}} + raw, _, err := runtime.perform(ctx, state, state.client, req, "route_source", "probe", 0) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil || body["cwd"] != want || body["symbol"] != repo.Prefix+"/"+repo.Corpus.Symbol { + t.Fatalf("same-view prober was dispatched with another repository's target: edit%d body=%v err=%v", state.step.Ordinal, body, err) + } + } + }) + } +} + +func TestEditProbeUnknownEditorCheckoutNeverFallsBackToPrimary(t *testing.T) { + run := &editProbeRuntime{ + p: &privateRun{Repos: []privateRepo{{Path: "/owned/primary", Prefix: "shared"}, {Path: "/owned/linked", Prefix: "shared"}}}, + editors: []*socketClient{{cwd: "/owned/primary"}, {cwd: "/owned/linked"}, {cwd: "/foreign"}, nil}, + } + for _, editor := range []int{-1, 2, 3, 4} { + if repo, err := run.repoFor(editProbeStep{Editor: editor}); err == nil { + t.Fatalf("unknown editor %d inherited route %+v", editor, repo) + } + } + for editor := range 2 { + repo, err := run.repoFor(editProbeStep{Editor: editor}) + if err != nil || repo.Path != run.editors[editor].cwd { + t.Fatalf("registered shared-prefix checkout lost: editor=%d repo=%+v err=%v", editor, repo, err) + } + } + state := &editProbeState{step: editProbeStep{Editor: 1}} + if _, _, err := run.perform(t.Context(), state, run.editors[0], request{}, "status", "probe", 0); err == nil { + t.Fatal("foreign primary session accepted for linked-checkout evidence") + } + for i, label := range []string{"primary", "linked"} { + run.p.Repos[i].Corpus = Corpus{Label: label, Commit: fmt.Sprintf("%040x", i+1), File: "main.go"} + c := run.p.Repos[i].Corpus + run.plan.Routes = append(run.plan.Routes, editProbeRoute{ID: label, Repo: label, Commit: c.Commit, File: c.File}) + } + for _, route := range []string{"", "unknown", "primary"} { + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: route}); err == nil { + t.Fatalf("invalid offered route %q inherited a checkout", route) + } + } + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: "linked"}); err != nil { + t.Fatal(err) + } + run.plan.Routes[1].Commit = strings.Repeat("f", 40) + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: "linked"}); err == nil { + t.Fatal("foreign pin inherited registered checkout") + } +} + +func editProbeRoutingCorpora() []Corpus { + corpora := make([]Corpus, 6) + for i := range corpora { + c := editProbeCorpus() + c.Label = fmt.Sprintf("route-%d", i) + c.Source = fmt.Sprintf("/private/route-%d", i) + c.Commit = fmt.Sprintf("%040x", i+1) + callee := fmt.Sprintf("Example%d", i) + c.Symbol = c.File + "::" + callee + c.EditAnchor = "func Caller() int { return " + callee + "() }" + c.EditTemplate = "\nfunc {{marker}}() int { return " + callee + "() }\n" + corpora[i] = c + } + return corpora +} + +func TestEditProbeRoutedPlansRejectMissingOrForeignOwnership(t *testing.T) { + o := runOptions{Scenario: editProbeScenario, Seed: 42, EditCount: 100, Editors: 6, EditRepos: 6} + for _, name := range []string{"empty", "too_few", "extra", "duplicate_label", "duplicate_source", "too_few_editors", "readers", "worktrees", "native", "cli"} { + t.Run(name, func(t *testing.T) { + corpora, options := editProbeRoutingCorpora(), o + switch name { + case "empty": + corpora = nil + case "too_few": + corpora = corpora[:5] + case "extra": + corpora = append(corpora, corpora[0]) + case "duplicate_label": + corpora[1].Label = corpora[0].Label + case "duplicate_source": + corpora[1].Source = corpora[0].Source + case "too_few_editors": + options.Editors = 5 + case "readers": + options.Agents = 1 + case "worktrees": + options.Worktrees = 1 + case "native": + corpora[1].EditSender = editProbeShellAfterTool + case "cli": + corpora[1].EditOutput = "cli" + } + if _, err := planEditProbeRoutes(corpora, options); err == nil { + t.Fatal("invalid route selection accepted") + } + }) + } + for _, name := range []string{"missing", "unknown", "foreign_editor", "foreign_pin", "missing_metadata"} { + t.Run(name, func(t *testing.T) { + corpora := editProbeRoutingCorpora() + plan, err := planEditProbeRoutes(corpora, o) + if err != nil { + t.Fatal(err) + } + switch name { + case "missing": + plan.Steps[0].Route = "" + case "unknown": + plan.Steps[0].Route = "unknown" + case "foreign_editor": + plan.Steps[0].Route = plan.Routes[(plan.Steps[0].Editor+1)%len(plan.Routes)].ID + case "foreign_pin": + corpora[1].Commit = strings.Repeat("f", 40) + case "missing_metadata": + plan.Routes = nil + } + if _, err := editProbeRouteOperations(plan, corpora); err == nil { + t.Fatal("missing or foreign route inherited primary operations") + } + }) + } +} + +func TestEditProbeRoutedPhaseRefusesUnsupportedTrafficBeforeAllocatingOwner(t *testing.T) { + corpora := editProbeRoutingCorpora() + corpora[1].EditOutput = "cli" + o := runOptions{Scenario: editProbeScenario, EditRepos: 6, Editors: 6, EditCount: 100, Seed: 42} + phase, _, artifacts, _, _, err := runPhase(t.Context(), o, editProbeScenario, corpora, "", t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "unsupported") || artifacts["private_root"] != "" || phase.editProbeRuntime != nil { + t.Fatalf("unsupported routed traffic allocated an execution owner: err=%v artifacts=%v", err, artifacts) + } +} diff --git a/bench/load/edit_probe_run.go b/bench/load/edit_probe_run.go index c6f9596f8..eaa1b3232 100644 --- a/bench/load/edit_probe_run.go +++ b/bench/load/edit_probe_run.go @@ -31,6 +31,8 @@ type editProbeReadback struct { type editProbeRecord struct { Ordinal int `json:"ordinal"` Editor int `json:"editor,omitempty"` + Route string `json:"route,omitempty"` + CheckoutID string `json:"checkout_id,omitempty"` Marker string `json:"marker"` MutationID string `json:"mutation_id"` PlannedDueNS int64 `json:"planned_due_ns"` @@ -123,6 +125,8 @@ type editProbeRuntime struct { modeCounts map[string]editProbeModeCounts states []*editProbeState original []byte + originals map[string][]byte // routed checkout path -> independently admitted original bytes + checkoutIDs map[string]string // linked checkout path -> exact admitted public view identity moveOriginal []byte origin time.Time mu sync.Mutex // Phase.Attempts, reads and mode counts only; request lifetimes are never serialized @@ -133,7 +137,7 @@ type editProbeRuntime struct { } func newEditProbeRecord(c Corpus, step editProbeStep) editProbeRecord { - record := editProbeRecord{Ordinal: step.Ordinal, Editor: step.Editor, Marker: step.Marker, MutationID: step.MutationID, PlannedDueNS: step.DueNS, Outcome: "phase_censored", Ack: unavailable("ms", "edit not sent"), PhasesMS: editProbePhases(nil, "no receipt status observed"), Readback: unavailable("ms", "edit not acknowledged"), Derived: unavailable("ms", "no receipt status observed"), FirstCorrect: make(map[string]Metric), Accounting: editProbeAccounting{State: "not_observed"}, Errors: []string{"scenario never reached edit dispatch"}} + record := editProbeRecord{Ordinal: step.Ordinal, Editor: step.Editor, Route: step.Route, Marker: step.Marker, MutationID: step.MutationID, PlannedDueNS: step.DueNS, Outcome: "phase_censored", Ack: unavailable("ms", "edit not sent"), PhasesMS: editProbePhases(nil, "no receipt status observed"), Readback: unavailable("ms", "edit not acknowledged"), Derived: unavailable("ms", "no receipt status observed"), FirstCorrect: make(map[string]Metric), Accounting: editProbeAccounting{State: "not_observed"}, Errors: []string{"scenario never reached edit dispatch"}} for _, oracle := range editProbeOracles(c, step) { record.FirstCorrect[oracle.name] = unavailable("ms", "not observed") } @@ -185,14 +189,84 @@ func newEditProbeReport(plan editProbePlan, c Corpus, required bool) *editProbeR return out } +// The editor's successful handshake owns its checkout. Prefixes are shared by +// linked worktrees and cannot identify a view. A singleton has one unambiguous +// route; with several checkouts an unknown session never inherits the primary. +func (r *editProbeRuntime) repoFor(step editProbeStep) (privateRepo, error) { + if len(r.p.Repos) == 1 { + return r.p.Repos[0], nil // standalone receipt collectors need no editor session + } + if step.Editor < 0 || step.Editor >= len(r.editors) { + return privateRepo{}, fmt.Errorf("edit %d has unknown editor %d", step.Ordinal, step.Editor) + } + editor := r.editors[step.Editor] + if editor == nil { + return privateRepo{}, fmt.Errorf("edit %d has no editor session", step.Ordinal) + } + var offered *editProbeRoute + for i := range r.plan.Routes { + if r.plan.Routes[i].ID == step.Route { + offered = &r.plan.Routes[i] + break + } + } + if len(r.plan.Routes) > 0 && (offered == nil || step.Route == "" || r.plan.Routes[step.Editor%len(r.plan.Routes)].ID != step.Route) || len(r.plan.Routes) == 0 && step.Route != "" { + return privateRepo{}, errors.New("edit has an unknown or foreign offered route") + } + for _, repo := range r.p.Repos { + if editor.cwd != "" && filepath.Clean(editor.cwd) == filepath.Clean(repo.Path) { + logicalRepo := repo.Corpus.Label + if r.plan.LinkedWorktrees > 0 { + logicalRepo = r.plan.Repo + } + if offered != nil && (offered.ID != repo.Corpus.Label || offered.Repo != logicalRepo || offered.Commit != repo.Corpus.Commit || offered.File != repo.Corpus.File) { + return privateRepo{}, errors.New("editor checkout differs from its offered route pin") + } + return repo, nil + } + } + return privateRepo{}, fmt.Errorf("edit %d editor checkout is not a registered route", step.Ordinal) +} + +func (r *editProbeRuntime) calleeFor(repo privateRepo) string { + if len(r.p.Repos) == 1 { + return r.plan.Callee + } + selected := repo.Corpus.Symbol + if repo.Corpus.EditKind == "add_crossfile_call" { + selected = repo.Corpus.EditCallee + } + separator := strings.LastIndex(selected, "::") + if separator < 0 { + return "" + } + return selected[separator+2:] +} + func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, editors []*socketClient, phase *Phase) (*editProbeRuntime, []*socketClient, error) { - plan, err := planEditProbe(p.Repos[0].Corpus, o) + corpora := make([]Corpus, len(p.Repos)) + for i, repo := range p.Repos { + corpora[i] = repo.Corpus + } + plan, err := planEditProbeRoutes(corpora, o) if err != nil { return nil, nil, err } + if len(plan.Routes) > 0 && len(plan.Routes) != len(p.Repos) { + return nil, nil, errors.New("offered edit routes differ from registered private checkout count") + } + if len(p.Repos) > 1 { + for _, repo := range p.Repos { + c := repo.Corpus + if plan.Kind != "add_intrafile_call" || plan.Tool != "edit.file" || plan.Mode != "json" || plan.Sender != "" || filepath.Ext(c.File) != ".go" || c.EditSender != "" || c.EditKind != "" && c.EditKind != "add_intrafile_call" || c.EditTool != "" && c.EditTool != "edit.file" || c.EditOutput != "" && c.EditOutput != "json" { + return nil, nil, errors.New("routed edit-probe runtime currently supports ordinary Go add_intrafile_call/edit.file JSON fixtures only") + } + } + } if len(editors) != max(1, plan.Editors) || len(readers) != plan.Readers { return nil, nil, fmt.Errorf("edit-and-probe plan has %d fixed editors and %d readers but %d editor and %d reader sessions", max(1, plan.Editors), plan.Readers, len(editors), len(readers)) } + r := &editProbeRuntime{options: o, p: p, phase: phase, plan: plan, editors: editors, readers: readers, requireAccounting: !o.editProbeBaseline} original, err := os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(plan.File))) if err != nil { return nil, nil, err @@ -220,7 +294,52 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, return nil, nil, errors.New("planned marker already exists before edit") } } - r := &editProbeRuntime{options: o, p: p, phase: phase, plan: plan, editors: editors, readers: readers, original: original, requireAccounting: !o.editProbeBaseline} + r.original = original + phase.EditProbe = newEditProbeReport(plan, p.Repos[0].Corpus, r.requireAccounting) + if len(p.Repos) > 1 { + r.originals = make(map[string][]byte, len(p.Repos)) + for _, repo := range p.Repos { + data, readErr := os.ReadFile(filepath.Join(repo.Path, filepath.FromSlash(repo.Corpus.File))) + if readErr != nil { + return r, nil, readErr + } + if err := editProbeWriteAdmission(repo.Corpus, data); err != nil { + return r, nil, err + } + if strings.Count(string(data), repo.Corpus.EditAnchor) != 1 { + return r, nil, errors.New("each routed edit-probe anchor must occur exactly once") + } + for _, step := range plan.Steps { + if strings.Contains(string(data), step.Marker) { + return r, nil, errors.New("planned marker already exists in a routed fixture") + } + } + r.originals[repo.Path] = data + } + } + if plan.LinkedWorktrees > 0 { + r.checkoutIDs = make(map[string]string, plan.LinkedWorktrees) + seen := map[string]bool{} + for _, repo := range p.Repos[1:] { + var admitted struct { + View *struct { + Kind string `json:"kind"` + Exact bool `json:"exact"` + Prefix string `json:"repo_prefix"` + ID string `json:"checkout_id"` + } `json:"view"` + } + if err := json.Unmarshal(phase.Gate[repo.Corpus.Label+".view"], &admitted); err != nil || admitted.View == nil || !admitted.View.Exact || admitted.View.Kind != "worktree" || admitted.View.Prefix != repo.Prefix || strings.TrimSpace(admitted.View.ID) == "" { + return r, nil, errors.New("linked checkout has no exact admitted public checkout identity") + } + id := admitted.View.ID + if seen[id] { + return r, nil, errors.New("linked checkouts share an admitted checkout identity") + } + seen[id] = true + r.checkoutIDs[repo.Path] = id + } + } if plan.Tool == "refactor.move" { r.moveOriginal, err = os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(p.Repos[0].Corpus.EditDestination))) if err != nil { @@ -230,7 +349,6 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, return r, nil, errors.New("move destination bytes differ from offered fixture") } } - phase.EditProbe = newEditProbeReport(plan, p.Repos[0].Corpus, r.requireAccounting) if plan.Sender == editProbeRestart { r.restarter = editProbeDaemonRestarter{p: p} } @@ -247,7 +365,12 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, var clients []*socketClient setup := time.Now() for _, step := range plan.Steps { - record := newEditProbeRecord(p.Repos[0].Corpus, step) + repo, err := r.repoFor(step) + if err != nil { + return r, nil, err + } + record := newEditProbeRecord(repo.Corpus, step) + record.CheckoutID = r.checkoutIDs[repo.Path] record.Outcome = "not_started" record.Errors = nil state := &editProbeState{step: step, record: record} @@ -255,13 +378,17 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, state.record.Evidence = filepath.Join(dir, fmt.Sprintf("edit-%03d.jsonl", step.Ordinal)) } for _, state := range r.states { + repo, err := r.repoFor(state.step) + if err != nil { + return r, clients, err + } sink, err := os.OpenFile(state.record.Evidence, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return r, clients, err } state.sink = sink state.startRaw() - state.client, err = openMCP(ctx, p.Socket, p.Repos[0].Path) + state.client, err = openMCP(ctx, p.Socket, repo.Path) if err != nil { return r, clients, err } @@ -270,11 +397,11 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, // paired delete is sent; it cannot be admitted here. if state.step.BeforeSymbol != "" && state.step.File == "" { req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": state.step.BeforeSymbol}, "options": map[string]any{"context_lines": 0}}} - raw, identity, readErr := state.client.callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, p.Repos[0])}) + raw, identity, readErr := state.client.callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, repo)}) if readErr == nil { readErr = checkToolResult(raw) } - proven := readErr == nil && editProbePresenceProof(raw, p.Repos[0].Corpus, state.step.BeforeSymbol) + proven := readErr == nil && editProbePresenceProof(raw, repo.Corpus, state.step.BeforeSymbol) if err := state.emit(map[string]any{"event": "before_presence_admission", "symbol": state.step.BeforeSymbol, "wire_id": identity.ID, "write_started": identity.WriteStarted, "finished": identity.Finished, "payload": raw, "proven": proven}); err != nil { return r, clients, err } @@ -447,6 +574,13 @@ func (r *editProbeRuntime) perform(ctx context.Context, s *editProbeState, clien } func (r *editProbeRuntime) performObserved(ctx context.Context, s *editProbeState, client *socketClient, req request, purpose, role string, stream int) (json.RawMessage, wireCallIdentity, []Attempt, error) { + repo, routeErr := r.repoFor(s.step) + if routeErr != nil { + return nil, wireCallIdentity{}, nil, routeErr + } + if len(r.p.Repos) > 1 && (client == nil || filepath.Clean(client.cwd) != filepath.Clean(repo.Path)) { + return nil, wireCallIdentity{}, nil, errors.New("edit-probe request session belongs to another checkout") + } budget := r.options.Deadline if strings.HasPrefix(purpose, "oracle:") || purpose == "status" { budget = min(budget, 20*time.Second) @@ -455,7 +589,7 @@ func (r *editProbeRuntime) performObserved(ctx context.Context, s *editProbeStat budget = min(budget, time.Until(end)) } caller := &editProbeCaller{runtime: r, state: s, client: client, purpose: purpose, request: req, role: role} - data, attempts, err := callEditProbeTool(ctx, caller, req, dispatchArgs(req, r.p.Repos[0]), role, 0, s.step.Ordinal, stream, budget, editProbeRetrySleep(r.options.Seed, s.step.Ordinal, stream)) + data, attempts, err := callEditProbeTool(ctx, caller, req, dispatchArgs(req, repo), role, 0, s.step.Ordinal, stream, budget, editProbeRetrySleep(r.options.Seed, s.step.Ordinal, stream)) r.mu.Lock() r.phase.Attempts = append(r.phase.Attempts, attempts...) r.mu.Unlock() @@ -486,14 +620,18 @@ func (r *editProbeRuntime) activateProbeState(step editProbeStep, cancel context } func (r *editProbeRuntime) observeProbe(s *editProbeState, oracle editProbeOracle, sent time.Time, data json.RawMessage, identity wireCallIdentity, err error, pending bool) error { + repo, routeErr := r.repoFor(s.step) + if routeErr != nil { + return routeErr + } ended := identity.Finished if !identity.WriteStarted.IsZero() { sent = identity.WriteStarted } body, decodeErr := decodeEditProbeOutput(data, r.probeOutput(oracle), oracle.call.name()) absence, exactRefusal := err.(*editProbeExpectedAbsence) - expectedAbsence := oracle.call.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+oracle.call.ExpectedAbsentSymbol - correct := expectedAbsence || err == nil && decodeErr == nil && editProbeCorrect(oracle.name, body, r.p.Repos[0].Corpus, s.step, r.plan.Callee) + expectedAbsence := oracle.call.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == repo.Prefix+"/"+oracle.call.ExpectedAbsentSymbol + correct := expectedAbsence || err == nil && decodeErr == nil && editProbeCorrect(oracle.name, body, repo.Corpus, s.step, r.calleeFor(repo)) fresh := decodeErr == nil && editProbeClaimsFresh(body) stale, precommit := false, false s.mu.Lock() @@ -535,7 +673,15 @@ func (r *editProbeRuntime) activate(ctx context.Context, step editProbeStep) { } func (r *editProbeRuntime) probe(probeCtx context.Context, cancel context.CancelFunc, s *editProbeState) { - for index, oracle := range editProbeOracles(r.p.Repos[0].Corpus, s.step) { + repo, err := r.repoFor(s.step) + if err != nil { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + cancel() + return + } + for index, oracle := range editProbeOracles(repo.Corpus, s.step) { s.probes.Go(func() { ticker := time.NewTicker(editProbePeriod) defer ticker.Stop() @@ -767,6 +913,12 @@ func (r *editProbeRuntime) finishEditAfterACK(ctx, callCtx context.Context, s *e } if statusErr == nil { if status, decodeErr := decodeEditProbeResult(raw); decodeErr == nil { + if expected := s.record.CheckoutID; expected != "" && status["checkout_id"] != expected { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "status checkout identity does not match the admitted linked view") + s.mu.Unlock() + break // Foreign terminal/accounting fields cannot settle this edit. + } accounting := checkEditProbeAccountingForTool(status, receipt, r.plan.Tool) s.mu.Lock() s.record.Accounting = accounting @@ -843,8 +995,15 @@ func (r *editProbeRuntime) finishEditAfterACK(ctx, callCtx context.Context, s *e // Observation-only kinds are recorded but neither gate the readback time nor // fail the record. func (r *editProbeRuntime) immediateReadbacks(callCtx context.Context, s *editProbeState, step editProbeStep, readbackKinds []string, observeOnly map[string]bool) (time.Time, bool) { + repo, err := r.repoFor(step) + if err != nil { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + return time.Time{}, false + } readbackEnd, readbacksCorrect := time.Time{}, true - file := r.plan.File + file := repo.Corpus.File if step.File != "" { file = step.File } @@ -876,12 +1035,12 @@ func (r *editProbeRuntime) immediateReadbacks(callCtx context.Context, s *editPr begin := time.Now() raw, identity, readErr := r.perform(callCtx, s, r.editorFor(step), req, "readback:"+kind, "readback", 100+index) payload, decodeErr := decodeEditProbeOutput(raw, editProbeRequestOutput(req), req.name()) - back := editProbeReadback{Kind: kind, Latency: unavailable("ms", "response endpoint unavailable"), Correct: readErr == nil && decodeErr == nil && editProbeCorrect(kind, payload, r.p.Repos[0].Corpus, step, r.plan.Callee)} + back := editProbeReadback{Kind: kind, Latency: unavailable("ms", "response endpoint unavailable"), Correct: readErr == nil && decodeErr == nil && editProbeCorrect(kind, payload, repo.Corpus, step, r.calleeFor(repo))} if !identity.Finished.IsZero() { back.Latency = measured(float64(identity.Finished.Sub(begin))/float64(time.Millisecond), "ms") } absence, exactRefusal := readErr.(*editProbeExpectedAbsence) - if req.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+req.ExpectedAbsentSymbol { + if req.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == repo.Prefix+"/"+req.ExpectedAbsentSymbol { back.Correct = true back.ExpectedOutcome = "symbol_absent" back.Refusal = absence.message @@ -1049,6 +1208,9 @@ func (r *editProbeRuntime) restore(ctx context.Context) error { return errors.New("restore batch unavailable while mutation outcome is unknown/pending") } } + if len(r.p.Repos) > 1 { + return r.restoreRoutes(ctx) + } current, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) if err != nil { return err diff --git a/bench/load/edit_probe_worktree_plan_test.go b/bench/load/edit_probe_worktree_plan_test.go new file mode 100644 index 000000000..6942bf526 --- /dev/null +++ b/bench/load/edit_probe_worktree_plan_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "encoding/json" + "fmt" + "testing" +) + +func TestEditProbeLinkedCheckoutPlansKeepPrimaryPinAndBothCounts(t *testing.T) { + for _, linked := range []int{3, 7, 15} { + t.Run(fmt.Sprintf("total-%d", linked+1), func(t *testing.T) { + total := linked + 1 + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", fmt.Sprint(total), "--worktrees", fmt.Sprint(linked), "--edit-count", "100", "--seed", "42"} + options, err := parseRun(args) + if err != nil { + t.Fatalf("explicit linked-checkout plan refused: %v", err) + } + corpus := editProbeCorpus() + report, err := newRunReport(options, []Corpus{corpus}, corpus.Commit, Host{}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(report.Workload.EditProbe) + if err != nil { + t.Fatal(err) + } + var plan struct { + Routes []struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + } `json:"routes"` + Steps []struct { + Editor int `json:"editor"` + Route string `json:"route"` + DueNS int64 `json:"due_ns"` + } `json:"steps"` + } + if err := json.Unmarshal(data, &plan); err != nil { + t.Fatal(err) + } + if report.Workload.LinkedWorktrees != linked || len(plan.Routes) != total || len(plan.Steps) != 100 { + t.Fatalf("primary/linked denominator lost: workload=%d routes=%d steps=%d", report.Workload.LinkedWorktrees, len(plan.Routes), len(plan.Steps)) + } + ids := make([]string, total) + for i, route := range plan.Routes { + expected := corpus.Label + if i > 0 { + expected = fmt.Sprintf("linked-%d", i) + } + if route.ID != expected || route.Repo != corpus.Label || route.Commit != corpus.Commit || route.File != corpus.File { + t.Fatalf("checkout%d lost logical primary pin: %+v", i, route) + } + ids[i] = route.ID + } + reference, err := makeEditProbeTrafficPlan(corpus, 42, 100, total) + if err != nil { + t.Fatal(err) + } + seen := make([]int, total) + expectedCounts := make([]int, total) + for i, step := range plan.Steps { + if step.Editor < 0 || step.Editor >= total || step.Route != ids[step.Editor] || step.DueNS != reference.Steps[i].DueNS { + t.Fatalf("checkout changed its fixed-editor offer: %+v", step) + } + seen[step.Editor]++ + expectedCounts[reference.Steps[i].Editor]++ + } + for editor := range total { + if seen[editor] != expectedCounts[editor] { + t.Fatalf("editor%d offered denominator changed: actual%d expected%d", editor, seen[editor], expectedCounts[editor]) + } + } + }) + } +} + +func TestEditProbeLinkedCheckoutPlansRefuseAmbiguousPrimaryOwnership(t *testing.T) { + options := runOptions{Scenario: editProbeScenario, Seed: 42, EditCount: 100, Editors: 4, EditRepos: 1, Worktrees: 3} + for _, name := range []string{"two_primaries", "too_few_editors", "readers", "mixed_six_repos", "linked_label_collision"} { + t.Run(name, func(t *testing.T) { + corpora, o := []Corpus{editProbeCorpus()}, options + switch name { + case "two_primaries": + corpora = append(corpora, corpora[0]) + case "too_few_editors": + o.Editors = 3 + case "readers": + o.Agents = 1 + case "mixed_six_repos": + o.EditRepos = 6 + case "linked_label_collision": + corpora[0].Label = "linked-1" + } + if _, err := planEditProbeRoutes(corpora, o); err == nil { + t.Fatal("ambiguous checkout ownership accepted") + } + }) + } +} diff --git a/bench/load/first_index_measurement_test.go b/bench/load/first_index_measurement_test.go new file mode 100644 index 000000000..086c2f87a --- /dev/null +++ b/bench/load/first_index_measurement_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" +) + +// TestMeasureStandaloneFirstIndex is an opt-in, single-build harness. Run the +// identical harness on the control and candidate with an owned clean corpus; +// compare reports separately. It never grants a ratio from a single run. +func TestMeasureStandaloneFirstIndex(t *testing.T) { + corpus := os.Getenv("GORTEX_FIRST_INDEX_CORPUS") + if corpus == "" { + t.Skip("set GORTEX_FIRST_INDEX_CORPUS and GORTEX_FIRST_INDEX_REPORT for an owned-fixture measurement") + } + reportPath := os.Getenv("GORTEX_FIRST_INDEX_REPORT") + if reportPath == "" { + t.Fatal("GORTEX_FIRST_INDEX_REPORT is required") + } + corpus, err := filepath.Abs(corpus) + if err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + result, err := exec.Command("git", append([]string{"-C", corpus}, args...)...).CombinedOutput() + if err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, result) + } + return string(result) + } + if status := git("status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("fixture must be clean: %s", status) + } + sourceSHA := strings.TrimSpace(git("rev-parse", "HEAD")) + manifest := sha256.Sum256([]byte(git("ls-files", "-s", "-z"))) + output, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + t.Fatal(err) + } + defer output.Close() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fresh.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }) + sampler := newProcessSampler() + type sample struct { + At time.Time `json:"at"` + Metrics map[string]Metric `json:"metrics"` + } + var mu sync.Mutex + var samples []sample + var before, after map[string]Metric + var startAt, endAt time.Time + started, finished, joined := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var startOnce, endOnce sync.Once + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core, zap.Hooks(func(entry zapcore.Entry) error { + switch entry.Message { + case "indexer: drain start (shadow → disk)": + startOnce.Do(func() { + mu.Lock() + startAt = entry.Time + before = sampler.sample(os.Getpid()) + samples = append(samples, sample{time.Now(), before}) + mu.Unlock() + close(started) + }) + case "indexer: FlushBulk complete": + endOnce.Do(func() { + mu.Lock() + endAt = entry.Time + after = sampler.sample(os.Getpid()) + samples = append(samples, sample{time.Now(), after}) + mu.Unlock() + close(finished) + }) + } + return nil + })) + go func() { + defer close(joined) + select { + case <-started: + case <-finished: + return + } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-finished: + return + case <-ticker.C: + mu.Lock() + if endAt.IsZero() { + samples = append(samples, sample{time.Now(), sampler.sample(os.Getpid())}) + } + mu.Unlock() + } + } + }() + registry := parser.NewRegistry() + languages.RegisterAll(registry) + idx := indexer.New(store, registry, config.Default().Index, logger) + t.Cleanup(idx.Close) + result, indexErr := idx.IndexCtx(t.Context(), corpus) + endOnce.Do(func() { close(finished) }) + <-joined + complete := indexErr == nil && !startAt.IsZero() && !endAt.IsZero() + deltas := make(map[string]Metric) + for _, name := range []string{"diskio_written_bytes", "logical_written_bytes", "syscall_written_bytes"} { + if complete { + deltas[name] = counterDelta(before[name], after[name]) + } else { + deltas[name] = unavailable("bytes", "complete drain endpoints unavailable") + } + } + entries := logs.FilterMessage("indexer: FlushBulk complete").All() + var drainFields map[string]any + if len(entries) == 1 { + drainFields = entries[0].ContextMap() + } + report := map[string]any{ + "go_version": runtime.Version(), "goos": runtime.GOOS, "goarch": runtime.GOARCH, "gomaxprocs": runtime.GOMAXPROCS(0), + "complete": complete, "corpus_sha": sourceSHA, "tracked_manifest_sha256": hex.EncodeToString(manifest[:]), + "result": result, "drain_start": startAt, "drain_end": endAt, "drain_fields": drainFields, + "process_write_deltas": deltas, "process_samples": samples, + "qualification": "process-wide counters over drain log boundaries; concurrent process activity can contribute; macOS logical writes exclude external-volume ledger; diskio is the physical meter; unavailable observations remain null; five-second samples plus endpoints; compare identical frozen corpus/harness/host/storage separately; no acceptance inferred", + } + if complete { + report["drain_and_seal_seconds"] = endAt.Sub(startAt).Seconds() + } + if indexErr != nil { + report["error"] = indexErr.Error() + } + if err := json.NewEncoder(output).Encode(report); err != nil { + t.Fatal(err) + } + if status := git("status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("fixture changed: %s", status) + } + if strings.TrimSpace(git("rev-parse", "HEAD")) != sourceSHA || sha256.Sum256([]byte(git("ls-files", "-s", "-z"))) != manifest { + t.Fatal("fixture identity changed") + } + if !complete { + t.Fatalf("first index did not complete one measured drain: %v", indexErr) + } +} diff --git a/bench/load/lifecycle.go b/bench/load/lifecycle.go index c7176643f..8c592b1c2 100644 --- a/bench/load/lifecycle.go +++ b/bench/load/lifecycle.go @@ -22,20 +22,22 @@ type privateRepo struct { } type privateRun struct { - Root string - SocketDir string - Socket string - Store string - Config string - Env []string - Repos []privateRepo - Binary string - process *exec.Cmd - done chan struct{} - exitErr error - stdout *os.File - stderr *os.File - startCount int + discoveryCatalog map[string]discoveryCatalogIdentity // discovery-only observed seed identities + Root string + SocketDir string + Socket string + Store string + Config string + Env []string + daemonEnv []string // optional owned scenario shim; harness Git keeps Env + Repos []privateRepo + Binary string + process *exec.Cmd + done chan struct{} + exitErr error + stdout *os.File + stderr *os.File + startCount int } func inside(root, path string) bool { @@ -343,7 +345,11 @@ func (p *privateRun) start() error { } cmd := exec.Command(p.Binary, "--config", p.Config, "daemon", "start", "--backend-path", p.Store, "--tools", "facade-v1", "--tools-mode", "hide") configureOwnedProcess(cmd) - cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, p.Env, p.stdout, p.stderr + env := p.Env + if p.daemonEnv != nil { + env = p.daemonEnv + } + cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, env, p.stdout, p.stderr if err := cmd.Start(); err != nil { _ = p.stdout.Close() _ = p.stderr.Close() diff --git a/bench/load/plan.go b/bench/load/plan.go index d17afa478..c80add08d 100644 --- a/bench/load/plan.go +++ b/bench/load/plan.go @@ -200,6 +200,10 @@ func validateAgentCounts(scenario string, readers, editors int) error { if readers < 1 || editors < 1 { return errors.New("all scenarios require at least one pure reader and one combined edit/readback agent") } + case checkoutDiscoveryScenario: + if readers+editors < 2 { + return errors.New("worktree discovery needs sessions in both initial repositories") + } case "read", "many", "worktrees", "sweep", "cold", "restart": if readers < 1 { return errors.New("zero pure readers are supported only for edit/many-edit") diff --git a/bench/load/report.go b/bench/load/report.go index a02c5f307..3203592a7 100644 --- a/bench/load/report.go +++ b/bench/load/report.go @@ -83,20 +83,22 @@ type Host struct { // Every attempt is retained, including busy refusals and phase-censored calls. // Workload and readiness/visibility probes have distinct Role denominators. type Attempt struct { - Tool string `json:"tool"` - Role string `json:"role"` - Request string `json:"request_sha256"` - Agent int `json:"agent"` - Step int `json:"step"` - Slot int `json:"slot"` - Retry int `json:"retry"` - Started time.Time `json:"started"` - DurationMS float64 `json:"duration_ms"` - LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time - Code int `json:"code"` - Error string `json:"error"` - ExpectedOutcome string `json:"expected_outcome,omitempty"` - Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored + CWD string `json:"cwd,omitempty"` + ErrorData map[string]any `json:"error_data,omitempty"` + Tool string `json:"tool"` + Role string `json:"role"` + Request string `json:"request_sha256"` + Agent int `json:"agent"` + Step int `json:"step"` + Slot int `json:"slot"` + Retry int `json:"retry"` + Started time.Time `json:"started"` + DurationMS float64 `json:"duration_ms"` + LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time + Code int `json:"code"` + Error string `json:"error"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` + Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored } type ToolMetrics struct { diff --git a/bench/load/rpc_error_observation.go b/bench/load/rpc_error_observation.go new file mode 100644 index 000000000..52bc08e22 --- /dev/null +++ b/bench/load/rpc_error_observation.go @@ -0,0 +1,29 @@ +package main + +// Error data is decoded JSON. Keep a private snapshot when an observation +// outlives its RPC, including nested objects and lists of pending paths. +func copyRPCErrorData(data map[string]any) map[string]any { + if data == nil { + return nil + } + out := make(map[string]any, len(data)) + for key, value := range data { + out[key] = copyRPCErrorValue(value) + } + return out +} + +func copyRPCErrorValue(value any) any { + switch value := value.(type) { + case map[string]any: + return copyRPCErrorData(value) + case []any: + out := make([]any, len(value)) + for i, item := range value { + out[i] = copyRPCErrorValue(item) + } + return out + default: + return value + } +} diff --git a/bench/load/rpc_error_observation_test.go b/bench/load/rpc_error_observation_test.go new file mode 100644 index 000000000..8a40e8c11 --- /dev/null +++ b/bench/load/rpc_error_observation_test.go @@ -0,0 +1,19 @@ +package main + +import "testing" + +func TestRPCErrorDataSnapshotOwnsNestedJSON(t *testing.T) { + original := map[string]any{"code": "view_pending", "details": map[string]any{"paths": []any{"one.go", map[string]any{"scope": "checkout"}}}} + snapshot := copyRPCErrorData(original) + original["code"] = "changed" + details := original["details"].(map[string]any) + details["paths"].([]any)[0] = "two.go" + details["paths"].([]any)[1].(map[string]any)["scope"] = "other" + kept := snapshot["details"].(map[string]any)["paths"].([]any) + if snapshot["code"] != "view_pending" || kept[0] != "one.go" || kept[1].(map[string]any)["scope"] != "checkout" { + t.Fatalf("snapshot borrowed decoded RPC data: %+v", snapshot) + } + if copyRPCErrorData(nil) != nil { + t.Fatal("nil data became a fabricated object") + } +} diff --git a/bench/load/run.go b/bench/load/run.go index a843aeea7..d7ed4a94f 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -39,8 +39,10 @@ type runOptions struct { SweepRepetitions int SweepBudget time.Duration EditCount int + EditRepos int EditProbeRole string editProbeBaseline bool + Discovery discoveryOptions Startup startupOptions pairedDriver func(context.Context, *privateRun, []*socketClient, []*socketClient, *Phase) error smokeState *privateRun // only the fixed smoke entry lends its freshly owned store @@ -58,6 +60,9 @@ func parseRun(args []string) (runOptions, error) { if f.NArg() != 0 { return o, errors.New("unexpected positional run arguments") } + if o.EditRepos < 1 { + return o, errors.New("--edit-repos must be positive") + } o.editProbeBaseline = o.EditProbeRole == "baseline" return o, errors.Join(validateRunOptions(o), validateMemoryOptions(o)) } @@ -70,7 +75,7 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.StringVar(&o.GitSHA, "git-sha", "", "full binary revision if absent from Go build metadata") f.StringVar(&o.Profile, "profile", "agent", "agent (default) or explicit saturation") f.StringVar(&o.Host, "host", "native", "native, reference, or floor (validated, never provisioned)") - f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, cold, restart, sweep, idle-stop, edit-and-probe, all") + f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, worktree-discovery, cold, restart, sweep, idle-stop, edit-and-probe, all") f.IntVar(&o.Agents, "agents", 4, "pure reading agents (may be 0 for edit, many-edit, or edit-and-probe; idle-stop requires 0)") f.IntVar(&o.Editors, "edit-agents", 4, "combined editing/readback/reading agents in edit scenarios, in addition to pure readers") f.IntVar(&o.Worktrees, "worktrees", 4, "linked worktrees in addition to primary") @@ -85,11 +90,17 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") bindStartupFlags(f, &o.Startup) + f.IntVar(&o.EditRepos, "edit-repos", 1, "edit-and-probe: one repository (default), or exactly six distinct pinned repositories") + bindDiscoveryFlags(f, &o.Discovery) f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } func validateRunOptions(o runOptions) error { + if o.EditRepos < 0 || o.EditRepos > 1 && (o.EditRepos != 6 || o.Scenario != editProbeScenario) { + return errors.New("--edit-repos selects one repository or exactly six for edit-and-probe only") + } + if err := validatePRSmokeDrain(o); err != nil { return err } @@ -99,6 +110,9 @@ func validateRunOptions(o runOptions) error { if err := validateAgentCounts(o.Scenario, o.Agents, o.Editors); err != nil { return err } + if err := o.Discovery.validate(o.Scenario, o.Worktrees); err != nil { + return err + } if err := o.Startup.validate(o.Scenario); err != nil { return err } @@ -108,8 +122,11 @@ func validateRunOptions(o runOptions) error { if _, err := parseProfile(o.Profile); err != nil { return err } - if o.Scenario == editProbeScenario && (o.Profile != "agent" || o.Agents > editProbeMaxReaders || o.Editors < 1 || o.Editors > editProbeMaxEditors || o.Worktrees != 0 || o.CPUProfile != 0 || o.EditCount != 1 && o.EditCount != 100 || o.EditCount == 1 && o.Editors != 1) { - return fmt.Errorf("edit-and-probe requires --profile agent --agents 0..%d --edit-agents 1..%d on one repository --worktrees 0, no CPU profile, and --edit-count 100 (or an explicit 1-edit, 1-editor diagnostic)", editProbeMaxReaders, editProbeMaxEditors) + if o.Scenario == editProbeScenario && (o.Profile != "agent" || o.Agents > editProbeMaxReaders || o.Editors < 1 || o.Editors > editProbeMaxEditors || o.Worktrees > 15 || o.CPUProfile != 0 || o.EditCount != 1 && o.EditCount != 100 || o.EditCount == 1 && o.Editors != 1) { + return fmt.Errorf("edit-and-probe requires --profile agent --agents 0..%d --edit-agents 1..%d on one repository or explicit routed checkouts, at most 15 linked worktrees, no CPU profile, and --edit-count 100 (or an explicit 1-edit, 1-editor diagnostic)", editProbeMaxReaders, editProbeMaxEditors) + } + if o.Scenario == editProbeScenario && (o.EditRepos > 1 || o.Worktrees > 0) && (o.EditRepos > 1 && o.Worktrees > 0 || o.Agents != 0 || o.EditCount != 100 || o.Editors < max(o.EditRepos, o.Worktrees+1)) { + return errors.New("routed edit-and-probe selects six repositories or linked checkouts, 100 edits, no pure readers and at least one fixed editor per checkout") } if o.EditProbeRole != "" && (o.Scenario != editProbeScenario || o.EditProbeRole != "candidate" && o.EditProbeRole != "baseline") { return errors.New("--edit-probe-role is candidate or baseline, for edit-and-probe only") @@ -128,7 +145,7 @@ func validateRunOptions(o runOptions) error { activeAgents += o.Editors } activeWorktrees := 1 - if o.Scenario == "worktrees" || o.Scenario == "all" { + if o.Scenario == "worktrees" || o.Scenario == "all" || o.Scenario == editProbeScenario && o.Worktrees > 0 { activeWorktrees += o.Worktrees } if o.Host == "floor" && (activeAgents > 4 || activeWorktrees > 4) { @@ -141,6 +158,12 @@ func validateRunOptions(o runOptions) error { } func selectedScenarios(o runOptions) ([]string, error) { + if o.Scenario == checkoutDiscoveryScenario { + if o.Agents+o.Editors < 2 || o.Host == "floor" { + return nil, errors.New("worktree-discovery requires at least two reading/editing sessions and a native or reference host") + } + return []string{checkoutDiscoveryScenario}, nil + } if o.Scenario == "idle-stop" { return []string{"idle-stop"}, nil } @@ -206,6 +229,9 @@ func runCommand(ctx context.Context, args []string) error { if err := validateOutputRoots(corpora, o.Scratch, o.Output); err != nil { return err } + if o.Discovery.Prepare != "" { + return prepareDiscoveryState(ctx, o, corpora) + } sha, err := binaryRevision(o.Binary, o.GitSHA) if err != nil { return err @@ -263,12 +289,15 @@ func newRunReport(o runOptions, corpora []Corpus, sha string, host Host, outputR report.Workload.Operations = append(report.Workload.Operations, ops...) } if o.Scenario == editProbeScenario { - plan, err := planEditProbe(corpora[0], o) + plan, err := planEditProbeRoutes(corpora, o) if err != nil { return report, err } report.Workload.EditProbe = &plan - report.Workload.Operations = editProbeOperations(plan, corpora[0]) + report.Workload.Operations, err = editProbeRouteOperations(plan, corpora) + if err != nil { + return report, err + } report.Workload.SessionModel = plan.SessionModel } report.Workload.Scenario, report.Workload.Agents, report.Workload.EditAgents, report.Workload.LinkedWorktrees = o.Scenario, o.Agents, o.Editors, o.Worktrees diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 079b1cf3b..fed246e34 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -61,14 +61,28 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp resultErr = errors.New(strings.Join(phase.Errors, "; ")) } }() + worktreeViews := scenario == "worktrees" || scenario == editProbeScenario && o.Worktrees > 0 + if scenario == editProbeScenario && (o.EditRepos > 1 || o.Worktrees > 0) { + if _, err := planEditProbeRoutes(corpora, o); err != nil { + return phase, nil, artifacts, crashes, "", err + } + if o.EditRepos > 1 { + if err := validateEditProbeSourceRepos(ctx, corpora, outputRoot); err != nil { + return phase, nil, artifacts, crashes, "", err + } + } + } selected := corpora[:1] - if scenario == "many" || scenario == "many-edit" { + if scenario == "many" || scenario == "many-edit" || scenario == checkoutDiscoveryScenario || scenario == editProbeScenario && o.EditRepos > 1 { if len(corpora) < 2 { return phase, nil, artifacts, crashes, "", errors.New("many-repository scenarios require at least two pinned repositories") } selected = corpora + if scenario == checkoutDiscoveryScenario { + selected = corpora[:2] + } } - editing := scenario == "edit" || scenario == "many-edit" || scenario == editProbeScenario + editing := scenario == "edit" || scenario == "many-edit" || scenario == editProbeScenario || scenario == checkoutDiscoveryScenario && o.Editors > 0 for _, corpus := range selected { operations, planErr := readingPlan(corpus) if planErr != nil { @@ -87,18 +101,25 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } } if scenario == editProbeScenario { - plan, err := planEditProbe(selected[0], o) + plan, err := planEditProbeRoutes(selected, o) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + phase.Operations, err = editProbeRouteOperations(plan, selected) if err != nil { return phase, nil, artifacts, crashes, "", err } - phase.Operations = editProbeOperations(plan, selected[0]) phase.EditProbe = newEditProbeReport(plan, selected[0], !o.editProbeBaseline) phase.SessionModel = plan.SessionModel } p := o.smokeState var err error if p == nil { - p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected) + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" { + p, err = resumeDiscoveryState(ctx, o, selected) + } else { + p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected) + } } else { err = validateSmokeOwner(p, o.Binary, selected) } @@ -106,6 +127,13 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp return phase, nil, artifacts, crashes, "", err } artifacts["private_root"] = p.Root + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" { + raw, err := os.ReadFile(o.Discovery.Resume) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + phase.Gate["discovery_seed_lineage"] = json.RawMessage(raw) + } firstStart := p.startCount + 1 if o.smokeState != nil { info, statErr := os.Stat(p.Store) @@ -151,7 +179,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp resultErr = errors.Join(resultErr, p.removeState()) } }() - if scenario == "worktrees" { + if worktreeViews { if err := p.addWorktrees(ctx, bootWorktrees(o)); err != nil { return phase, nil, artifacts, crashes, "", err } @@ -164,10 +192,33 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } warmCtx, cancelWarm := context.WithTimeout(ctx, o.WarmTimeout) defer cancelWarm() - configured, err := p.writeBootRepos(scenario == "worktrees") + var configured []privateRepo + if scenario == checkoutDiscoveryScenario { + configured, err = p.writeDiscoveryBootRepos() + } else { + configured, err = p.writeBootRepos(worktreeViews) + } if err != nil { return phase, nil, artifacts, crashes, "", err } + if scenario == checkoutDiscoveryScenario && o.Discovery.StallFirst { + release, files, err := p.installDiscoveryFault() + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + for name, path := range files { + artifacts[name] = path + } + defer func() { + resultErr = errors.Join(resultErr, release()) + raw, err := os.ReadFile(files["discovery_git_events"]) + observed := err == nil && len(bytes.TrimSpace(raw)) > 0 + phase.Gate["discovery_git_fault"], _ = json.Marshal(map[string]any{"configured": true, "observed": observed, "raw_events": string(raw), "absolute_child_bound_seconds": 60}) + if !observed { + resultErr = errors.Join(resultErr, errors.New("git stall never entered the intended new checkout"), err) + } + }() + } coldStart := time.Now() if err := p.start(); err != nil { return phase, nil, artifacts, crashes, "", err @@ -227,16 +278,24 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp for i := range p.Repos { for _, tracked := range readinessState.TrackedRepos { if filepath.Clean(tracked.Path) == filepath.Clean(p.Repos[i].Path) { + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" && p.Repos[i].Prefix != tracked.Prefix { + return phase, nil, artifacts, crashes, "", errors.New("resumed discovery changed a prepared repository identity prefix") + } p.Repos[i].Prefix = tracked.Prefix } } - if i > 0 && scenario == "worktrees" { + if i > 0 && worktreeViews || i >= 2 && scenario == checkoutDiscoveryScenario { p.Repos[i].Prefix = p.Repos[0].Prefix } if p.Repos[i].Prefix == "" { return phase, nil, artifacts, crashes, "", errors.New("readiness has no actual repo prefix") } } + if observed, err := p.verifyDiscoveryCatalog(ctx); err != nil { + return phase, nil, artifacts, crashes, "", err + } else if observed != nil { + phase.Gate["resumed_catalog"] = observed + } var clients []*socketClient defer func() { for _, client := range clients { @@ -260,7 +319,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp version = client.version if i < len(p.Repos) { subscribeInvalidations(warmCtx, client, repo.Corpus.Label, o.Deadline, &phase) - if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0, o.Deadline, &phase); err != nil { + if err := preflightRoot(warmCtx, p, client, repo, worktreeViews && i > 0 || scenario == checkoutDiscoveryScenario && i >= 2, o.Deadline, &phase); err != nil { return phase, nil, artifacts, crashes, version, err } gate, rows, err := warmAnalysis(warmCtx, client, o.Deadline) @@ -429,6 +488,8 @@ func measureStandalone(ctx context.Context, o runOptions, p *privateRun, clients sweepCtx, cancelSweep := context.WithTimeout(ctx, o.SweepBudget) defer cancelSweep() err = runSweep(sweepCtx, o, sweepClients, p.Repos[0], sweepRequests, sweep, phase, waitContext) + } else if phase.Name == checkoutDiscoveryScenario { + err = measureCheckoutDiscovery(ctx, o, p, clients, phase) } else if phase.Name != "cold" && phase.Name != "restart" { err = runAgents(ctx, o, p, clients, phase.Name == "edit" || phase.Name == "many-edit", phase) } diff --git a/bench/load/selftest_test.go b/bench/load/selftest_test.go index 208a28bde..6f8277c00 100644 --- a/bench/load/selftest_test.go +++ b/bench/load/selftest_test.go @@ -143,12 +143,14 @@ func fixtureDaemon(mode string) int { case "status": result = map[string]any{"ready": true, "enriched": true, "views": map[string]any{"coordinators": coordinators()}, "storage": map[string]any{"wal_pending_frames": 0}, "pprof_addr": pprof.Addr().String()} case "search_symbols": + checkoutID := "" if path, _ := frame.Params["path"].(string); path != "" { viewsMu.Lock() views[filepath.Clean(path)] = true viewsMu.Unlock() + checkoutID = "fixture-checkout-" + filepath.Base(path) } - result = map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": config.Repos[0].Name}} + result = map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": config.Repos[0].Name, "checkout_id": checkoutID}} case "shutdown": result = map[string]any{"stopping": true} default: diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index 56717f6fb..305f9f633 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -258,12 +258,22 @@ "code": { "type": "integer" }, + "cwd": { + "type": "string" + }, "duration_ms": { "type": "number" }, "error": { "type": "string" }, + "error_data": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, "expected_outcome": { "type": "string" }, @@ -697,6 +707,9 @@ } ] }, + "checkout_id": { + "type": "string" + }, "editor": { "type": "integer" }, @@ -1456,6 +1469,9 @@ } ] }, + "route": { + "type": "string" + }, "stale_claimed_fresh": { "type": "boolean" }, @@ -1780,6 +1796,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -1807,6 +1826,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -1905,6 +1974,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, @@ -3565,6 +3637,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -3592,6 +3667,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -3690,6 +3815,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index ac950dfa7..94b97962f 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -245,12 +245,22 @@ "code": { "type": "integer" }, + "cwd": { + "type": "string" + }, "duration_ms": { "type": "number" }, "error": { "type": "string" }, + "error_data": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, "expected_outcome": { "type": "string" }, @@ -684,6 +694,9 @@ } ] }, + "checkout_id": { + "type": "string" + }, "editor": { "type": "integer" }, @@ -1443,6 +1456,9 @@ } ] }, + "route": { + "type": "string" + }, "stale_claimed_fresh": { "type": "boolean" }, @@ -1767,6 +1783,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -1794,6 +1813,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -1892,6 +1961,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, @@ -3552,6 +3624,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -3579,6 +3654,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -3677,6 +3802,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index df8ec5892..e32b17b7d 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -163,6 +163,17 @@ type daemonWatcherSlot struct { ready bool } +// PreflightTrack checks authoritative ownership without indexing, persisting, +// or taking the controller mutation lock. A clear answer grants no registration. +func (c *realController) PreflightTrack(ctx context.Context, p daemon.TrackParams) error { + if c.multiIndexer == nil || c.lifecycle == nil { + return fmt.Errorf("checkout lifecycle not initialized") + } + return c.lifecycle.PreflightTrack(ctx, config.RepoEntry{ + Path: p.Path, Name: p.Name, Ref: p.Ref, AsWorktree: p.AsWorktree, + }) +} + // Track indexes a new repository and persists it to the global config. // Path is resolved to an absolute form before the MultiIndexer sees it. func (c *realController) Track(ctx context.Context, p daemon.TrackParams) (json.RawMessage, error) { @@ -1782,6 +1793,14 @@ func storageStatusFor(g graph.Store) *daemon.StorageStatus { CloseCheckpointEstimateMS: estimate.Milliseconds(), WALReclaim: walReclaimStatus(reporter.WALReclaimStats()), } + if reporter, ok := g.(interface{ DenseIndexesMissing() []string }); ok { + out.DenseIndexesMissing = reporter.DenseIndexesMissing() + } + if reporter, ok := g.(interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + }); ok { + out.LastFirstIndexDrain = reporter.LastFirstIndexDrains() + } if policy, ok := g.(interface{ CloseCheckpointPolicy() string }); ok { out.CloseCheckpointPolicy = policy.CloseCheckpointPolicy() } diff --git a/cmd/gortex/daemon_mcp.go b/cmd/gortex/daemon_mcp.go index 44912b21e..d8d47e1e7 100644 --- a/cmd/gortex/daemon_mcp.go +++ b/cmd/gortex/daemon_mcp.go @@ -423,17 +423,23 @@ func checkoutAdmissionError(cwd string, inbound []byte, err error) []byte { } _ = json.Unmarshal(inbound, &req) code, retryable := checkoutAdmissionFailure(err) + data := map[string]any{"error_code": code, "path": cwd, "retryable": retryable} + // A pending discovery names the checkout root it applies to, so a caller + // can tell one pending worktree from an unavailable daemon. + var pending *indexer.CheckoutDiscoveryPendingError + if errors.As(err, &pending) { + data["family"], data["reason"], data["state"] = pending.Family, pending.Reason, "discovery_pending" + if pending.Root != "" { + data["path"] = pending.Root + } + } out, _ := json.Marshal(map[string]any{ "jsonrpc": "2.0", "id": req.ID, "error": map[string]any{ "code": -32000, "message": "checkout discovery unavailable: " + err.Error(), - "data": map[string]any{ - "error_code": code, - "path": cwd, - "retryable": retryable, - }, + "data": data, }, }) return out @@ -845,16 +851,19 @@ func (d *mcpDispatcher) notTrackedError(sess *daemon.Session, inbound []byte) [] } _ = json.Unmarshal(inbound, &peek) + // Tool arguments cannot change session admission. Point callers to an + // explicit listing instead of enumerating unrelated repositories per error. + const recovery = "Use `gortex repos` to list tracked repositories, then reconnect MCP with its working directory inside the intended repository." resp := map[string]any{ "jsonrpc": "2.0", "id": peek.ID, "error": map[string]any{ "code": -32000, - "message": fmt.Sprintf("repository not tracked: %s", sess.CWD), + "message": fmt.Sprintf("repository not tracked: %s. %s", sess.CWD, recovery), "data": map[string]any{ "error_code": "repo_not_tracked", "path": sess.CWD, - "suggestion": fmt.Sprintf("Run `gortex track %s` to include this repo in the shared graph.", sess.CWD), + "suggestion": fmt.Sprintf("%s Alternatively, run `gortex track %s` to include this repo in the shared graph.", recovery, sess.CWD), }, }, } diff --git a/cmd/gortex/daemon_mcp_checkout_admission_test.go b/cmd/gortex/daemon_mcp_checkout_admission_test.go index 4a38224da..21195c028 100644 --- a/cmd/gortex/daemon_mcp_checkout_admission_test.go +++ b/cmd/gortex/daemon_mcp_checkout_admission_test.go @@ -65,6 +65,22 @@ func TestDispatcher_CheckoutAdmissionFailurePreservesClassification(t *testing.T } } +// Hooks and agents key on this refusal to tell one pending worktree from an +// unavailable daemon. +func TestCheckoutAdmissionErrorPendingDiscoveryGolden(t *testing.T) { + err := fmt.Errorf("observe checkout: %w", &indexer.CheckoutDiscoveryPendingError{Root: "/repos/app-wt", Family: "family-0123", Reason: "family_capacity"}) + got := checkoutAdmissionError("/repos/app-wt/internal", []byte(`{"jsonrpc":"2.0","id":7,"method":"tools/call"}`), err) + const want = `{"error":{"code":-32000,"data":{"error_code":"view_building","family":"family-0123","path":"/repos/app-wt","reason":"family_capacity","retryable":true,"state":"discovery_pending"},` + + `"message":"checkout discovery unavailable: observe checkout: indexer: checkout mutation lane is busy; retry: checkout discovery is pending for /repos/app-wt (family family-0123, reason family_capacity)"},"id":7,"jsonrpc":"2.0"}` + require.Equal(t, want, string(got)) + + // A refusal that names no checkout root keeps the session's path. + got = checkoutAdmissionError("/repos/app-wt/internal", []byte(`{"jsonrpc":"2.0","id":8,"method":"tools/call"}`), &indexer.CheckoutDiscoveryPendingError{Reason: "in_progress"}) + var parsed map[string]any + require.NoError(t, json.Unmarshal(got, &parsed)) + require.Equal(t, "/repos/app-wt/internal", errorData(t, parsed)["path"], "%s", got) +} + type checkoutAdmissionSQLiteError int func (e checkoutAdmissionSQLiteError) Error() string { return fmt.Sprintf("SQLite error %d", e) } diff --git a/cmd/gortex/daemon_mcp_discovery_scope_test.go b/cmd/gortex/daemon_mcp_discovery_scope_test.go new file mode 100644 index 000000000..8ce7d85c0 --- /dev/null +++ b/cmd/gortex/daemon_mcp_discovery_scope_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/gitstate" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/graphview" + "github.com/zzet/gortex/internal/indexer" + gortexmcp "github.com/zzet/gortex/internal/mcp" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/pathkey" + "github.com/zzet/gortex/internal/query" + "github.com/zzet/gortex/internal/reconcile" + "github.com/zzet/gortex/internal/search" +) + +func discoveryScopeRepo(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "fixture.go"), []byte("package fixture\n\nfunc Value() int { return 1 }\n"), 0o600)) + guidanceGit(t, dir, "init", "-q", "-b", "main") + guidanceGit(t, dir, "add", ".") + guidanceGit(t, dir, "commit", "-q", "-m", "fixture") +} + +// A checkout discovery that hangs refuses only its own worktree. The primary +// checkout, its directories outside a pending nested worktree, an unrelated +// tracked repository and another family's new worktree keep answering. +func TestDispatcher_PendingDiscoveryStaysInItsWorktree(t *testing.T) { + base, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + primary, other := filepath.Join(base, "primary"), filepath.Join(base, "other") + discoveryScopeRepo(t, primary) + discoveryScopeRepo(t, other) + gc := &config.GlobalConfig{} + gc.SetConfigPath(filepath.Join(base, "config.yaml")) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(gc.ConfigPath()) + require.NoError(t, err) + store, err := store_sqlite.Open(filepath.Join(base, "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + registry := parser.NewRegistry() + registry.Register(languages.NewGoExtractor()) + mi := indexer.NewMultiIndexer(store, registry, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { _ = mi.Close(context.Background()) }) + lifecycle, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: zap.NewNop()}) + require.NoError(t, err) + t.Cleanup(func() { _ = lifecycle.Close() }) + // The primary family's new worktrees hang in Git metadata and ignore + // cancellation, like Git on a hung mount. + stuck, nested := filepath.Join(base, "stuck"), filepath.Join(primary, "nested") + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + reconcile.WithHEADSampler(func(ctx context.Context, root string) (gitstate.HEADState, error) { + if root = pathkey.CanonicalExistingRoot(root); pathkey.HasPathPrefix(root, stuck) || pathkey.HasPathPrefix(root, nested) { + <-release + return gitstate.HEADState{}, errors.New("released hung HEAD sample") + } + return gitstate.SampleHEAD(ctx, root) + })(lifecycle.Reconciler()) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + families := map[string]string{} + for name, root := range map[string]string{"primary": primary, "other": other} { + registered, err := lifecycle.Register(ctx, config.RepoEntry{Path: root, Name: name}, indexer.TrackSourceCLI) + require.NoError(t, err) + require.NoError(t, registered.CatalogErr) + families[name] = registered.FamilyID + } + srv := gortexmcp.NewServer(query.NewEngine(store), store, nil, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm, CheckoutLifecycle: lifecycle}) + srv.SetMaterializer(&graphview.Materializer{Store: store, Catalog: store.Catalog()}) + d := newMCPDispatcher(srv, mi, zap.NewNop()) + + late := filepath.Join(base, "late") + guidanceGit(t, primary, "worktree", "add", "-q", "-b", "stuck", stuck) + guidanceGit(t, primary, "worktree", "add", "-q", "-b", "nested", nested) + guidanceGit(t, other, "worktree", "add", "-q", "-b", "late", late) + const initializeFrame = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}` + const searchFrame = `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search","arguments":{"query":"Value"}}}` + searchFrom := func(id, cwd string) map[string]any { + sess := &daemon.Session{ID: id, CWD: cwd} + dispatchJSON(t, d, sess, initializeFrame) + return dispatchJSON(t, d, sess, searchFrame) + } + + // Sessions in many directories of the hung worktree. The second round + // joins the first round's discoveries, so their workers hang. + stuckSessions := make([]*daemon.Session, 40) + for i := range stuckSessions { + cwd := filepath.Join(stuck, fmt.Sprintf("session-%d", i)) + require.NoError(t, os.Mkdir(cwd, 0o700)) + stuckSessions[i] = &daemon.Session{ID: fmt.Sprintf("stuck-%d", i), CWD: cwd} + } + stuckReplies := make([][]byte, len(stuckSessions)) + stuckErrs := make([]error, len(stuckSessions)) + for range 2 { + var wg sync.WaitGroup + for i, sess := range stuckSessions { + wg.Go(func() { stuckReplies[i], stuckErrs[i] = d.Dispatch(context.Background(), sess, []byte(searchFrame)) }) + } + wg.Wait() + } + + // The checks below run once the hung discovery is past its work deadline + // and still running: a session waiting on it is then refused as draining. + waiting := -1 + for i, raw := range stuckReplies { + require.NoError(t, stuckErrs[i]) + var parsed map[string]any + require.NoError(t, json.Unmarshal(raw, &parsed)) + if errorData(t, parsed)["reason"] == "in_progress" { + waiting = i + break + } + } + require.NotEqual(t, -1, waiting, "no session waits on a hung discovery") + drainBy := time.Now().Add(6 * time.Second) + for { + data := errorData(t, dispatchJSON(t, d, stuckSessions[waiting], searchFrame)) + if data["reason"] == "draining" { + break + } + require.Equal(t, "in_progress", data["reason"], "%v", data) + require.True(t, time.Now().Before(drainBy), "the hung discovery did not pass its work deadline: %v", data) + time.Sleep(50 * time.Millisecond) + } + + // Another family's new worktree gets its identity; it may wait for its own + // discovery, never for the hung one. + deadline := time.Now().Add(30 * time.Second) + for { + reply := searchFrom("late", late) + refusal, refused := reply["error"].(map[string]any) + if !refused { + break + } + data, _ := refusal["data"].(map[string]any) + require.Equal(t, "in_progress", data["reason"], "another family's discovery refused this worktree: %v", reply) + require.True(t, time.Now().Before(deadline), "the other family's worktree discovery did not finish: %v", reply) + time.Sleep(10 * time.Millisecond) + } + overview, err := lifecycle.FamiliesOverview(ctx, families["other"]) + require.NoError(t, err) + allocated := false + for _, family := range overview.Families { + for _, checkout := range family.Checkouts { + allocated = allocated || pathkey.EqualPaths(checkout.RootPath, late) + } + } + require.True(t, allocated, "the other family's new worktree has no identity: %+v", overview.Families) + + for _, cwd := range []string{primary, filepath.Join(primary, "internal"), other} { + reply := searchFrom("answer-"+cwd, cwd) + require.Contains(t, toolResultText(t, reply), "Value", "%s must answer while another checkout's discovery hangs: %v", cwd, reply) + require.NotEqual(t, true, reply["result"].(map[string]any)["isError"], "%s: %v", cwd, reply) + } + + // Only a path inside the unregistered nested worktree waits for discovery. + reply := searchFrom("nested", nested) + text := toolResultText(t, reply) + require.Equal(t, true, reply["result"].(map[string]any)["isError"], "nested worktree answered from its parent: %v", reply) + require.Contains(t, text, "view_building") + require.Contains(t, text, nested, "the refusal names the pending nested worktree") + + familyCapacity := 0 + for i, raw := range stuckReplies { + require.NoError(t, stuckErrs[i]) + var parsed map[string]any + require.NoError(t, json.Unmarshal(raw, &parsed)) + data := errorData(t, parsed) + require.Equal(t, "view_building", data["error_code"], "%v", parsed) + require.Equal(t, true, data["retryable"], "%v", parsed) + path, _ := data["path"].(string) + require.True(t, pathkey.EqualPaths(stuck, path), "the refusal names the pending checkout root, not the session directory: %v", parsed) + require.Equal(t, families["primary"], data["family"], "%v", parsed) + require.Contains(t, []any{"in_progress", "family_capacity"}, data["reason"], "%v", parsed) + if data["reason"] == "family_capacity" { + familyCapacity++ + } + } + require.NotZero(t, familyCapacity, "one family's discoveries are bounded") +} diff --git a/cmd/gortex/daemon_mcp_recovery_test.go b/cmd/gortex/daemon_mcp_recovery_test.go new file mode 100644 index 000000000..8514dbd02 --- /dev/null +++ b/cmd/gortex/daemon_mcp_recovery_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/daemon" +) + +func TestDispatcherUntrackedRecoveryNamesWorkingDirectoryRemedy(t *testing.T) { + tracked, untracked := t.TempDir(), t.TempDir() + d, _ := trackedPathMCPSetup(t, tracked) + sess := &daemon.Session{ID: "untracked_recovery", CWD: untracked} + + for _, tool := range []string{"explore", "search_symbols"} { + t.Run(tool, func(t *testing.T) { + // Repository arguments cannot bypass session admission. The recovery + // must tell the client to change its MCP working directory instead. + frame := fmt.Appendf(nil, `{"jsonrpc":"2.0","id":19,"method":"tools/call","params":{"name":%q,"arguments":{"path":%q,"repo":"intended"}}}`, tool, tracked) + reply, err := d.Dispatch(context.Background(), sess, frame) + require.NoError(t, err) + var response struct { + ID int `json:"id"` + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + ErrorCode string `json:"error_code"` + Path string `json:"path"` + Suggestion string `json:"suggestion"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.Unmarshal(reply, &response)) + assert.Equal(t, 19, response.ID) + assert.Equal(t, -32000, response.Error.Code) + assert.Equal(t, "repo_not_tracked", response.Error.Data.ErrorCode) + assert.Equal(t, untracked, response.Error.Data.Path, "retain the existing machine-readable attempted path") + for _, text := range []string{response.Error.Message, response.Error.Data.Suggestion} { + assert.Contains(t, text, "gortex repos") + assert.Contains(t, text, "working directory") + assert.NotContains(t, text, tracked) + assert.Contains(t, text, untracked, "retain the attempted-path diagnostic") + assert.NotContains(t, text, "path=") + assert.NotContains(t, text, "repo=") + } + assert.Contains(t, response.Error.Data.Suggestion, "reconnect") + assert.Contains(t, response.Error.Data.Suggestion, "gortex track "+untracked) + }) + } +} diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 2d7d81663..785de26c7 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -745,7 +745,7 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() defer func() { if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil { logger.Error("daemon: coordinated cold bulk-load cleanup failed", zap.Error(err)) } } @@ -844,9 +844,14 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z // final seal includes them in bulk index creation, checkpointing, and // planner statistics instead of reopening ordinary write transactions // immediately after a large cold load. + // The finalize is the window's last call: the deferred end above is + // skipped once it has run, and nothing retries a failed seal. So a + // failed end is followed by an abort, which closes the window and + // leaves the indexes it could not rebuild to the store's repair, rather + // than keeping the warmup's relaxed-durability writer until shutdown. var finalizeBulk func() error if coordinatedBulkActive { - finalizeBulk = coordinatedBulk.EndCoordinatedBulkLoad + finalizeBulk = func() error { return graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk) } } drain := state.multiIndexer.RunDeferredGoModAll if batchCtx.Err() != nil { diff --git a/cmd/gortex/daemon_warmup_bulk_test.go b/cmd/gortex/daemon_warmup_bulk_test.go index 393bf6000..61b3980ab 100644 --- a/cmd/gortex/daemon_warmup_bulk_test.go +++ b/cmd/gortex/daemon_warmup_bulk_test.go @@ -1,10 +1,45 @@ package main import ( + "context" + "encoding/json" "errors" + "io" + "log" + "os" + "path/filepath" + "strings" "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" ) +// The existing missing-index log is emitted after the atomic record is installed +// and before repair is scheduled. Observe status at that causal boundary instead +// of racing a successful background repair after warmup returns. +type warmupSealStatusProbe struct { + output io.Writer + missing func() +} + +func (p warmupSealStatusProbe) Write(data []byte) (int, error) { + if strings.Contains(string(data), "store_sqlite: dense indexes missing after bulk window close") { + p.missing() + } + return p.output.Write(data) +} + func TestDrainGoModAndFinalizeWarmupBulkOrdersDrainBeforeFinalize(t *testing.T) { var order []string wantErr := errors.New("finalize failed") @@ -38,3 +73,92 @@ func TestDrainGoModAndFinalizeWarmupBulkWithoutBulkFinalizer(t *testing.T) { t.Fatalf("finalize elapsed = %s, want 0", finalizeElapsed) } } + +// TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow: a cold warmup whose +// final seal fails logs the failure and closes the coordinated window, rather +// than keeping its relaxed-durability writer until the daemon shuts down. The +// writer is back at its own synchronous level, the planner probe no longer +// sees a live window, and the store's repair rebuilds the index the seal +// could not. +func TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow(t *testing.T) { + t.Setenv("GORTEX_PARSER_ISOLATION", "0") + root := t.TempDir() + t.Setenv("HOME", filepath.Join(root, "home")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(root, "data")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(root, "cache")) + repo := filepath.Join(root, "repo") + require.NoError(t, os.Mkdir(repo, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(repo, "main.go"), []byte("package main\nfunc Main() {}\n"), 0o600)) + store, err := store_sqlite.Open(filepath.Join(root, "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + ctx := context.Background() + writerSync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err) + cm, err := config.NewConfigManager(filepath.Join(root, "config.yaml")) + require.NoError(t, err) + require.NoError(t, cm.Global().AddRepo(config.RepoEntry{Path: repo})) + reg := parser.NewRegistry() + reg.Register(languages.NewGoExtractor()) + logs, observed := observer.New(zap.InfoLevel) + logger := zap.New(logs) + idx := indexer.New(store, reg, config.Default().Index, logger) + mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + state := &daemonState{graph: store, indexer: idx, multiIndexer: mi, configManager: cm} + injected := errors.New("injected final seal failure") + restore := store.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + + missingStatus := make(chan []string, 1) + priorOutput := log.Writer() + log.SetOutput(warmupSealStatusProbe{output: priorOutput, missing: func() { + select { + case missingStatus <- storageStatusFor(store).DenseIndexesMissing: + default: + } + }}) + defer log.SetOutput(priorOutput) + _, _ = warmupDaemonState(state, logger, func() {}) + select { + case missing := <-missingStatus: + assert.Contains(t, missing, "nodes_by_kind", "status reads the existing missing-index record before repair is scheduled") + default: + t.Error("the failed seal never published its missing-index boundary") + } + restore() + require.NotEmpty(t, observed.FilterMessage("daemon: coordinated cold bulk-load finalize failed").All(), + "the fixture must fail the coordinated window's final seal") + gotSync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err) + assert.Equal(t, writerSync, gotSync, "writer durability must be restored") + health, err := store.PlannerStatsHealth(ctx) + require.NoError(t, err) + assert.NotEqual(t, "bulk_window_active", health.Reason, "the failed window must not stay open") + deadline := time.Now().Add(10 * time.Second) + for health.Reason == "dense_indexes_missing" && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + health, err = store.PlannerStatsHealth(ctx) + require.NoError(t, err) + } + assert.NotEqual(t, "dense_indexes_missing", health.Reason, "the repair must rebuild the index the seal left out") +} + +func TestStorageStatusReportsFirstIndexDrainWithoutInventingEmptyResult(t *testing.T) { + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "status.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + empty, err := json.Marshal(storageStatusFor(store)) + require.NoError(t, err) + assert.NotContains(t, string(empty), "last_first_index_drain") + store.RecordFirstIndexDrain("repo", graph.FirstIndexDrain{Window: "coordinated", Nodes: 7, Edges: 9}) + response := storageStatusFor(store) + require.NotNil(t, response) + assert.Equal(t, 7, response.LastFirstIndexDrain["repo"].Nodes) + assert.Equal(t, "coordinated", response.LastFirstIndexDrain["repo"].Window) +} diff --git a/cmd/gortex/track.go b/cmd/gortex/track.go index 4d2a1b42a..c067375c1 100644 --- a/cmd/gortex/track.go +++ b/cmd/gortex/track.go @@ -247,7 +247,17 @@ func runTrack(cmd *cobra.Command, args []string) error { } } if owner, held := owners[prefix]; held && !pathkey.SamePathIdentity(pathkey.CanonicalPath(owner), absPath) { - return indexer.NewPrefixCollisionError(prefix, owner, absPath) + collision := indexer.NewPrefixCollisionError(prefix, owner, absPath) + // Let a running daemon make its own read-only ownership decision, + // including its deduplicated warning, before refusing locally. + // Never send a mutating Track or persist this rejected entry. + if err := preflightDaemonTrack(cmd.Context(), absPath, daemonName); err != nil { + var authoritative *trackPrefixCollisionError + if errors.As(err, &authoritative) { + return authoritative + } + } + return collision } if err := gc.AddRepo(entry); err != nil { return err @@ -795,3 +805,49 @@ func emitUntrackSummary(w io.Writer, target string, r untrackResult) { fmt.Fprintln(w, " "+progress.StatStrip(stats...)) fmt.Fprintln(w) } + +// preflightDaemonTrack never starts a daemon or authorizes a mutation. Its +// deadline includes dialing/handshake, and stays below the CLI refusal budget. +// Older daemons reject the distinct verb instead of treating it as Track. +func preflightDaemonTrack(ctx context.Context, path, name string) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + deadline := time.Now().Add(750 * time.Millisecond) + if callerDeadline, ok := ctx.Deadline(); ok && callerDeadline.Before(deadline) { + deadline = callerDeadline + } + if trackWait && trackWaitTimeout > 0 { + if waitDeadline := time.Now().Add(trackWaitTimeout); waitDeadline.Before(deadline) { + deadline = waitDeadline + } + } + client, err := daemon.DialToWithDeadline(daemon.SocketPath(), daemon.Handshake{Mode: daemon.ModeControl, ClientName: "cli"}, deadline) + if err != nil { + return err + } + defer client.Close() + stop := context.AfterFunc(ctx, func() { _ = client.Close() }) + defer stop() + if err := ctx.Err(); err != nil { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return context.DeadlineExceeded + } + response, err := client.ControlWithTimeout(daemon.ControlTrackPreflight, daemon.TrackParams{Path: path, Name: name, AsWorktree: trackAsWorktree}, remaining) + if err != nil { + return err + } + if !response.OK { + if response.ErrorCode == daemon.ErrPrefixCollision { + return &trackPrefixCollisionError{message: response.ErrorMsg} + } + return fmt.Errorf("track preflight rejected: %s %s", response.ErrorCode, response.ErrorMsg) + } + return nil +} diff --git a/cmd/gortex/track_catalog_test.go b/cmd/gortex/track_catalog_test.go index abfb213ed..ca84d8e1d 100644 --- a/cmd/gortex/track_catalog_test.go +++ b/cmd/gortex/track_catalog_test.go @@ -28,6 +28,11 @@ import ( ) func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) (*config.ConfigManager, *indexer.MultiIndexer) { + t.Helper() + return startCatalogTrackDaemonWithLogger(t, entries, loaded, zap.NewNop()) +} + +func startCatalogTrackDaemonWithLogger(t *testing.T, entries, loaded []config.RepoEntry, logger *zap.Logger) (*config.ConfigManager, *indexer.MultiIndexer) { t.Helper() dir := testenv.ShortTempDir(t) socket := filepath.Join(dir, "s") @@ -43,9 +48,9 @@ func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) ( languages.RegisterAll(reg) cm, err := config.NewConfigManager(gc.ConfigPath()) require.NoError(t, err) - idx := indexer.New(store, reg, config.Default().Index, zap.NewNop()) - mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, zap.NewNop()) - lc, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: zap.NewNop()}) + idx := indexer.New(store, reg, config.Default().Index, logger) + mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, logger) + lc, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: logger}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, lc.Close()) @@ -58,10 +63,10 @@ func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) ( require.NoError(t, err) require.NoError(t, result.CatalogErr) } - srv := gortexmcp.NewServer(query.NewEngine(store), store, idx, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm}) - d := daemon.New(socket, "test", zap.NewNop()) - d.Controller = &realController{graph: store, multiIndexer: mi, configManager: cm, lifecycle: lc, logger: zap.NewNop()} - d.MCPDispatcher = newMCPDispatcher(srv, mi, zap.NewNop()) + srv := gortexmcp.NewServer(query.NewEngine(store), store, idx, nil, logger, nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm}) + d := daemon.New(socket, "test", logger) + d.Controller = &realController{graph: store, multiIndexer: mi, configManager: cm, lifecycle: lc, logger: logger} + d.MCPDispatcher = newMCPDispatcher(srv, mi, logger) require.NoError(t, d.Listen()) done := make(chan error, 1) go func() { done <- d.Serve() }() diff --git a/cmd/gortex/track_collision_authority_test.go b/cmd/gortex/track_collision_authority_test.go new file mode 100644 index 000000000..8916a7c2b --- /dev/null +++ b/cmd/gortex/track_collision_authority_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" +) + +// The private in-process control server runs the real ownership lifecycle. +// No child daemon, timing measurement, or application store is involved. +func TestRunTrackConfiguredCollisionReachesDaemonAuthority(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + owner, loser := filepath.Join(t.TempDir(), "src"), filepath.Join(t.TempDir(), "src") + for _, root := range []string{owner, loser} { + require.NoError(t, os.MkdirAll(root, 0o755)) + wtGit(t, root, "init", "-q", "-b", "main") + require.NoError(t, os.WriteFile(filepath.Join(root, "main.go"), []byte("package sample\nfunc Alpha() {}\n"), 0o644)) + wtGit(t, root, "add", ".") + wtGit(t, root, "commit", "-q", "-m", "isolated ownership fixture") + } + core, logs := observer.New(zap.WarnLevel) + cm, mi := startCatalogTrackDaemonWithLogger(t, []config.RepoEntry{{Path: owner, Name: "src"}}, []config.RepoEntry{{Path: owner, Name: "src"}}, zap.New(core)) + before, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + ownerMetadata := mi.GetMetadata("src") + ownerConfig := cm.GetRepoConfig("src") + oldWait, oldName, oldAsWorktree := trackWait, trackName, trackAsWorktree + t.Cleanup(func() { trackWait, trackName, trackAsWorktree = oldWait, oldName, oldAsWorktree }) + trackName, trackAsWorktree = "", false + for _, wait := range []bool{false, true, false} { + trackWait = wait + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + err := runTrack(cmd, []string{loser}) + require.ErrorContains(t, err, owner) + var authoritative *trackPrefixCollisionError + require.ErrorAs(t, err, &authoritative) + after, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + require.Equal(t, before, after, "collision must never save the loser") + require.Same(t, ownerMetadata, mi.GetMetadata("src")) + require.Equal(t, ownerConfig, cm.GetRepoConfig("src")) + warnings := logs.FilterMessage("repository not loaded").All() + require.Len(t, warnings, 1, "CLI refusal must reach the daemon ownership decision; retries must deduplicate") + require.Contains(t, warnings[0].ContextMap()["error"], owner) + require.Contains(t, warnings[0].ContextMap()["error"], loser) + require.Contains(t, warnings[0].ContextMap()["error"], "--name") + } + trackWait = false + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + require.NoError(t, runTrack(cmd, []string{owner})) + require.Len(t, logs.FilterMessage("repository not loaded").All(), 1, "same-root no-op adds no collision warning") +} + +func TestRunTrackLocalCollisionCannotRegisterWhenDaemonHasNoOwner(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + owner, loser := filepath.Join(t.TempDir(), "src"), filepath.Join(t.TempDir(), "src") + for _, root := range []string{owner, loser} { + require.NoError(t, os.MkdirAll(root, 0o755)) + } + cm, mi := startCatalogTrackDaemon(t, []config.RepoEntry{{Path: owner, Name: "src"}}, nil) + before, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + oldWait, oldName, oldAsWorktree := trackWait, trackName, trackAsWorktree + t.Cleanup(func() { trackWait, trackName, trackAsWorktree = oldWait, oldName, oldAsWorktree }) + trackWait, trackName, trackAsWorktree = false, "", false + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + require.ErrorContains(t, runTrack(cmd, []string{loser}), owner) + after, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + require.Equal(t, before, after) + require.Nil(t, mi.GetMetadata("src"), "read-only preflight must never register the loser when daemon ownership is unresolved") +} diff --git a/docs/agents.md b/docs/agents.md index a28c27c44..5abdb023e 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -413,7 +413,7 @@ Current Codex hook coverage: | ------- | -------- | | `SessionStart` | Matches `startup|resume|clear|compact` and emits graph-tools orientation for new, resumed, cleared, and compacted sessions. | | Hook-observable local-tool `PreToolUse` terminal gate | Uses one match-all installer-owned hook so an enforceable localization `answer_ready` contract blocks every local tool Codex routes through `PreToolUse`, including Bash, `apply_patch`, MCP tools, and other local function tools. Hosted tools such as `WebSearch`, and specialized paths that opt out of the default hook path, are outside this host boundary. Without a terminal marker it is a local no-op; advisory completion still permits non-navigation contract operations. | -| Bash `PreToolUse` | Advises by default; `deny` hard-blocks graph-detectable fallback reads/searches; `rewrite` converts only an unambiguous indexed `cat ` into the exact public `gortex call read` mirror. Compound or ambiguous commands remain advisory. A shell command that would *rewrite* indexed source — `sed -i` / `perl -pi`, a `>` / `>>` redirect, `tee`, or an inline interpreter script opening the path for writing (or reading it and writing it back) — gets the same redirect Edit and Write get, escalating to a hard block under `GORTEX_HOOK_BLOCK_EDIT`. Writing a path the daemon does not know is a new file and passes through. Inline Python (`python3 -c`, a `python3 - <` into the exact public `gortex call read` mirror. Compound or ambiguous commands remain advisory. A shell command that would *rewrite* indexed source — `sed -i` / `perl -pi`, a `>` / `>>` redirect, `tee`, or an inline interpreter script opening the path for writing (or reading it and writing it back) — gets the same redirect Edit and Write get, escalating to a hard block under `GORTEX_HOOK_BLOCK_EDIT`. Writing a path the daemon does not know is a new file and passes through. Inline Python (`python3 -c`, a `python3 - < 0 { + log.Printf("store_sqlite: dense indexes missing after bulk window close indexes=%s", strings.Join(s.denseIndexesMissingNames(), ",")) + } +} + +// setDenseIndexesMissingLocked replaces the missing-index record. The caller +// holds writeMu; the slice is never modified after it is published. +func (s *Store) setDenseIndexesMissingLocked(missing []bulkDroppableIndex) { + if len(missing) == 0 { + s.denseIndexesMissing.Store(nil) + return + } + s.denseIndexesMissing.Store(&missing) +} + +// denseIndexesMissingRecord returns a copy of the missing-index record, or nil. +func (s *storeCore) denseIndexesMissingRecord() []bulkDroppableIndex { + missing := s.denseIndexesMissing.Load() + if missing == nil { + return nil + } + return append([]bulkDroppableIndex(nil), (*missing)...) +} + +// denseIndexesMissingNames returns the names in the missing-index record, or +// nil. +func (s *storeCore) denseIndexesMissingNames() []string { + missing := s.denseIndexesMissing.Load() + if missing == nil { + return nil + } + names := make([]string, 0, len(*missing)) + for _, idx := range *missing { + names = append(names, idx.name) + } + return names +} + +func schemaIndexNames(ctx context.Context, conn *sql.Conn) (map[string]bool, error) { + rows, err := conn.QueryContext(ctx, `SELECT name FROM sqlite_schema WHERE type = 'index'`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + present := make(map[string]bool) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + present[name] = true + } + return present, rows.Err() +} + func wrapBulkRestoreError(pragma string, err error) error { if err == nil { return nil @@ -1381,14 +1561,48 @@ func shouldBackoffBulkRowCheckpoint(err error) bool { // rows has been indexed before, whichever view wrote them, so it is not the // cold store this fast path is for. func coldGraphStoreEmpty(ctx context.Context, conn *sql.Conn) bool { - var empty int + return coldGraphStoreRefusal(ctx, conn) == "" +} + +// One bounded probe retains the cold-store proof and identifies its first +// failed predicate without counting or hydrating any graph rows. +func coldGraphStoreRefusal(ctx context.Context, conn *sql.Conn) string { + var reason string err := conn.QueryRowContext(ctx, ` -SELECT NOT EXISTS(SELECT 1 FROM nodes WHERE view_gen = ?) - AND NOT EXISTS(SELECT 1 FROM edges WHERE view_gen = ?) - AND NOT EXISTS(SELECT 1 FROM file_mtimes) - AND NOT EXISTS(SELECT 1 FROM repo_index_state)`, - baseViewGeneration, baseViewGeneration).Scan(&empty) - return err == nil && empty == 1 +SELECT CASE + WHEN EXISTS(SELECT 1 FROM nodes WHERE view_gen = ?) THEN 'nodes' + WHEN EXISTS(SELECT 1 FROM edges WHERE view_gen = ?) THEN 'edges' + WHEN EXISTS(SELECT 1 FROM file_mtimes) THEN 'file_mtimes' + WHEN EXISTS(SELECT 1 FROM repo_index_state) THEN 'repo_index_state' + ELSE '' END`, baseViewGeneration, baseViewGeneration).Scan(&reason) + if err != nil { + return "cold_probe_error" + } + return reason +} + +// WriterPragmaForTest reads an integer PRAGMA off the store's writer +// connection: the pinned bulk connection while a window holds one, the single +// writer-pool connection otherwise. PRAGMAs such as synchronous are +// connection-local, so a read-pool connection cannot show what the writer runs +// with. It takes the write gate under ctx, so a deadline also proves the gate +// is free. name must be a bare PRAGMA name. +func (s *Store) WriterPragmaForTest(ctx context.Context, name string) (int64, error) { + for _, r := range name { + if (r < 'a' || r > 'z') && r != '_' { + return 0, fmt.Errorf("store_sqlite: not a pragma name: %q", name) + } + } + if err := s.writeMu.LockContext(ctx); err != nil { + return 0, err + } + defer s.writeMu.Unlock() + conn, release, err := s.activeWriteConnLocked(ctx) + if err != nil { + return 0, err + } + defer release() + return pragmaInt(ctx, conn, name) } // pragmaInt reads a single-integer PRAGMA (synchronous, cache_size, wal_autocheckpoint) off the diff --git a/internal/graph/store_sqlite/bulk_load_generation_test.go b/internal/graph/store_sqlite/bulk_load_generation_test.go index 67ae10223..0f3597b08 100644 --- a/internal/graph/store_sqlite/bulk_load_generation_test.go +++ b/internal/graph/store_sqlite/bulk_load_generation_test.go @@ -210,6 +210,35 @@ func TestGenerationBulkLoadRefusesWhatItCannotProveEmpty(t *testing.T) { }) } +// TestAbortCoordinatedBulkLoadLeavesAGenerationWindowToItsOwner: an owner that +// aborts after its coordinated end failed may find the writer pinned by a +// payload generation opened since. The abort must leave that window, and the +// checkpoint lease only its own end releases, to the generation's owner. +func TestAbortCoordinatedBulkLoadLeavesAGenerationWindowToItsOwner(t *testing.T) { + store, _ := openTempStore(t) + engaged, err := store.BeginGenerationBulkLoad(3) + if err != nil || !engaged { + t.Fatalf("BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + if err := store.AbortCoordinatedBulkLoad(); err != nil { + t.Fatalf("AbortCoordinatedBulkLoad: %v", err) + } + if id, open := store.InGenerationBulkLoad(); id != 3 || !open { + t.Fatalf("generation window after abort = (%d, %v), want (3, true)", id, open) + } + if err := store.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } + // The owner's end released the lease: the next generation can open. + engaged, err = store.BeginGenerationBulkLoad(4) + if err != nil || !engaged { + t.Fatalf("next BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + if err := store.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } +} + func TestGenerationBulkLoadLeavesBaseReadersAndIndexesAlone(t *testing.T) { store, _ := openTempStore(t) baseNodes, baseEdges := bulkFixture(256, 512) diff --git a/internal/graph/store_sqlite/bulk_load_test.go b/internal/graph/store_sqlite/bulk_load_test.go index 528c7b659..dd846d7c1 100644 --- a/internal/graph/store_sqlite/bulk_load_test.go +++ b/internal/graph/store_sqlite/bulk_load_test.go @@ -835,6 +835,83 @@ func TestUncoordinatedBulkSealFailureRetriesAtFlushBoundary(t *testing.T) { integrityOK(t, s.db) } +// TestFlushBulkSealPanicDoesNotDeadlock: a seal that panics with a non-error +// value must unwind with the write gate released and the window still +// retryable, so the owner's next exit call seals, restores and closes it. +func TestFlushBulkSealPanicDoesNotDeadlock(t *testing.T) { + for _, tc := range []struct { + name string + begin func(*Store) bool + end func(*Store) error + }{ + {"standalone", func(s *Store) bool { s.BeginBulkLoad(); return s.bulkConn != nil }, (*Store).FlushBulk}, + {"coordinated", (*Store).BeginCoordinatedBulkLoad, (*Store).EndCoordinatedBulkLoad}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _ := openTempStore(t) + syncBefore := pragmaIntDB(t, s.writerDB, "synchronous") + if !tc.begin(s) { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + calls := 0 + defer s.SetBulkSealHookForTest(func(string) error { + calls++ + if calls == 1 { + panic("injected seal panic") + } + return nil + })() + + recovered := func() (r any) { + defer func() { r = recover() }() + _ = tc.end(s) + return nil + }() + if recovered != "injected seal panic" { + t.Fatalf("recovered %v, want the injected seal panic", recovered) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := s.writeMu.LockContext(ctx); err != nil { + // Hand the leaked hold back so cleanup's Close cannot hang too. + s.writeMu.Unlock() + t.Fatalf("write gate still held after the seal panic: %v", err) + } + retryable := s.bulkConn != nil && s.bulkIndexesDeferred && s.bulkWindowOpen.Load() && + s.coordinatedBulkLoad == (tc.name == "coordinated") + s.writeMu.Unlock() + if !retryable { + t.Fatal("seal panic lost the window's retryable state") + } + + if err := tc.end(s); err != nil { + t.Fatalf("exit retry after the seal panic: %v", err) + } + if calls <= 1 { + t.Fatalf("seal hook calls = %d, want the retry to seal again", calls) + } + if s.bulkConn != nil || s.bulkIndexesDeferred || s.coordinatedBulkLoad || s.bulkWindowOpen.Load() { + t.Fatalf("retry left bulk state: conn=%v deferred=%v coordinated=%v open=%v", + s.bulkConn != nil, s.bulkIndexesDeferred, s.coordinatedBulkLoad, s.bulkWindowOpen.Load()) + } + if got := pragmaIntDB(t, s.writerDB, "synchronous"); got != syncBefore { + t.Fatalf("writer synchronous = %d, want restored %d", got, syncBefore) + } + present := indexNames(t, s.db) + for _, idx := range bulkDroppableIndexes { + if !present[idx.name] { + t.Fatalf("index %s missing after the retried seal", idx.name) + } + } + if got := s.NodeCount(); got != 1 { + t.Fatalf("node count = %d, want 1", got) + } + integrityOK(t, s.db) + }) + } +} + func TestCoordinatedBulkLoadRoutesSingleRowSidecarsOnPinnedConnection(t *testing.T) { s, _ := openTempStore(t) s.db.SetMaxOpenConns(1) diff --git a/internal/graph/store_sqlite/dense_index_repair.go b/internal/graph/store_sqlite/dense_index_repair.go new file mode 100644 index 000000000..a8e5a0764 --- /dev/null +++ b/internal/graph/store_sqlite/dense_index_repair.go @@ -0,0 +1,230 @@ +package store_sqlite + +import ( + "context" + "fmt" + "log" + "math" + "path/filepath" + "strings" + "time" + + "github.com/zzet/gortex/internal/platform" +) + +// Dense-index repair: the maintenance-lane job that rebuilds the dense indexes +// a cold bulk window closed without (denseIndexesMissing). The usual cause is +// the volume filling up during the seal, so the job is built around not +// making that worse: +// +// - it never runs inline. The exit that recorded the missing indexes posts +// one request after it has released the write gate, and the lane worker +// runs it as a quiescing whole-database job, like VACUUM. +// - it does not try while the disk is still full. An attempt is skipped +// while the free space on the store's volume is below the store's used +// bytes (with a floor of denseIndexRepairMinFreeBytes). The dense indexes +// are a fraction of the used pages, and a CREATE INDEX needs about twice +// the index size at its peak (the sort plus the WAL frames), so a volume +// with the store's own size free can always take the rebuild while a full +// one is never asked to. +// - it backs off. A skipped or failed attempt re-posts itself after +// denseIndexRepairBaseDelay, doubling up to denseIndexRepairMaxDelay, +// so a volume that stays full costs one statfs per backoff step and one +// log line, not a retry loop. +// - it holds the write gate one index at a time. The DDL is exactly what +// createGraphCoreIndexes runs (bulkDroppableIndexes is the single source +// for both, copied into the record by the window that closed without it, +// so the lane never reads that list), but each CREATE INDEX takes and +// releases writeMu on its own, so a writer waits behind at most one index +// build rather than the set. +// +// It is deliberately not part of the lane's settle point (waitMaintenanceIdle): +// a long rebuild must not hold AwaitMaintenanceIdle callers, which sit on +// publication paths. A completed repair asks the lane for a statistics pass, +// because the rebuilt indexes have no sqlite_stat1 rows yet. + +// Vars rather than constants only so the in-package cases can shorten them. +var ( + denseIndexRepairBaseDelay = 30 * time.Second + denseIndexRepairMaxDelay = 15 * time.Minute + denseIndexRepairMinFreeBytes = int64(64 << 20) +) + +// scheduleDenseIndexRepair posts one immediate repair request, resetting any +// backoff an earlier record had built up. The caller has just recorded the +// missing set and must not hold writeMu; like scheduleWALDrain it takes only +// maintenanceSched and never blocks. +func (s *Store) scheduleDenseIndexRepair() { + if s.coreless() { + return + } + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal == nil { + return + } + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + if s.denseRepairOwed { + return + } + s.denseRepairOwed = true + select { + case s.maintenanceSignal <- struct{}{}: + default: + } +} + +// takeDenseIndexRepair claims an owed repair for the lane worker. +func (s *Store) takeDenseIndexRepair() bool { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if !s.denseRepairOwed || s.maintenanceClosed { + return false + } + s.denseRepairOwed = false + return true +} + +// retryDenseIndexRepairLater re-posts the repair once the current backoff +// elapses and doubles the backoff for the attempt after it. It returns the +// delay it armed, or 0 when the lane is closed. +func (s *Store) retryDenseIndexRepairLater() time.Duration { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal == nil { + return 0 + } + delay := s.denseRepairDelay + if delay <= 0 { + delay = denseIndexRepairBaseDelay + } + s.denseRepairDelay = min(2*delay, denseIndexRepairMaxDelay) + s.stopDenseIndexRepairTimerLocked() + signal := s.maintenanceSignal + s.denseRepairTimer = time.AfterFunc(delay, func() { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal != signal || s.denseRepairOwed { + return + } + s.denseRepairOwed = true + select { + case signal <- struct{}{}: + default: + } + }) + return delay +} + +// stopDenseIndexRepairTimerLocked cancels a pending retry. The caller holds +// maintenanceSched. +func (s *Store) stopDenseIndexRepairTimerLocked() { + if s.denseRepairTimer != nil { + s.denseRepairTimer.Stop() + s.denseRepairTimer = nil + } +} + +// runDenseIndexRepair is one attempt, run by the lane worker. Nothing it hits +// is an error to anyone: the record stays, the attempt is logged, and the +// backoff decides when the next one comes. +func (s *Store) runDenseIndexRepair(ctx context.Context) { + missing := s.denseIndexesMissingNames() + if len(missing) == 0 { + // A later seal already rebuilt them. + s.maintenanceSched.Lock() + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + s.maintenanceSched.Unlock() + return + } + avail, need, ok := s.denseIndexRepairSpace() + if !ok { + s.denseIndexRepairSkips.Add(1) + delay := s.retryDenseIndexRepairLater() + log.Printf("store_sqlite: dense index repair skipped: low free space avail_bytes=%d need_bytes=%d indexes=%s retry_in=%s", avail, need, strings.Join(missing, ","), delay) + return + } + started := time.Now() + err := s.runMaintenance(ctx, maintenanceDenseIndexRepair, true, func(ctx context.Context) error { + s.denseIndexRepairAttempts.Add(1) + return s.repairDenseIndexes(ctx) + }) + if err != nil { + if ctx.Err() != nil { + return + } + delay := s.retryDenseIndexRepairLater() + log.Printf("store_sqlite: dense index repair deferred indexes=%s retry_in=%s error=%q", strings.Join(s.denseIndexesMissingNames(), ","), delay, err) + return + } + s.maintenanceSched.Lock() + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + s.maintenanceSched.Unlock() + s.denseIndexRepairs.Add(1) + log.Printf("store_sqlite: dense index repair complete indexes=%s elapsed=%s", strings.Join(missing, ","), time.Since(started).Round(time.Millisecond)) + // The rebuilt indexes carry no statistics yet; the lane's next pass + // notices that and analyzes them. + s.schedulePublishMaintenance() +} + +// denseIndexRepairSpace reports the free space on the store's volume, the +// amount an attempt needs, and whether the attempt may run. An unreadable +// free-space figure is a skip: the gate exists for the full volume, which is +// exactly where a probe may fail too. So is an unreadable store size +// (CompactStats reads 0, 0 when its PRAGMAs fail), which would otherwise +// shrink the need to the floor whatever the store's real size. +func (s *Store) denseIndexRepairSpace() (avail, need int64, ok bool) { + free, total := s.CompactStats() + if total <= 0 { + return -1, -1, false + } + need = max(total-free, denseIndexRepairMinFreeBytes) + probe := platform.DiskAvailBytes + if override := s.denseRepairDiskAvail.Load(); override != nil && *override != nil { + probe = *override + } + bytes, err := probe(filepath.Dir(s.dbPath)) + if err != nil { + return -1, need, false + } + avail = int64(min(bytes, uint64(math.MaxInt64))) + return avail, need, avail >= need +} + +// repairDenseIndexes rebuilds every index in the missing record, one write-gate +// hold per index. It refuses, like VACUUM, while a bulk window pins the writer: +// that window drops and seals the same indexes itself. +func (s *Store) repairDenseIndexes(ctx context.Context) error { + for _, idx := range s.denseIndexesMissingRecord() { + if err := s.repairDenseIndex(ctx, idx); err != nil { + return err + } + } + return nil +} + +func (s *Store) repairDenseIndex(ctx context.Context, idx bulkDroppableIndex) error { + if err := s.writeMu.LockContext(ctx); err != nil { + return fmt.Errorf("%w: %s: write gate: %w", ErrMaintenanceBusy, maintenanceDenseIndexRepair, err) + } + defer s.writeMu.Unlock() + if s.bulkConn != nil { + return fmt.Errorf("%w: %s: bulk writer pinned", ErrMaintenanceBusy, maintenanceDenseIndexRepair) + } + if _, err := s.writerDB.ExecContext(ctx, idx.ddl); err != nil { + return fmt.Errorf("store_sqlite: repair index %s: %w", idx.name, err) + } + // Under the hold that built it, so whichever window recorded the name, the + // index it names is present now. + var remaining []bulkDroppableIndex + for _, missing := range s.denseIndexesMissingRecord() { + if missing.name != idx.name { + remaining = append(remaining, missing) + } + } + s.setDenseIndexesMissingLocked(remaining) + return nil +} diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go new file mode 100644 index 000000000..b65bccec6 --- /dev/null +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -0,0 +1,306 @@ +package store_sqlite + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/zzet/gortex/internal/graph" +) + +// shortenDenseIndexRepairBackoff makes a skipped repair retry within +// milliseconds for the rest of the test. +func shortenDenseIndexRepairBackoff(t *testing.T) { + t.Helper() + base, maxDelay := denseIndexRepairBaseDelay, denseIndexRepairMaxDelay + denseIndexRepairBaseDelay, denseIndexRepairMaxDelay = 5*time.Millisecond, 20*time.Millisecond + t.Cleanup(func() { denseIndexRepairBaseDelay, denseIndexRepairMaxDelay = base, maxDelay }) +} + +// TestFailedSealReportsDenseIndexesMissing: a cold window that closes without +// rebuilding its dense indexes must still restore durability and close, must +// say which indexes it left absent instead of letting the planner probe read +// their absence as a clean store, must not retry the rebuild while the volume +// is full, and must rebuild them in one maintenance attempt once it is not, +// followed by the statistics pass the rebuilt indexes need. +func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { + shortenDenseIndexRepairBackoff(t) + allDroppable := make([]string, 0, len(bulkDroppableIndexes)) + for _, idx := range bulkDroppableIndexes { + allDroppable = append(allDroppable, idx.name) + } + for _, tc := range []struct { + name string + // fail closes the window the case's way and returns the indexes it + // expects to be left missing, in declaration order. + fail func(t *testing.T, s *Store) []string + }{ + {"flush seal error", func(t *testing.T, s *Store) []string { + s.BeginBulkLoad() + if s.bulkConn == nil { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + injected := errors.New("injected seal failure") + defer s.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" || index == "edges_by_kind" { + return injected + } + return nil + })() + err := s.FlushBulk() + if !errors.Is(err, injected) { + t.Fatalf("FlushBulk error = %v, want the injected seal failure", err) + } + for _, name := range []string{"nodes_by_kind", "edges_by_kind"} { + if !strings.Contains(err.Error(), name) { + t.Fatalf("FlushBulk error %q does not name %s", err, name) + } + } + return []string{"nodes_by_kind", "edges_by_kind"} + }}, + {"abort before seal", func(t *testing.T, s *Store) []string { + if !s.BeginCoordinatedBulkLoad() { + t.Fatal("coordinated fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + if err := s.AbortCoordinatedBulkLoad(); err != nil { + t.Fatalf("AbortCoordinatedBulkLoad: %v", err) + } + return allDroppable + }}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) + syncBefore := pragmaIntDB(t, s.writerDB, "synchronous") + want := tc.fail(t, s) + + if s.bulkConn != nil || s.coordinatedBulkLoad || s.bulkWindowOpen.Load() { + t.Fatal("window left open after closing unsealed") + } + if got := pragmaIntDB(t, s.writerDB, "synchronous"); got != syncBefore { + t.Fatalf("writer synchronous = %d, want restored %d", got, syncBefore) + } + health, err := s.PlannerStatsHealth(context.Background()) + if err != nil { + t.Fatalf("PlannerStatsHealth: %v", err) + } + if health.Reason != plannerStatsDenseIndexesMissingReason || health.Stale { + t.Fatalf("health reason=%q stale=%v, want %q and not stale", health.Reason, health.Stale, plannerStatsDenseIndexesMissingReason) + } + if !reflect.DeepEqual(health.DenseIndexesMissing, want) { + t.Fatalf("DenseIndexesMissing = %v, want %v", health.DenseIndexesMissing, want) + } + present := indexNames(t, s.db) + for _, name := range allDroppable { + missing := false + for _, w := range want { + missing = missing || w == name + } + if present[name] == missing { + t.Fatalf("index %s present=%v, want present=%v", name, present[name], !missing) + } + } + + // While the volume reads as full the lane keeps skipping on its + // backoff and never reaches a CREATE INDEX. + waitUntil(t, "a repair attempt skipped for free space", func() bool { return s.denseIndexRepairSkips.Load() >= 2 }) + if got := s.denseIndexRepairAttempts.Load(); got != 0 { + t.Fatalf("repair attempts while full = %d, want 0", got) + } + if health, err := s.PlannerStatsHealth(context.Background()); err != nil || health.Reason != plannerStatsDenseIndexesMissingReason { + t.Fatalf("health while full = %q, %v; want %q", health.Reason, err, plannerStatsDenseIndexesMissingReason) + } + + // Once space is back, the next retry rebuilds every listed index. + full.Store(false) + waitUntil(t, "the dense index repair", func() bool { return s.denseIndexRepairs.Load() == 1 }) + if got := s.denseIndexRepairAttempts.Load(); got != 1 { + t.Fatalf("repair attempts = %d, want one", got) + } + health, err = s.PlannerStatsHealth(context.Background()) + if err != nil { + t.Fatalf("PlannerStatsHealth after repair: %v", err) + } + if health.Reason == plannerStatsDenseIndexesMissingReason || len(health.DenseIndexesMissing) != 0 { + t.Fatalf("after repair reason=%q missing=%v, want both cleared", health.Reason, health.DenseIndexesMissing) + } + present = indexNames(t, s.db) + for _, name := range allDroppable { + if !present[name] { + t.Fatalf("index %s missing after the repair", name) + } + } + // A rebuilt index has no sqlite_stat1 row, and once the repair is + // done nothing else asks for a statistics pass, so the repair must + // ask for one itself. nodes_by_kind holds the case's node. + waitUntil(t, "statistics for the rebuilt nodes_by_kind", func() bool { + return s.plannerStatsIndexesWithStats(context.Background())["nodes_by_kind"] + }) + integrityOK(t, s.db) + }) + } +} + +// TestDenseIndexRepairBuildsFromTheRecordedDDL: the repair runs the DDL the +// closing window recorded with each missing name, never a lookup in +// bulkDroppableIndexes made when the lane gets to it. In-package cases replace +// that list to force a seal failure and put it back while the lane may still be +// on its way to the repair; a lookup there reads the list unsynchronised with +// that restore, and finds no DDL for a name the record still holds. +func TestDenseIndexRepairBuildsFromTheRecordedDDL(t *testing.T) { + shortenDenseIndexRepairBackoff(t) + s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) + + originalIndexes := bulkDroppableIndexes + bulkDroppableIndexes = append(append([]bulkDroppableIndex(nil), originalIndexes...), bulkDroppableIndex{ + name: "forced_bad_index", + ddl: `CREATE INDEX forced_bad_index ON definitely_missing_table(id)`, + }) + s.BeginBulkLoad() + if s.bulkConn == nil { + bulkDroppableIndexes = originalIndexes + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + err := s.FlushBulk() + bulkDroppableIndexes = originalIndexes + if err == nil || !strings.Contains(err.Error(), "forced_bad_index") { + t.Fatalf("FlushBulk error = %v, want the forced index's DDL failure", err) + } + + // The list no longer names the index; the record does, with the DDL that + // cannot succeed. Every attempt must run it and fail, not report a repair. + full.Store(false) + waitUntil(t, "a finished repair attempt", func() bool { + return s.denseIndexRepairs.Load() > 0 || s.denseIndexRepairAttempts.Load() >= 2 + }) + if got := s.denseIndexRepairs.Load(); got != 0 { + t.Fatalf("repairs = %d for an index whose DDL cannot run, want 0", got) + } + if got := s.denseIndexesMissingNames(); !reflect.DeepEqual(got, []string{"forced_bad_index"}) { + t.Fatalf("missing record = %v, want [forced_bad_index]", got) + } +} + +// TestDenseIndexRepairSkipsWithoutTheStoreSize: an attempt is sized by the +// store's own used bytes, and CompactStats reads 0, 0 when it cannot read +// them. That must skip the attempt, as an unreadable free-space figure does, +// not shrink the need to the floor and let a large store rebuild on a volume +// that is nearly full. +func TestDenseIndexRepairSkipsWithoutTheStoreSize(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + s := &Store{storeCore: &storeCore{db: db, dbPath: filepath.Join(t.TempDir(), "store.sqlite")}} + diskAvail := func(string) (uint64, error) { return 1 << 40, nil } + s.denseRepairDiskAvail.Store(&diskAvail) + if _, total := s.CompactStats(); total != 0 { + t.Fatalf("CompactStats total = %d on a closed pool, want 0", total) + } + if avail, need, ok := s.denseIndexRepairSpace(); ok { + t.Fatalf("attempt allowed with the store size unreadable: avail=%d need=%d", avail, need) + } +} + +// TestDenseIndexRepairWaitsForAPinnedWriter: a repair that comes due while a +// window pins the writer defers, as VACUUM does, instead of building under +// that window, and builds on a later retry once the window has closed. A +// generation window is the one that can open on the populated store a failed +// seal leaves behind. +func TestDenseIndexRepairWaitsForAPinnedWriter(t *testing.T) { + shortenDenseIndexRepairBackoff(t) + s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) + s.BeginBulkLoad() + if s.bulkConn == nil { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + injected := errors.New("injected seal failure") + restore := s.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + err := s.FlushBulk() + restore() + if !errors.Is(err, injected) { + t.Fatalf("FlushBulk error = %v, want the injected seal failure", err) + } + engaged, err := s.BeginGenerationBulkLoad(3) + if err != nil || !engaged { + t.Fatalf("BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + + full.Store(false) + waitUntil(t, "two repair attempts under the pinned writer", func() bool { return s.denseIndexRepairAttempts.Load() >= 2 }) + if got := s.denseIndexRepairs.Load(); got != 0 { + t.Fatalf("repairs under a pinned writer = %d, want 0", got) + } + if got := s.denseIndexesMissingNames(); !reflect.DeepEqual(got, []string{"nodes_by_kind"}) { + t.Fatalf("missing record under a pinned writer = %v, want [nodes_by_kind]", got) + } + if indexNames(t, s.db)["nodes_by_kind"] { + t.Fatal("nodes_by_kind was built under the pinned writer") + } + + if err := s.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } + waitUntil(t, "the repair after the window closed", func() bool { return s.denseIndexRepairs.Load() == 1 }) + if !indexNames(t, s.db)["nodes_by_kind"] { + t.Fatal("nodes_by_kind missing after the repair") + } + if got := s.denseIndexesMissingNames(); len(got) != 0 { + t.Fatalf("missing record after the repair = %v, want empty", got) + } +} + +func waitUntil(t *testing.T, what string, done func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !done() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/graph/store_sqlite/first_index_drain.go b/internal/graph/store_sqlite/first_index_drain.go new file mode 100644 index 000000000..8bc764897 --- /dev/null +++ b/internal/graph/store_sqlite/first_index_drain.go @@ -0,0 +1,48 @@ +package store_sqlite + +import "github.com/zzet/gortex/internal/graph" + +// BulkLoadWindow snapshots the owner while the writer is between drain chunks. +func (s *Store) BulkLoadWindow() string { + s.writeMu.Lock() + defer s.writeMu.Unlock() + switch { + case s.generationBulkLoad != 0: + return "generation_noop" + case s.coordinatedBulkLoad: + return "coordinated_noop" + case s.bulkConn != nil: + return "cold" + default: + return "none" + } +} + +// RecordFirstIndexDrain retains one value per repository, shared by every +// generation handle and temporary indexer over this open store. +func (s *Store) RecordFirstIndexDrain(repo string, observation graph.FirstIndexDrain) { + s.firstIndexDrainsMu.Lock() + defer s.firstIndexDrainsMu.Unlock() + if s.firstIndexDrains == nil { + s.firstIndexDrains = make(map[string]graph.FirstIndexDrain) + } + s.firstIndexDrains[repo] = observation +} + +// LastFirstIndexDrains does no SQL or writer admission. Return a detached map +// so status and cached health payloads cannot mutate or race the live record. +func (s *Store) LastFirstIndexDrains() map[string]graph.FirstIndexDrain { + s.firstIndexDrainsMu.Lock() + defer s.firstIndexDrainsMu.Unlock() + if len(s.firstIndexDrains) == 0 { + return nil + } + out := make(map[string]graph.FirstIndexDrain, len(s.firstIndexDrains)) + for repo, observation := range s.firstIndexDrains { + out[repo] = observation + } + return out +} + +// DenseIndexesMissing reports the existing atomic repair record without SQL. +func (s *Store) DenseIndexesMissing() []string { return s.denseIndexesMissingNames() } diff --git a/internal/graph/store_sqlite/first_index_drain_test.go b/internal/graph/store_sqlite/first_index_drain_test.go new file mode 100644 index 000000000..76bfe4580 --- /dev/null +++ b/internal/graph/store_sqlite/first_index_drain_test.go @@ -0,0 +1,41 @@ +package store_sqlite + +import ( + "testing" + "time" + + "github.com/zzet/gortex/internal/graph" +) + +func TestFirstIndexDrainSnapshotAvoidsWriterAndCallerMutation(t *testing.T) { + s, _ := openTempStore(t) + s.RecordFirstIndexDrain("repo", graph.FirstIndexDrain{Window: "standalone", Nodes: 3}) + s.writeMu.Lock() + result := make(chan map[string]graph.FirstIndexDrain, 1) + go func() { result <- s.LastFirstIndexDrains() }() + var snapshot map[string]graph.FirstIndexDrain + select { + case snapshot = <-result: + s.writeMu.Unlock() + case <-time.After(5 * time.Second): + s.writeMu.Unlock() + t.Fatal("status snapshot queued behind writer") + } + delete(snapshot, "repo") + actual := s.LastFirstIndexDrains() + if actual["repo"].Nodes != 3 { + t.Fatalf("caller changed retained observation: %v", actual) + } + if err := s.RekeyRepoPrefix("repo", "renamed"); err != nil { + t.Fatal(err) + } + if got := s.LastFirstIndexDrains(); len(got) != 1 || got["renamed"].Nodes != 3 { + t.Fatalf("rekey lost or duplicated observation: %v", got) + } + if err := s.PurgeRepo("renamed"); err != nil { + t.Fatal(err) + } + if got := s.LastFirstIndexDrains(); len(got) != 0 { + t.Fatalf("purged repository retained observation: %v", got) + } +} diff --git a/internal/graph/store_sqlite/planner_stats_freshness.go b/internal/graph/store_sqlite/planner_stats_freshness.go index 86dbae87b..193aa2aef 100644 --- a/internal/graph/store_sqlite/planner_stats_freshness.go +++ b/internal/graph/store_sqlite/planner_stats_freshness.go @@ -150,6 +150,14 @@ const ( // finalize anyway. const plannerStatsBulkWindowReason = "bulk_window_active" +// plannerStatsDenseIndexesMissingReason is reported instead of a verdict while +// a cold bulk window that closed unsealed still owes dense indexes (listed in +// PlannerStatsFreshness.DenseIndexesMissing). Some of the critical indexes may +// be among them, and every rule would read an absent index as a clean store, +// so the probe names the real problem instead. No ANALYZE can fix it: the +// maintenance lane's repair rebuilds the indexes and the reason clears with it. +const plannerStatsDenseIndexesMissingReason = "dense_indexes_missing" + // Reasons a stale verdict can come back UNREFRESHED. None of them is a // failure, and none of them stamps the ledger or arms the anti-loop guard: the // verdict stands, the resume cursor keeps the indexes already finished, and the @@ -620,6 +628,13 @@ func (s *Store) plannerStatsHealth(ctx context.Context, probeReceivers bool) (gr health.Reason = plannerStatsBulkWindowReason return health, bases, nil } + // Same shape as the window above: not a judgeable state, Stale stays + // false and Known stays false, so nothing is seeded from it. + if missing := s.denseIndexesMissingNames(); len(missing) > 0 { + health.Reason = plannerStatsDenseIndexesMissingReason + health.DenseIndexesMissing = missing + return health, bases, nil + } present, err := s.plannerStatsPresentIndexes(ctx) if err != nil { diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index 3fec85127..a8d1ec5a9 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -66,6 +66,9 @@ type backgroundCheckpointCoordination struct { } type storeCore struct { + firstIndexDrainsMu sync.Mutex + firstIndexDrains map[string]graph.FirstIndexDrain + // baseFTSPages retains only exact ranked rows keyed by same-transaction // global FTS state and mutable base ownership revision. baseFTSPages symbolFTSPageCache @@ -340,6 +343,22 @@ type storeCore struct { maintenanceDrainRunning bool maintenanceDrainReason string + // Dense-index repair (dense_index_repair.go). denseRepairOwed is the + // one-slot request the lane worker consumes, posted by a bulk-window exit + // that left indexes missing or by the backoff timer; denseRepairDelay is + // the backoff the next retry waits and denseRepairTimer that pending + // retry. All three are guarded by maintenanceSched. The counters record + // attempts that reached their DDL, attempts skipped for free space, and + // completed repairs; denseRepairDiskAvail is a test override of the + // free-space probe. + denseRepairOwed bool + denseRepairDelay time.Duration + denseRepairTimer *time.Timer + denseIndexRepairAttempts atomic.Int64 + denseIndexRepairSkips atomic.Int64 + denseIndexRepairs atomic.Int64 + denseRepairDiskAvail atomic.Pointer[func(dir string) (uint64, error)] + // Lane counters. Requests counts scheduling calls, passes counts lane // passes actually started (so a burst of requests collapsing into one // pass is observable), jobs counts actions that reached their SQL, @@ -436,6 +455,11 @@ type storeCore struct { // synchronously under writeMu on cold paths; live FTS finalize emits after // releasing the gate. Cold observers must not call back into Store. bulkFinalizeObserver func(bulkFinalizeEvent) + // bulkSealHook and batchWriteHook are SetBulkSealHookForTest's and + // SetBatchWriteHookForTest's hooks. Per store rather than package-wide so + // parallel tests that each own a store do not share them. + bulkSealHook atomic.Pointer[func(index string) error] + batchWriteHook atomic.Pointer[func(nodeRows, edgeRows int) error] // Live FTS-finalize statement trace for tests; invoked only on those two // maintenance calls, never on ordinary graph/edit writes. ftsMaintenanceObserver func(string, []any) @@ -507,6 +531,16 @@ type storeCore struct { // health probe. Written only under writeMu, by syncBulkWindowLocked. bulkWindowOpen atomic.Bool + // denseIndexesMissing holds the dense indexes a cold bulk window closed + // without rebuilding (its seal failed, or it was aborted before sealing), + // in bulkDroppableIndexes order and each with the DDL that builds it; nil + // when none are owed. Written only under writeMu and replaced, never + // mutated in place, so the health probe reads it without a lock for the + // same reason it reads bulkWindowOpen, and the maintenance lane's repair + // builds from it rather than reading bulkDroppableIndexes itself. + // Not persisted: Open's createGraphCoreIndexes rebuilds them on restart. + denseIndexesMissing atomic.Pointer[[]bulkDroppableIndex] + // Prepared statements (compiled once in Open, closed in Close). stmtInsertNode *sql.Stmt stmtGetNode *sql.Stmt diff --git a/internal/graph/store_sqlite/store_compact.go b/internal/graph/store_sqlite/store_compact.go index e1a6959ab..da1ce87af 100644 --- a/internal/graph/store_sqlite/store_compact.go +++ b/internal/graph/store_sqlite/store_compact.go @@ -92,6 +92,9 @@ const ( maintenancePlannerStats maintenanceJob = "planner_stats" maintenanceVacuum maintenanceJob = "vacuum" maintenanceCheckpoint maintenanceJob = "wal_checkpoint_truncate" + // maintenanceDenseIndexRepair rebuilds the dense indexes an unsealed cold + // window left missing (dense_index_repair.go). + maintenanceDenseIndexRepair maintenanceJob = "dense_index_repair" ) // preemptible reports whether a job in flight may be asked to yield the lane. @@ -640,8 +643,16 @@ func (s *Store) runMaintenanceLane(ctx context.Context, signal <-chan struct{}, return } } + // Ahead of the pass for the same reason as the drain, and because a + // completed repair owes the pass that analyzes what it rebuilt. + if s.takeDenseIndexRepair() { + s.runDenseIndexRepair(ctx) + if ctx.Err() != nil { + return + } + } if !s.statsPassOwed() { - // The wakeup carried a drain and nothing else. + // The wakeup carried a drain or a repair and nothing else. continue } if !s.claimMaintenancePass() { @@ -731,6 +742,8 @@ func (s *Store) stopMaintenanceLane() { s.maintenanceClosed = true s.maintenanceOwed = false s.maintenanceDrainOwed = false + s.denseRepairOwed = false + s.stopDenseIndexRepairTimerLocked() cancel := s.maintenanceCancel done := s.maintenanceDone s.maintenanceSched.Unlock() diff --git a/internal/graph/store_sqlite/store_purge.go b/internal/graph/store_sqlite/store_purge.go index e3805ef9a..72ff63704 100644 --- a/internal/graph/store_sqlite/store_purge.go +++ b/internal/graph/store_sqlite/store_purge.go @@ -141,6 +141,9 @@ func (s *Store) PurgeRepo(prefix string) error { if err := tx.Commit(); err != nil { return err } + s.firstIndexDrainsMu.Lock() + delete(s.firstIndexDrains, prefix) + s.firstIndexDrainsMu.Unlock() s.finishAnalysisMutationLocked(len(ids) > 0) if changed { s.payloadInputAdminRevision.Add(1) @@ -351,6 +354,14 @@ func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error { if err := tx.Commit(); err != nil { return err } + s.firstIndexDrainsMu.Lock() + if observation, ok := s.firstIndexDrains[oldPrefix]; ok { + if current, exists := s.firstIndexDrains[newPrefix]; !exists || current.CompletedAt.Before(observation.CompletedAt) { + s.firstIndexDrains[newPrefix] = observation + } + delete(s.firstIndexDrains, oldPrefix) + } + s.firstIndexDrainsMu.Unlock() if changed { s.payloadInputAdminRevision.Add(1) s.markMutationReceiptsIncompleteLocked() diff --git a/internal/hooks/bash_nested.go b/internal/hooks/bash_nested.go new file mode 100644 index 000000000..598d28e47 --- /dev/null +++ b/internal/hooks/bash_nested.go @@ -0,0 +1,136 @@ +package hooks + +import ( + "path/filepath" + "strings" + "time" +) + +// bash_nested.go answers the commands a command runs as text: the string of +// `bash -c '…'`, and the literal command lines an inline Python script hands +// to os.system, os.popen or subprocess. Each gets the whole Bash policy, as +// if the agent had typed it, one level deep: a command nested in a nested +// command has no opinion. + +// maxNestedCommands bounds the nested commands one command is examined for, +// and maxNestedCommand the size of each; past them the hook stops looking. +const ( + maxNestedCommands = 8 + maxNestedCommand = 64 << 10 +) + +// isNestingShell names the shells whose `-c` string is a command of its own. +func isNestingShell(name string) bool { + switch name { + case "bash", "sh", "dash", "zsh": + return true + } + return false +} + +// shellInline returns the command string of a shell run with -c: `bash -c +// CMD`, `bash -lc CMD`, `sh -ec CMD`, `bash -o pipefail -c CMD`. The string +// is the first operand after the options; without -c the operand is a +// script file, which is not inline. +func shellInline(words []string) (string, bool) { + hasC := false + for i := 1; i < len(words); i++ { + w := words[i] + switch { + case w == "--" || w == "-": + if hasC && i+1 < len(words) { + return words[i+1], true + } + return "", false + case w == "--rcfile" || w == "--init-file": + i++ + case strings.HasPrefix(w, "--"): + case len(w) > 1 && (w[0] == '-' || w[0] == '+'): + for _, f := range w[1:] { + switch f { + case 'c': + hasC = hasC || w[0] == '-' + case 'o', 'O': + i++ // -o pipefail, +O extglob + } + } + default: + return w, hasC + } + } + return "", false +} + +// shellQuoteWord quotes one word for a POSIX shell, leaving a word that +// needs no quoting bare, so shellWords reads it back exactly. +func shellQuoteWord(w string) string { + if w != "" && strings.Trim(w, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:,+@%-") == "" { + return w + } + return shellSingleQuote(w) +} + +// nestedCommand is one command another command runs, and the directory it +// runs in. +type nestedCommand struct { + cmd, cwd string +} + +// nestedCommands lists the commands cmd runs as text, in order: each shell +// -c string, then each literal command line of its inline Python. A command +// whose working directory the hook cannot tell — after a cd it cannot +// follow, under a launcher that moves it, or from a script that may chdir — +// is left out rather than read with absolute paths only: it fails open. +func nestedCommands(shells []inlineScript, scripts []analyzedScript, dirs commandDirs, cwd string) []nestedCommand { + var out []nestedCommand + add := func(cmd, dir string) { + switch { + case filepath.IsAbs(dir): + case dir == "": + dir = cwd + case cwd == "": + return + default: + dir = filepath.Join(cwd, dir) + } + out = append(out, nestedCommand{cmd, dir}) + } + for _, shell := range shells { + if dir, ok := dirs.at(shell); ok { + add(shell.code, dir) + } + } + for _, script := range scripts { + if dir, ok := dirs.pythonDir(script.inlineScript); ok { + for _, line := range script.facts.commands { + add(line, dir) + } + } + } + return out +} + +// enrichNestedCommands runs the Bash policy over the nested commands of a +// command whose own answer is outer, which is not a deny. A nested deny wins; +// otherwise outer stands, or the first nested answer when outer is empty. +// The commands share the outer command's deadline, and those past the bound +// or the deadline are not examined. +func enrichNestedCommands(outer enrichResult, nested []nestedCommand, probe fileScopeProbe, depth int, deadline time.Time) enrichResult { + answer := outer + for i, n := range nested { + if i >= maxNestedCommands || !time.Now().Before(deadline) { + break + } + if len(n.cmd) > maxNestedCommand { + continue + } + r := enrichBashCommand(n.cmd, n.cwd, probe, depth+1, deadline) + if r.deny { + return r + } + if answer == (enrichResult{}) { + answer = r + } + } + return answer +} diff --git a/internal/hooks/bash_script_read.go b/internal/hooks/bash_script_read.go index 05a9b7ead..953511067 100644 --- a/internal/hooks/bash_script_read.go +++ b/internal/hooks/bash_script_read.go @@ -31,12 +31,14 @@ const ( inlineScriptTime = 500 * time.Millisecond ) -// pythonScript is one inline Python program and where it starts in the -// command, so a `cd` before it can be told from one after it. -type pythonScript struct { +// inlineScript is one inline program — a Python script, or the command +// string of `bash -c` — and where it starts in the command, so a `cd` before +// it can be told from one after it. +type inlineScript struct { code string pos int cwdUnknown bool // a launcher ran it elsewhere (env -C, uv run --directory) + wrapped bool // found only by looking through skipLaunchWrappers' wrappers } // pythonBinaryRE matches python, python3, python3.12, pypy3 and the like. @@ -64,10 +66,21 @@ var interpreterLaunchers = map[string]struct { // Python. cwdChanged reports a launcher option that runs it in another // directory. func pythonCommand(words []string) (python []string, cwdChanged bool) { - words = skipCommandPrefix(words) - for len(words) > 0 { + return launchedCommand(words, pythonBinaryRE.MatchString, false) +} + +// launchedCommand returns the words from the first binary `is` accepts on, +// looking past the same prefixes and launchers as pythonCommand, and with +// wrappers also past skipLaunchWrappers' wrappers before every command. +func launchedCommand(words []string, is func(name string) bool, wrappers bool) (command []string, cwdChanged bool) { + for words = skipCommandPrefix(words); len(words) > 0; { + if wrappers { + if words = skipLaunchWrappers(words); len(words) == 0 { + break + } + } name := baseCommandName(words[0]) - if pythonBinaryRE.MatchString(name) { + if is(name) { return words, cwdChanged } launcher, ok := interpreterLaunchers[name] @@ -105,6 +118,80 @@ func pythonCommand(words []string) (python []string, cwdChanged bool) { return nil, false } +// findPython is pythonCommand, then, with extra and when that finds no +// Python, the same search through skipLaunchWrappers' wrappers; wrapped +// reports the second. +func findPython(words []string, extra bool) (python []string, cwdChanged, wrapped bool) { + if python, cwdChanged = pythonCommand(words); python != nil || !extra { + return python, cwdChanged, false + } + python, cwdChanged = launchedCommand(words, pythonBinaryRE.MatchString, true) + return python, cwdChanged, python != nil +} + +// skipLaunchWrappers is skipCommandPrefix that also looks through the +// wrappers that run the command after them in the same filesystem and +// directory — `exec`, `nohup`, `setsid`, `nice`, `command`, `nvm exec +// `, `direnv exec ` — when finding an inline interpreter or a +// nested shell. The shell read and write shapes do not use it: a wrapped +// first statement would stop their classification short of a later one. +// Wrappers that run the command against another root or host (chroot, +// docker, ssh) are not looked through, and neither is `command -v`, which +// runs nothing. +func skipLaunchWrappers(tokens []string) []string { + for tokens = skipCommandPrefix(tokens); len(tokens) > 0; tokens = skipCommandPrefix(tokens) { + var opts []string + switch t := tokens[0]; { + case t == "nohup": + tokens = tokens[1:] + case t == "exec": + tokens, _ = skipWrapperOptions(tokens[1:], "a") + case t == "setsid": + tokens, _ = skipWrapperOptions(tokens[1:], "") + case t == "nice": + tokens, _ = skipWrapperOptions(tokens[1:], "n") + case t == "command": + rest := tokens + if tokens, opts = skipWrapperOptions(tokens[1:], ""); strings.ContainsAny(strings.Join(opts, ""), "vV") { + return rest + } + case t == "nvm" && len(tokens) > 1 && tokens[1] == "exec": + if tokens, _ = skipWrapperOptions(tokens[2:], ""); len(tokens) > 0 && nvmVersionRE.MatchString(tokens[0]) { + tokens = tokens[1:] + } + case t == "direnv" && len(tokens) > 2 && tokens[1] == "exec": + tokens = tokens[3:] + default: + return tokens + } + } + return tokens +} + +// nvmVersionRE matches the version `nvm exec` takes before the command: 20, +// v20.1.0, node, lts/iron. The version is optional, so a word that is not +// one is the command. +var nvmVersionRE = regexp.MustCompile(`^(?:v?[0-9][0-9.]*|node|iojs|stable|lts/\S+)$`) + +// skipWrapperOptions drops the options a wrapper takes before its command +// and returns them. A short option in valued takes the next token as its +// value unless the value is attached; `--` ends the options. +func skipWrapperOptions(tokens []string, valued string) (rest, opts []string) { + for len(tokens) > 0 && strings.HasPrefix(tokens[0], "-") && tokens[0] != "-" { + opt := tokens[0] + tokens = tokens[1:] + if opt == "--" { + break + } + opts = append(opts, opt) + if last := opt[len(opt)-1:]; !strings.HasPrefix(opt, "--") && strings.Contains(valued, last) && len(tokens) > 0 { + opts = append(opts, tokens[0]) + tokens = tokens[1:] + } + } + return tokens, opts +} + // pythonInline reads a Python command line: the code of `-c CODE`, or // fromStdin when the program comes from standard input (`python3 -` or no // script at all). `-m module` and a script file are not inline code. @@ -207,15 +294,24 @@ func shellWords(s string) []string { } // pythonScriptsAt returns every inline Python program in cmd with its -// position: each `-c` statement, then each heredoc body fed to a Python -// reading standard input. Positions come from cursors that only move +// position. +func pythonScriptsAt(cmd string) []inlineScript { + scripts, _ := inlineScriptsAt(cmd, false) + return scripts +} + +// inlineScriptsAt returns every inline Python program in cmd with its +// position — each `-c` statement, then each heredoc body fed to a Python +// reading standard input — and the command string of each `bash -c` (or +// sh, dash, zsh) statement. Positions come from cursors that only move // forward, so a long command costs one pass and a repeated statement gets -// its own position. -func pythonScriptsAt(cmd string) []pythonScript { - if !strings.Contains(cmd, "python") && !strings.Contains(cmd, "pypy") { - return nil +// its own position. Without extra it finds only what pythonCommand finds, +// and no shells: no Python found only through a wrapper. +func inlineScriptsAt(cmd string, extra bool) (scripts, shells []inlineScript) { + hasPython := strings.Contains(cmd, "python") || strings.Contains(cmd, "pypy") + if !hasPython && (!extra || !strings.Contains(cmd, "sh")) { + return nil, nil } - var scripts []pythonScript // A text not found verbatim (a segment the splitter rewrote, such as // a backslash-newline) loses the cursor: what follows has no position, // which only gives up on relative paths, and the scan stays one pass. @@ -236,12 +332,20 @@ func pythonScriptsAt(cmd string) []pythonScript { for _, segment := range allCommandSegments(cmd) { pos := at(&segCursor, segment) command, _ := splitSegmentRedirects(segment) - if words, moved := pythonCommand(shellWords(command)); words != nil { - if code, fromStdin, ok := pythonInline(words); ok && !fromStdin && code != "" { - scripts = append(scripts, pythonScript{code: code, pos: pos, cwdUnknown: moved}) + words := shellWords(command) + if python, moved, wrapped := findPython(words, extra); python != nil { + if code, fromStdin, ok := pythonInline(python); ok && !fromStdin && code != "" { + scripts = append(scripts, inlineScript{code: code, pos: pos, cwdUnknown: moved, wrapped: wrapped}) + } + } else if shell, moved := launchedCommand(words, isNestingShell, true); extra && shell != nil { + if code, ok := shellInline(shell); ok && code != "" { + shells = append(shells, inlineScript{code: code, pos: pos, cwdUnknown: moved}) } } } + if !hasPython { + return nil, shells + } _, heredocs := splitHeredocs(cmd) docCursor := 0 for _, heredoc := range heredocs { @@ -265,13 +369,13 @@ func pythonScriptsAt(cmd string) []pythonScript { continue } command, _ := splitSegmentRedirects(segments[len(segments)-1]) - if words, moved := pythonCommand(shellWords(command)); words != nil { + if words, moved, wrapped := findPython(shellWords(command), extra); words != nil { if _, fromStdin, ok := pythonInline(words); ok && fromStdin { - scripts = append(scripts, pythonScript{code: heredoc.body, pos: pos, cwdUnknown: moved}) + scripts = append(scripts, inlineScript{code: heredoc.body, pos: pos, cwdUnknown: moved, wrapped: wrapped}) } } } - return scripts + return scripts, shells } // segmentChangesDir reports whether a segment runs cd or pushd anywhere in @@ -313,60 +417,146 @@ func cdPositions(cmd string, leading int) []int { return positions } +// analyzedScript is an inline Python script and what its analysis found. +type analyzedScript struct { + inlineScript + facts pyScriptFacts +} + +// analyzeInlineScripts analyses up to maxInlineScripts scripts by deadline, +// dropping each one the analysis has no opinion on; extra is the +// analysis mode (see analyzePythonScript). The scripts found only through a +// wrapper come last and are capped on their own. +func analyzeInlineScripts(scripts []inlineScript, deadline time.Time, extra bool) []analyzedScript { + var out []analyzedScript + for _, wrapped := range []bool{false, true} { + n := 0 + for _, script := range scripts { + if script.wrapped != wrapped || n >= maxInlineScripts { + continue + } + n++ + if facts, ok := analyzePythonScript(script.code, deadline, extra); ok { + out = append(out, analyzedScript{script, facts}) + } + } + } + return out +} + +// commandDirs is where the statements of one command run: the directory its +// leading `cd` chain ends in, and where each cd the chain cannot follow +// starts. +type commandDirs struct { + dir string + cds []int +} + +func newCommandDirs(cmd string) commandDirs { + dir, segments := leadingCdChain(cmd) + return commandDirs{dir: dir, cds: cdPositions(cmd, segments)} +} + +// at returns the directory a script runs in — relative to the hook's cwd +// unless absolute, "" for the cwd itself — or false when a cd the chain +// cannot follow comes before it, or a launcher ran it elsewhere. +func (d commandDirs) at(script inlineScript) (string, bool) { + for _, at := range d.cds { + if at < 0 || script.pos < 0 || at < script.pos { + return "", false + } + } + if script.cwdUnknown { + return "", false + } + return d.dir, true +} + +// pythonDir is at for a Python script, which may also change its own +// working directory (os.chdir, fchdir, chdir imported or aliased from os): +// then nothing relative in it resolves against the hook's cwd. +func (d commandDirs) pythonDir(script inlineScript) (string, bool) { + if strings.Contains(script.code, "chdir") { + return "", false + } + return d.at(script) +} + +// resolve returns the path a script's literal path p names, relative to +// the hook's cwd unless absolute, or false when it cannot be told. +func (d commandDirs) resolve(script inlineScript, p string) (string, bool) { + if filepath.IsAbs(p) { + return p, true + } + dir, ok := d.pythonDir(script) + if !ok { + return "", false + } + if dir != "" { + return filepath.Join(dir, p), true + } + return p, true +} + +// scriptWrites returns the files the scripts open for writing through +// os.open, which no shell write pattern sees, as `script write` shapes: such +// a write gets the answer open(p, 'w') gets. +func scriptWrites(scripts []analyzedScript, dirs commandDirs) []BashWrite { + var writes []BashWrite + for _, script := range scripts { + for _, p := range script.facts.writes { + if at, ok := dirs.resolve(script.inlineScript, p); ok && len(writes) < bashWriteProbeLimit { + writes = append(writes, BashWrite{Path: at, Shape: "script write"}) + } + } + } + return writes +} + // enrichBashScriptRead answers inline Python that reads indexed source. // answered is false when the command runs no such script, so the caller // falls through to its own verdict. -func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe) (enrichResult, bool) { +// The deadline bounds the analysis; a zero one is inlineScriptTime from now. +// What only the extra analysis sees is answered by enrichBashCommand after +// this, with its own budget. +func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe, deadline time.Time) (enrichResult, bool) { started := time.Now() - deadline := started.Add(inlineScriptTime) + if deadline.IsZero() { + deadline = started.Add(inlineScriptTime) + } scripts := pythonScriptsAt(cmd) if len(scripts) == 0 { return enrichResult{}, false } - if len(scripts) > maxInlineScripts { - scripts = scripts[:maxInlineScripts] - } - cdDir, cdSegments := leadingCdChain(cmd) - cds := cdPositions(cmd, cdSegments) - resolve := func(script pythonScript, p string) (string, bool) { - if filepath.IsAbs(p) { - return p, true - } - // A cd the chain cannot follow, made before the script runs, or a - // chdir inside it: the relative path is no longer the hook's. - for _, at := range cds { - if at < 0 || script.pos < 0 || at < script.pos { - return "", false - } - } - // A possible working-directory change inside the script (os.chdir, - // fchdir, chdir imported or aliased from os): a relative path no - // longer resolves against the hook's cwd. - if script.cwdUnknown || strings.Contains(script.code, "chdir") { - return "", false - } - if cdDir != "" { - return filepath.Join(cdDir, p), true - } - return p, true - } + dirs := newCommandDirs(cmd) + return scriptReadAnswer(analyzeInlineScripts(scripts, deadline, false), false, dirs, cwd, scopeProbe, started) +} +// scriptReadAnswer answers the reads and edits of the scripts: with second +// false, the open()/pathlib reads and edits of the scripts found without a +// wrapper; with second true, os.open reads and the reads and edits of the +// scripts found only through a wrapper. Each call has its own probe budget. +func scriptReadAnswer(scripts []analyzedScript, second bool, dirs commandDirs, cwd string, scopeProbe fileScopeProbe, started time.Time) (enrichResult, bool) { type candidate struct { - script pythonScript + script inlineScript path string guessed bool // the write target is unnamed: it may be another file } var edits, reads []candidate for _, script := range scripts { - facts, ok := analyzePythonScript(script.code, deadline) - if !ok { - continue + facts := script.facts + if script.wrapped == second { + for _, p := range facts.reads { + if facts.writeBack[p] { + edits = append(edits, candidate{script.inlineScript, p, facts.guessed[p]}) + } else { + reads = append(reads, candidate{script.inlineScript, p, false}) + } + } } - for _, p := range facts.reads { - if facts.writeBack[p] { - edits = append(edits, candidate{script, p, facts.guessed[p]}) - } else { - reads = append(reads, candidate{script, p, false}) + if second { + for _, p := range facts.osReads { + reads = append(reads, candidate{script.inlineScript, p, false}) } } } @@ -379,7 +569,7 @@ func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe) (enrichRes if probes >= bashWriteProbeLimit { return fileIndexStatus{}, false } - at, ok := resolve(c.script, c.path) + at, ok := dirs.resolve(c.script, c.path) if !ok { return fileIndexStatus{}, false } diff --git a/internal/hooks/enrich_bash_script_test.go b/internal/hooks/enrich_bash_script_test.go index 19b8c6bfe..84136c995 100644 --- a/internal/hooks/enrich_bash_script_test.go +++ b/internal/hooks/enrich_bash_script_test.go @@ -29,6 +29,11 @@ func TestEnrichBash_InlinePython(t *testing.T) { silent = "silent" // nothing at all allow = "allow" // no deny (another door may still advise) catDeny = "catDeny" // the shell read's own deny + readA = "readA" // the shell read's deny, for a.go + sedA = "sedA" // the shell-write advisory, for sed -i a.go + scriptW = "scriptW" // the shell-write advisory, for a script write to a.go + + pyRead = `python3 -c "print(open('a.go').read())"` ) for _, tc := range []struct { name, cmd, want string @@ -69,6 +74,7 @@ func TestEnrichBash_InlinePython(t *testing.T) { // Everything else passes. {"pure write", `python3 -c "open('a.go', \"w\").write('package a')"`, allow}, + {"pure write, plain quotes", `python3 -c "open('a.go', 'w').write('package a')"`, scriptW}, {"syntax check", `python -c 'import ast; ast.parse(open("a.go").read()); print("syntax ok")'`, silent}, {"data file", `python3 -c "import json; print(json.load(open('package.json')))"`, silent}, {"unindexed source", `python3 -c "print(open('c.go').read())"`, silent}, @@ -119,6 +125,110 @@ func TestEnrichBash_InlinePython(t *testing.T) { {"env -C", `env -C /tmp python3 -c "print(open('a.go').read())"`, silent}, {"uv run --directory", `uv run --directory=/tmp python -c "print(open('a.go').read())"`, silent}, {"env -C attached", `env -iC/tmp python3 -c "print(open('a.go').read())"`, silent}, + + // Wrappers that run the command in the same filesystem and directory + // are looked through; those that run it against another root, and + // `command -v`, which runs nothing, are not. + {"exec", "exec " + pyRead, deny}, + {"nohup", "nohup " + pyRead, deny}, + {"setsid", "setsid -f " + pyRead, deny}, + {"nice -n", "nice -n 5 " + pyRead, deny}, + {"nice -N", "nice -5 " + pyRead, deny}, + {"nice --adjustment", "nice --adjustment=3 " + pyRead, deny}, + {"command", "command " + pyRead, deny}, + {"nvm exec", "nvm exec 20 " + pyRead, deny}, + {"nvm exec without a version", "nvm exec " + pyRead, deny}, + {"direnv exec", "direnv exec . " + pyRead, deny}, + {"wrapped launcher", "timeout 5 nohup " + pyRead, deny}, + {"command -v", "command -v " + pyRead, silent}, + // The shell read and write shapes do not look through them: a wrapped + // first statement must not stop the classification before a later one. + // What the hook newly sees (os.open reads, wrapped scripts) has its + // own probe budget: it never spends the one a later read needs. + {"os.open reads before a read", `python3 -c "import os; os.open('x0.py', os.O_RDONLY); os.open('x1.py', os.O_RDONLY); os.open('x2.py', os.O_RDONLY); print(open('a.go').read())"`, deny}, + {"os.open creates before a read", `python3 -c "import os; os.open('/tmp/x0.py', os.O_CREAT); os.open('/tmp/x1.py', os.O_CREAT); os.open('/tmp/x2.py', os.O_CREAT); print(open('a.go').read())"`, deny}, + {"os.open reads, then a read", `python3 -c "import os; os.open('x0.py', os.O_RDONLY); os.open('x1.py', os.O_RDONLY); os.open('x2.py', os.O_RDONLY)" && ` + pyRead, deny}, + {"wrapped reads before a read", `nohup python3 -c "print(open('x0.py').read())"; nohup python3 -c "print(open('x1.py').read())"; nohup python3 -c "print(open('x2.py').read())"; ` + pyRead, deny}, + {"wrapped script reading three files before a read", `nice -n 5 python3 -c "print(open('x0.py').read(), open('x1.py').read(), open('x2.py').read())"; ` + pyRead, deny}, + {"wrapped rewrites before a rewrite", `exec python3 -c "p='x0.py'; s=open(p).read(); open(p,'w').write(s)"; exec python3 -c "p='x1.py'; s=open(p).read(); open(p,'w').write(s)"; exec python3 -c "p='x2.py'; s=open(p).read(); open(p,'w').write(s)"; python3 -c "p='a.go'; s=open(p).read(); open(p,'w').write(s)"`, edit}, + {"wrapped command, then a cat", `nohup ls && cat a.go`, readA}, + {"command, then a tail", `command ls; tail a.go`, readA}, + {"wrapped cat of other source, then a cat", `nice -n 5 cat c.go && cat a.go`, readA}, + {"wrapped cat", `nohup cat a.go`, silent}, + {"chroot", "chroot /x " + pyRead, silent}, + + // os.open reads and writes by its literal flags. + {"os.open read", `python3 -c "import os; fd = os.open('a.go', os.O_RDONLY); print(os.read(fd, 99))"`, deny}, + {"aliased os.open", `python3 -c "from os import open as o, read, O_RDONLY; print(read(o('a.go', O_RDONLY), 99))"`, silent}, + {"os.open bare flag", `python3 -c "import os; from os import O_RDONLY, O_CLOEXEC; print(os.read(os.open('a.go', O_RDONLY | O_CLOEXEC), 99))"`, deny}, + // An os.open write is a write target only: never a read, never what + // turns a read into an edit, so it hides no Read deny. + {"os.open write and a read of the same file", `python3 -c "import os; print(open('a.go').read()); os.open('a.go', os.O_WRONLY)"`, deny}, + {"os.open read-write and a read of another file", `python3 -c "import os; os.open('a.go', os.O_RDWR); print(open('b.go').read())"`, deny}, + {"Python read, then an os.open read-write", `python3 -c "print(open('b.go').read())" && python3 -c "import os; os.open('a.go', os.O_RDWR)"`, deny}, + {"os.open O_PATH", `python3 -c "import os; os.open('a.go', os.O_RDONLY | os.O_PATH)"`, silent}, + {"os.open O_DIRECTORY", `python3 -c "import os; os.open('a.go', os.O_DIRECTORY)"`, silent}, + {"os.open read-modify-write", "python3 - <<'PY'\nimport os\nfd = os.open('a.go', os.O_RDWR)\ns = os.read(fd, 1 << 20)\nos.lseek(fd, 0, 0)\nos.write(fd, s)\nPY", scriptW}, + {"os.open write", `python3 -c "import os; os.write(os.open('a.go', os.O_WRONLY | os.O_TRUNC), b'package a')"`, scriptW}, + {"os.open variable flags", `python3 -c "import os, sys; f = int(sys.argv[1]); print(os.read(os.open('a.go', f), 99))"`, silent}, + {"os.open flag not from os", `python3 -c "import os; print(os.read(os.open('a.go', O_RDONLY), 99))"`, silent}, + {"os.open create", `python3 -c "import os; print(os.read(os.open('a.go', os.O_RDONLY | os.O_CREAT), 99))"`, deny}, + {"os.open dir_fd", `python3 -c "import os; d = os.open('src', os.O_RDONLY); print(os.read(os.open('a.go', os.O_RDONLY, dir_fd=d), 99))"`, silent}, + {"os aliased onto another module", `python3 -c "import mymod as os; print(os.read(os.open('a.go', os.O_RDONLY), 99))"`, silent}, + + // An os.open write never softens a deny the rest of the command earns. + {"os.open write, then a cat", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && cat b.go`, catDeny}, + {"os.open write after a cat", `cat b.go && python3 -c "import os; os.open('a.go', os.O_WRONLY)"`, catDeny}, + {"os.open write, then a head", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && head b.go`, deny}, + {"os.open write and a read", `python3 -c "import os; os.open('a.go', os.O_WRONLY); print(open('b.go').read())"`, deny}, + {"os.open write, then a Python read", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && python3 -c "print(open('b.go').read())"`, deny}, + {"open write, then a cat", `python3 -c "open('a.go', 'w').write('x')" && cat b.go`, scriptW}, + {"os.open write, unindexed", `python3 -c "import os; os.write(os.open('c.go', os.O_WRONLY), b'x')"`, silent}, + + // A command run as text gets the answer it would get typed: the + // -c string of a shell, and a literal command line Python runs. + {"bash -c", `bash -c "cat a.go"`, readA}, + {"bash -lc", `bash -lc 'head -5 a.go'`, readA}, + {"sh with options", `sh -o pipefail -ec 'cat a.go'`, readA}, + {"wrapped shell", `nohup bash -c 'cat a.go'`, readA}, + {"python in a shell", `sh -c 'python3 -c "print(open(\"a.go\").read())"'`, deny}, + {"shell write in a shell", `bash -c "sed -i s/x/y/ a.go"`, sedA}, + {"nested deny beats an outer advisory", `sed -i s/x/y/ a.go && bash -c 'cat b.go'`, catDeny}, + {"os.popen", `python3 -c "import os; print(os.popen('cat a.go').read())"`, readA}, + {"os.system", `python3 -c "import os; os.system('cat a.go')"`, readA}, + {"subprocess list", `python3 -c "import subprocess; subprocess.run(['cat', 'a.go'])"`, readA}, + {"check_output tuple", `python3 -c "import subprocess; print(subprocess.check_output(('tail', '-n', '5', 'a.go')))"`, readA}, + {"getoutput", `python3 -c "import subprocess; print(subprocess.getoutput('head a.go'))"`, readA}, + {"subprocess shell write", `python3 -c "import subprocess; subprocess.run('sed -i s/x/y/ a.go', shell=True)"`, sedA}, + {"subprocess python", `python3 -c "import subprocess; subprocess.run(['python3', '-c', 'print(open(\'a.go\').read())'])"`, silent}, + {"subprocess python, quoted", "python3 - <<'PY'\nimport subprocess\nsubprocess.run(['python3', '-c', \"print(open('a.go').read())\"])\nPY", deny}, + + {"module imported with a submodule", `python3 -c "import os, os.path; os.system('cat a.go')"`, readA}, + {"subprocess string with shell=True", `python3 -c "import subprocess; print(subprocess.check_output('cat a.go', shell=True))"`, readA}, + {"subprocess string without a shell", `python3 -c "import subprocess; subprocess.run('cat a.go')"`, silent}, + {"os.system list", `python3 -c "import os; os.system(['cat', 'a.go'])"`, silent}, + {"os aliased onto another module, system", `python3 -c "import mylib as os; os.system('cat a.go')"`, silent}, + {"subprocess rebound", "python3 - <<'PY'\nimport subprocess\nsubprocess = None\nsubprocess.run(['cat', 'a.go'])\nPY", silent}, + {"local class os", "python3 - <<'PY'\nclass os:\n def system(c): pass\nos.system('cat a.go')\nPY", silent}, + {"os never imported", `python3 -c "os.system('cat a.go')"`, silent}, + {"nested script file", `bash run.sh a.go`, silent}, + {"nested unindexed", `bash -c "cat c.go" && python3 -c "import os; os.system('cat c.go')"`, silent}, + {"two levels", `bash -c "bash -c 'cat a.go'"`, silent}, + {"two levels through python", `bash -c 'python3 -c "import os; os.system(\"cat a.go\")"'`, silent}, + {"shell list from python", `python3 -c "import subprocess; subprocess.run(['bash', '-c', 'cat a.go'])"`, silent}, + {"subprocess cwd", `python3 -c "import subprocess; subprocess.run(['cat', 'a.go'], cwd='/tmp')"`, silent}, + {"subprocess executable", `python3 -c "import subprocess; subprocess.run('cat a.go', shell=True, executable='/bin/zsh')"`, silent}, + {"subprocess *args", `python3 -c "import subprocess, sys; subprocess.run(['cat', 'a.go'], *sys.argv[1:])"`, silent}, + {"subprocess **kwargs", `python3 -c "import subprocess; kw = {'cwd': '/tmp'}; subprocess.run(['cat', 'a.go'], **kw)"`, silent}, + {"subprocess variable", `python3 -c "import subprocess; c = ['cat', 'a.go']; subprocess.run(c)"`, silent}, + {"os.system f-string", `python3 -c "import os; f = 'a.go'; os.system(f'cat {f}')"`, silent}, + {"list handed to the shell", `python3 -c "import subprocess; subprocess.run(['cat a.go'], shell=True)"`, silent}, + {"nested after cd to a variable", `cd $V && bash -c "cat a.go"`, silent}, + {"python nested after cd to a variable", `cd $V && python3 -c "import os; os.system('cat a.go')"`, silent}, + {"nested after chdir", `python3 -c "import os; os.chdir('/tmp'); os.system('cat a.go')"`, silent}, + {"nested under env -C", `env -C /tmp bash -c "cat a.go"`, silent}, + {"within the nested bound", strings.Repeat(`bash -c "cat c.go"; `, 7) + `bash -c "cat a.go"`, readA}, + {"past the nested bound", strings.Repeat(`bash -c "cat c.go"; `, 8) + `bash -c "cat a.go"`, silent}, } { t.Run(tc.name, func(t *testing.T) { redirectTelemetry(t) @@ -138,6 +248,15 @@ func TestEnrichBash_InlinePython(t *testing.T) { if !r.deny || !strings.Contains(r.reason, "Bash `cat b.go` reads indexed source") { t.Errorf("want the cat deny, got %+v", r) } + case readA: + if !r.deny || !strings.Contains(r.reason, " a.go` reads indexed source") { + t.Errorf("want the shell read deny, got %+v", r) + } + case sedA, scriptW: + shape := map[string]string{sedA: "sed -i", scriptW: "script write"}[tc.want] + if r.deny || !strings.Contains(r.context, "Bash `"+shape+"` writes a.go") { + t.Errorf("want the %s advisory, got %+v", shape, r) + } case maybe: if r.deny || !strings.Contains(r.context, "Python script may rewrite") { t.Errorf("want the hedged edit advisory, got %+v", r) @@ -167,6 +286,51 @@ func TestEnrichBash_InlinePythonEditBlocking(t *testing.T) { } } +// A write-only os.open gets the shell-write deny under edit blocking, as +// open(p, 'w') does. +func TestEnrichBash_InlinePythonOSOpenEditBlocking(t *testing.T) { + redirectTelemetry(t) + stubIndexedPaths(t, "a.go") + t.Setenv("GORTEX_HOOK_BLOCK_EDIT", "1") + for _, cmd := range []string{ + `python3 -c "open('a.go', 'w').write('x')"`, + `python3 -c "import os; os.open('a.go', os.O_WRONLY)"`, + } { + r := enrichBash(map[string]any{"command": cmd}, "/repo") + if !r.deny || !strings.Contains(r.reason, "Bash `script write` writes a.go") { + t.Errorf("%s: want the shell-write deny, got %+v", cmd, r) + } + } +} + +// What only the extra analysis looks for (os.open, command lines) costs +// the read analysis nothing: a script it covers just under its step budget +// keeps the Read deny however many os.open calls it carries. +func TestEnrichBash_InlinePythonExtraAnalysisBudget(t *testing.T) { + redirectTelemetry(t) + stubIndexedPaths(t, "a.go") + t.Setenv("GORTEX_HOOK_BLOCK_EDIT", "") + lines := []string{"import os", "print(open('a.go').read())"} + for i := 0; i < 90; i++ { + lines = append(lines, "os.open("+strings.Repeat("'a'+", 199)+"'a', os.O_RDONLY)") + } + code := strings.Join(lines, "\n") + if _, ok := analyzePythonScript(code, time.Now().Add(time.Minute), true); ok { + t.Fatal("the extra analysis of this script should exceed its own step budget") + } + facts, ok := analyzePythonScript(code, time.Now().Add(time.Minute), false) + if !ok || len(facts.reads) != 1 || facts.reads[0] != "a.go" { + t.Fatalf("the read analysis must cover the script, got %v %+v", ok, facts) + } + // The whole policy, under a deadline that a slow (-race) build cannot + // miss. + cmd := "python3 - <<'EOF'\n" + code + "\nEOF" + r := enrichBashCommand(cmd, "/repo", queryFileIndexScope, 0, time.Now().Add(time.Minute)) + if !r.deny || !strings.Contains(r.reason, "reads indexed source") { + t.Errorf("want the Read deny, got %+v", r) + } +} + // A redirected Python patch script is counted with the other shell writes. func TestEnrichBash_InlinePythonEditTelemetry(t *testing.T) { log := redirectTelemetry(t) diff --git a/internal/hooks/posttask.go b/internal/hooks/posttask.go index 268597984..b9827450b 100644 --- a/internal/hooks/posttask.go +++ b/internal/hooks/posttask.go @@ -136,25 +136,106 @@ type changeSet struct { Scope string `json:"scope"` Repo string `json:"repo"` RepoRoot string `json:"repo_root"` + // The label of an answer whose relations lag: graph_freshness on a + // primary checkout, graph_status on a worktree. + GraphFreshness string `json:"graph_freshness"` + GraphStatus string `json:"graph_status"` + PendingFileCount int `json:"pending_file_count"` } +// detectChangesBudget bounds the detect_changes round trip of a briefing. A +// daemon still publishing edits answers inside it with a labelled partial; a +// stalled one is reported rather than waited on. +const detectChangesBudget = 300 * time.Millisecond + +// detectChangesWait is how far into the budget the briefing's wait_deadline +// lets the daemon wait for pending edits before it labels them. The rest of +// the budget is for the diff walk and the transport. +const detectChangesWait = detectChangesBudget / 2 + // detectChangeSet asks the bridge for the current diff and decodes it. -// Returns nil when the bridge is unreachable or the payload is unusable. -func detectChangeSet(port int) *changeSet { - raw := callServerTool(port, "detect_changes", map[string]any{ +// answered is false when the bridge did not reply within detectChangesBudget, +// or replied with nothing once detectChangesWait had passed; changes is nil +// when it is unreachable or the payload is unusable. +func detectChangeSet(port int) (changes *changeSet, answered bool) { + started := time.Now() + raw, answered := callServerToolWithin(port, "detect_changes", map[string]any{ "scope": postTaskDiffScope, // The briefings never read by_depth; summary_only drops the // largest part of the response. - "summary_only": true, - }) + "summary_only": true, + "wait_deadline": started.Add(detectChangesWait).UTC().Format(time.RFC3339Nano), + }, detectChangesBudget) if raw == "" { - return nil + // The daemon refuses a wait_deadline that passed before it took + // the call up: a busy daemon, not an unreachable one. + return nil, answered && time.Since(started) < detectChangesWait } - var changes changeSet if err := json.Unmarshal([]byte(raw), &changes); err != nil { - return nil + return nil, true } - return &changes + return changes, true +} + +// relationsLine is the label line of a change set whose relations lag, or "" +// when the graph has published every change it reports. +func (c *changeSet) relationsLine() string { + state := c.GraphFreshness + if state == "" && c.GraphStatus != "" && c.GraphStatus != "ready" { + state = c.GraphStatus + } + if state == "" { + return "" + } + files := c.PendingFileCount + if files == 0 { + files = len(c.ChangedFiles) + } + return fmt.Sprintf("relations: %s (%d files)", state, files) +} + +// laggingChangesBriefing is the briefing for a change set the graph has not +// caught up with: the relations line and the changed files, with the +// symbol-level diagnostics left to a turn that can trust them. A stalled +// detect_changes (nil changes) is reported the same way instead of as +// silence, except to a session that recorded no writes: it has no change set +// to warn about. lagging is false when the change set is complete. +func laggingChangesBriefing(heading string, scope sessionScope, changes *changeSet) (briefing string, lagging bool) { + line := fmt.Sprintf("relations: pending (detect_changes did not answer within %v)", detectChangesBudget) + why := "" + var files []string + if changes == nil { + if scope.SessionID != "" && len(loadSessionState(scope.SessionID).WrittenPaths) == 0 { + return "", true + } + } else { + if line = changes.relationsLine(); line == "" { + return "", false + } + why = " the graph is still publishing edits, so" + files = changes.ChangedFiles + if attr := attributeChanges(scope, changes); attr.Attributed { + if len(attr.OwnedFiles) == 0 { + return "", true + } + files = attr.OwnedFiles + } + } + var sb strings.Builder + sb.WriteString(heading + "\n\n") + sb.WriteString(line + " —" + why + " symbol-level diagnostics are skipped; run detect_changes again before handoff.\n") + if len(files) > 0 { + sb.WriteString("\nChanged files:\n") + const show = 10 + for i, f := range files { + if i == show { + fmt.Fprintf(&sb, "- (+%d more)\n", len(files)-show) + break + } + fmt.Fprintf(&sb, "- `%s`\n", f) + } + } + return sb.String(), true } // briefingScope names the working tree detect_changes actually diffed. @@ -248,10 +329,13 @@ func renderUnattributedFiles(files []string, capped bool) string { // named rather than dropped. When the session cannot be identified at all the // briefing falls back to the whole tree and labels itself as such. func buildPostTaskBriefing(port int, scope sessionScope) string { - changes := detectChangeSet(port) - if changes == nil { + changes, answered := detectChangeSet(port) + if answered && changes == nil { return "" } + if briefing, lagging := laggingChangesBriefing("## Gortex Working-Tree Diagnostics", scope, changes); lagging { + return briefing + } if len(changes.ChangedSymbols) == 0 { // No indexed symbols touched — skip silently. The agent doesn't need @@ -368,8 +452,14 @@ func buildPostTaskBriefing(port int, scope sessionScope) string { // so nothing is ever recorded for it and it always resolves to the labeled // whole-tree fallback — this does not give Codex attribution coverage. func buildMutationBriefing(port int, scope sessionScope) string { - changes := detectChangeSet(port) - if changes == nil || len(changes.ChangedSymbols) == 0 { + changes, answered := detectChangeSet(port) + if answered && changes == nil { + return "" + } + if briefing, lagging := laggingChangesBriefing("## Gortex mutation follow-up", scope, changes); lagging { + return briefing + } + if len(changes.ChangedSymbols) == 0 { return "" } @@ -464,7 +554,9 @@ func renderGuardViolations(port int, idsCSV string) string { // renderDeadCodeHits filters analyze:dead_code results to the intersection // with the currently-changed symbols (i.e. "did this task leave anything -// orphaned"). Emits nothing when the intersection is empty. +// orphaned"). Emits nothing when the intersection is empty. A symbol listed +// on a "pending_unknown " line has unknown liveness while edits are pending +// and is never reported as dead. func renderDeadCodeHits(port int, ids []string) string { raw := callServerTool(port, "analyze", map[string]any{ "kind": "dead_code", @@ -475,14 +567,22 @@ func renderDeadCodeHits(port int, ids []string) string { return "" } + lines := strings.Split(raw, "\n") idSet := make(map[string]bool, len(ids)) for _, id := range ids { idSet[id] = true } + for _, line := range lines { + if rest, unknown := strings.CutPrefix(line, "pending_unknown "); unknown { + for _, field := range strings.Fields(rest) { + delete(idSet, field) + } + } + } var hits []string - for _, line := range strings.Split(raw, "\n") { - if line == "" { + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "pending_unknown ") { continue } // Each compact line starts with the symbol descriptor including the diff --git a/internal/hooks/posttask_test.go b/internal/hooks/posttask_test.go index b55f14b90..2dda76944 100644 --- a/internal/hooks/posttask_test.go +++ b/internal/hooks/posttask_test.go @@ -3,9 +3,12 @@ package hooks import ( "encoding/json" "fmt" + "net/http" + "net/http/httptest" "path/filepath" "strings" "testing" + "time" "github.com/zzet/gortex/internal/daemon" "github.com/zzet/gortex/internal/localizationauth" @@ -189,6 +192,114 @@ func TestRunPostTask_DeadCodeFiltersToChanged(t *testing.T) { if strings.Contains(out, "API Contract Issues") { t.Errorf("contract section should be omitted when clean:\n%s", out) } + + // A changed symbol whose liveness is unknown while edits are pending is + // never reported as dead. + pending := newFakeServer(map[string]string{ + "analyze": "relations: pending (1 files)\npending_unknown function foo.go::Foo foo.go:3\n", + }) + defer pending.Close() + if hits := renderDeadCodeHits(portFromURL(t, pending.URL), []string{"foo.go::Foo"}); hits != "" { + t.Errorf("pending_unknown symbol reported as dead:\n%s", hits) + } +} + +// While the graph is still publishing edits, detect_changes answers with a +// labelled partial. Both briefings must say so and name the changed files +// rather than render diagnostics over lagging relations or fall silent. +func TestPostTaskBriefingPrintsPendingRelations(t *testing.T) { + for name, detect := range map[string]string{ + "primary": `{"changed_files":["a.go","b.go"],"changed_symbols":[{"id":"a.go::A","name":"A","kind":"function"}], + "risk":"LOW","complete":false,"graph_freshness":"pending","pending_file_count":2,"pending_files":["a.go","b.go"]}`, + "worktree": `{"changed_files":["a.go"],"changed_symbols":[],"risk":"UNKNOWN","complete":false,"graph_status":"pending"}`, + } { + srv := newRecordingFakeServer(map[string]string{"detect_changes": detect, "get_test_targets": "a_test.go TestA\n"}) + port := portFromURL(t, srv.URL) + started := time.Now() + for builder, briefing := range map[string]string{ + "stop": buildPostTaskBriefing(port, sessionScope{}), + "mutation": buildMutationBriefing(port, sessionScope{}), + } { + want := "relations: pending (2 files)" + if name == "worktree" { + want = "relations: pending (1 files)" + } + if !strings.Contains(briefing, want) || !strings.Contains(briefing, "`a.go`") { + t.Errorf("%s %s briefing missing %q and the changed files:\n%s", name, builder, want, briefing) + } + } + if n := srv.callCount("get_test_targets"); n != 0 { + t.Errorf("%s: diagnostics ran over lagging relations (%d get_test_targets calls)", name, n) + } + // The daemon waits for pending edits until wait_deadline and labels + // the rest. Without one it waits past the budget, and the labelled + // answer arrives after the briefing has given up on it. + for _, args := range srv.argsFor("detect_changes") { + deadline, err := time.Parse(time.RFC3339Nano, fmt.Sprint(args["wait_deadline"])) + if err != nil || !deadline.After(started) || !deadline.Before(started.Add(detectChangesBudget)) { + t.Errorf("%s: detect_changes wait_deadline %v does not end inside the %v budget", name, args["wait_deadline"], detectChangesBudget) + } + } + srv.Close() + } +} + +// A detect_changes that does not answer within its budget is reported, and +// the briefing returns well inside the half-second hook budget. So is one the +// daemon reached only after its wait_deadline had passed, which it refuses. +// A session that recorded no writes is not told about a slow daemon. +func TestPostTaskBriefingBoundedWhenDetectStalls(t *testing.T) { + t.Setenv(hookSessionDirEnvVar, t.TempDir()) + saveSessionState("writer", sessionState{WrittenPaths: []string{"a.go"}}) + release := make(chan struct{}) + stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-time.After(5 * time.Second): + } + http.NotFound(w, r) + })) + defer stalled.Close() + defer close(release) + late := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Arguments map[string]any `json:"arguments"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if deadline, err := time.Parse(time.RFC3339Nano, fmt.Sprint(req.Arguments["wait_deadline"])); err == nil { + time.Sleep(time.Until(deadline) + 10*time.Millisecond) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"content":[{"type":"text","text":"wait_deadline is not in the future"}],"isError":true}`)) + })) + defer late.Close() + + for name, srv := range map[string]*httptest.Server{"stalled": stalled, "refused late": late} { + port := portFromURL(t, srv.URL) + for builder, build := range map[string]func(int, sessionScope) string{ + "stop": buildPostTaskBriefing, + "mutation": buildMutationBriefing, + } { + for session, report := range map[string]bool{"": true, "writer": true, "reader": false} { + if name != "stalled" && session != "" { + continue + } + started := time.Now() + briefing := build(port, sessionScope{SessionID: session}) + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Errorf("%s %s briefing took %v", name, builder, elapsed) + } + switch { + case !report && briefing != "": + t.Errorf("%s %s briefing warned session %q, which wrote nothing:\n%q", name, builder, session, briefing) + case report && !strings.Contains(briefing, "relations: pending"): + t.Errorf("%s %s briefing did not report the busy detect_changes to session %q:\n%q", name, builder, session, briefing) + case report && strings.Contains(briefing, "publishing"): + t.Errorf("%s %s briefing claims edits are publishing without an answer:\n%q", name, builder, briefing) + } + } + } + } } // contractsCompact builds a realistic `contracts check --compact` payload: diff --git a/internal/hooks/precompact.go b/internal/hooks/precompact.go index a086c2393..a7cc8ddd4 100644 --- a/internal/hooks/precompact.go +++ b/internal/hooks/precompact.go @@ -232,6 +232,23 @@ func callServerTool(port int, name string, args map[string]any) string { return callServerToolHTTP(port, name, args) } +// callServerToolWithin is callServerTool bounded by timeout. answered is false +// when the call did not return in time: the bridge is reachable but busy, +// which a briefing reports instead of folding it into the silence of an +// unreachable bridge. The abandoned call ends with the hook process. +func callServerToolWithin(port int, name string, args map[string]any, timeout time.Duration) (raw string, answered bool) { + done := make(chan string, 1) + go func() { done <- callServerTool(port, name, args) }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case raw = <-done: + return raw, true + case <-timer.C: + return "", false + } +} + // callServerToolHTTP issues a POST /v1/tools/{name} against the server and // returns the text content from the first response block. Returns empty string // on any error (server unreachable, non-200, missing content). diff --git a/internal/hooks/pretooluse.go b/internal/hooks/pretooluse.go index 987760724..568ace12d 100644 --- a/internal/hooks/pretooluse.go +++ b/internal/hooks/pretooluse.go @@ -1455,7 +1455,60 @@ func enrichBashWithProbe(toolInput map[string]any, cwd string, probe fileScopePr if cmd == "" { return enrichResult{} } + return enrichBashCommand(cmd, cwd, probe, 0, time.Time{}) +} + +// enrichBashCommand answers one command at the given nesting depth: 0 for +// the command the agent typed, 1 for a command it runs as text (`bash -c`, +// a Python os.system), whose own nested commands are not examined. +// +// The shapes the hook has always answered come first, exactly as before: +// their analysis and probes share nothing with what follows. Only a +// command they do not deny is looked at again for what they cannot see — +// os.open, scripts behind a wrapper, nested commands — in a second +// analysis with its own budget, starting when it begins. A zero deadline +// gives each part its own; a nested command gets the deadline of the +// command it is nested in. +func enrichBashCommand(cmd, cwd string, probe fileScopeProbe, depth int, deadline time.Time) enrichResult { + r := enrichBashShapes(cmd, cwd, probe, deadline) + if r.deny { + return r + } + if deadline.IsZero() { + deadline = time.Now().Add(inlineScriptTime) + } + pyScripts, shells := inlineScriptsAt(cmd, true) + scripts := analyzeInlineScripts(pyScripts, deadline, true) + if len(scripts) == 0 && len(shells) == 0 { + return r + } + dirs := newCommandDirs(cmd) + if extra, ok := scriptReadAnswer(scripts, true, dirs, cwd, probe, time.Now()); ok { + if extra.deny { + return extra + } + if r == (enrichResult{}) { + r = extra + } + } + // A Python os.open write is a write shape the patterns cannot see: it + // gets the shell-write answer. + if writes := scriptWrites(scripts, dirs); len(writes) > 0 { + if w, answered := enrichBashWrite(BashClassification{Action: BashActionWriteSource, Writes: writes}, cwd); answered { + if w.deny { + return w + } + r = w + } + } + if depth > 0 { + return r + } + return enrichNestedCommands(r, nestedCommands(shells, scripts, dirs, cwd), probe, depth, deadline) +} +// enrichBashShapes answers the shell shapes and inline Python of a command. +func enrichBashShapes(cmd, cwd string, probe fileScopeProbe, deadline time.Time) enrichResult { c := classifyBashCommand(cmd) if c.Action == BashActionWriteSource { if result, answered := enrichBashWrite(c, cwd); answered { @@ -1488,7 +1541,7 @@ func enrichBashWithProbe(toolInput map[string]any, cwd string, probe fileScopePr // read-only or loading allowance) must not cost the command a deny its // shell read earns on its own, so the shell answer is taken first when // it denies. - py, pyAnswered := enrichBashScriptRead(cmd, cwd, probe) + py, pyAnswered := enrichBashScriptRead(cmd, cwd, probe, deadline) if pyAnswered && (py.deny || c.Action == BashActionPassthrough) { return py } diff --git a/internal/hooks/python_script_facts.go b/internal/hooks/python_script_facts.go index 2a34847ca..e9bae7e41 100644 --- a/internal/hooks/python_script_facts.go +++ b/internal/hooks/python_script_facts.go @@ -2,6 +2,7 @@ package hooks import ( "context" + "slices" "sort" "strings" "time" @@ -25,6 +26,9 @@ type pyScriptFacts struct { writeBack map[string]bool // literal paths written back guessed map[string]bool // written back only by a write it cannot name seen map[string]bool + osReads []string // literal source paths os.open opens read-only + writes []string // literal paths os.open opens for writing + commands []string // literal command lines the script runs (os.system, subprocess) } // pyMaxSteps bounds the analysis of one script and pyMaxDepth its expression @@ -54,6 +58,12 @@ type pyAnalysis struct { classes map[uintptr]bool ambiguous map[string]bool bound map[string]bool // every name bound anywhere, built on first use + osImports map[string]bool // names imported by `from os import NAME` + imports map[string]int // bindings of a name by a plain `import name` + fromNames map[string]int // bindings of a name as part of `from module import …` + bindCount map[string]int // bindings of a name in any scope, built on first use + osStar bool // `from os import *` + extra bool // collect what only the extra analysis answers loopEnd uint32 // end of the outermost loop around the node collect is at deadline time.Time root uintptr @@ -78,12 +88,19 @@ var ( "os.path.normpath": true, "os.fspath": true, "str": true, "Path": true, "pathlib.Path": true, "PurePath": true, "pathlib.PurePath": true} pyOpenModules = map[string]bool{"io": true, "codecs": true, "builtins": true} + // pyCommandCalls run a command line, or an argument vector as a list. + pyCommandCalls = map[string]bool{"os.system": true, "os.popen": true, + "subprocess.run": true, "subprocess.call": true, "subprocess.check_call": true, + "subprocess.check_output": true, "subprocess.Popen": true, + "subprocess.getoutput": true, "subprocess.getstatusoutput": true} ) // analyzePythonScript parses one script and collects what it reads and // writes. ok is false when the script does not parse cleanly or exceeds the -// analysis budget. -func analyzePythonScript(code string, deadline time.Time) (pyScriptFacts, bool) { +// analysis budget. With extra it also collects os.open accesses and the +// command lines the script runs; without, it does no work for them at all, +// so the reads and edits the hook has always answered cost what they did. +func analyzePythonScript(code string, deadline time.Time, extra bool) (pyScriptFacts, bool) { if len(code) > maxInlineScript || !time.Now().Before(deadline) { return pyScriptFacts{}, false } @@ -114,7 +131,11 @@ func analyzePythonScript(code string, deadline time.Time) (pyScriptFacts, bool) compr: map[uintptr]bool{}, classes: map[uintptr]bool{}, ambiguous: map[string]bool{}, + osImports: map[string]bool{}, + imports: map[string]int{}, + fromNames: map[string]int{}, deadline: deadline, + extra: extra, root: root.Id(), facts: pyScriptFacts{writeBack: map[string]bool{}, guessed: map[string]bool{}, seen: map[string]bool{}}, } @@ -238,6 +259,10 @@ func (a *pyAnalysis) bind(n *sitter.Node, scope uintptr, depth int) { } case "augmented_assignment": a.bindTargets(n.ChildByFieldName("left"), scope, "") + case "import_statement", "import_from_statement": + if a.extra { + a.recordImport(n) + } case "parameters", "lambda_parameters": // Parameters hold no path. Their defaults run where the function is // written, so a walrus in one binds the scope around it. @@ -616,6 +641,35 @@ func (a *pyAnalysis) collectCall(n *sitter.Node, scope uintptr) { return m, ok } switch { + case a.extra && (name == "os.open" || pyCommandCalls[name]) && !a.realModule(strings.SplitN(name, ".", 2)[0]): + // A name the script binds to something other than the module. + case a.extra && name == "os.open" && (kw["dir_fd"] != nil || hasSplat(pos)): + // Relative to a directory the hook cannot see, or flags it cannot. + case a.extra && name == "os.open": + path := kw["path"] + if len(pos) > 0 { + path = pos[0] + } + flags := kw["flags"] + if len(pos) > 1 { + flags = pos[1] + } + // A read is a read; a write is only a write target, answered after + // the rest of the command: it never makes a read an edit. + if write, ok := a.osOpenWrites(flags); ok && path != nil { + switch lit := a.eval(path, scope, 0); { + case !write: + if lit != "" && looksLikeSourceFile(lit) && !strings.ContainsAny(lit, "*?$") && !slices.Contains(a.facts.osReads, lit) && len(a.facts.osReads) < bashWriteProbeLimit { + a.facts.osReads = append(a.facts.osReads, lit) + } + case lit != "" && looksLikeSourceFile(lit) && !strings.ContainsAny(lit, "*?$") && len(a.facts.writes) < bashWriteProbeLimit: + a.facts.writes = append(a.facts.writes, lit) + } + } + case a.extra && pyCommandCalls[name]: + if line, ok := a.commandLine(name, pos, kw); ok && len(a.facts.commands) < maxNestedCommands { + a.facts.commands = append(a.facts.commands, line) + } case name == "open" && a.shadowed("open"): // The script defines or imports its own open. case name == "open" || (object != nil && method == "open" && pyOpenModules[a.dotted(object)]): @@ -641,6 +695,170 @@ func (a *pyAnalysis) collectCall(n *sitter.Node, scope uintptr) { } } +// osOpenWrites reads the flags of os.open built from os.O_* names joined by +// `|`: write is true when they open the file for writing (O_WRONLY, O_RDWR, +// O_TRUNC, O_APPEND), false for a read — O_CREAT alone changes no file that +// exists. ok is false when the flags are anything else, since a value the +// hook cannot read could be a write, and for O_PATH or O_DIRECTORY, which +// read no file content. +func (a *pyAnalysis) osOpenWrites(flags *sitter.Node) (write, ok bool) { + names := map[string]bool{} + var walk func(n *sitter.Node, depth int) bool + walk = func(n *sitter.Node, depth int) bool { + if n == nil || depth > pyMaxDepth || !a.step(depth) { + return false + } + var flag string + switch n.Type() { + case "binary_operator": + op := n.ChildByFieldName("operator") + return op != nil && a.text(op) == "|" && + walk(n.ChildByFieldName("left"), depth+1) && walk(n.ChildByFieldName("right"), depth+1) + case "parenthesized_expression": + return walk(n.NamedChild(0), depth+1) + case "attribute": + var ok bool + if flag, ok = strings.CutPrefix(a.dotted(n), "os."); !ok || !a.realModule("os") { + return false + } + case "identifier": + // Imported from os once and never rebound. + flag = a.text(n) + imported := a.osImports[flag] && a.bindings[pyBindingKey{flag, a.root}] != nil && a.bindingCount(flag) == 1 + if !imported && (!a.osStar || a.shadowed(flag)) { + return false + } + default: + return false + } + if !strings.HasPrefix(flag, "O_") || strings.Contains(flag, ".") { + return false + } + names[flag] = true + return true + } + if !walk(flags, 0) || names["O_PATH"] || names["O_DIRECTORY"] { + return false, false + } + return names["O_WRONLY"] || names["O_RDWR"] || names["O_TRUNC"] || names["O_APPEND"], true +} + +// commandLine returns the command line a pyCommandCalls call runs when it +// is literal: a string the shell runs as it is, or a list or tuple of +// strings joined with shell quoting. false for anything else, and for a +// call that runs it in another directory (cwd=) or with another program +// (executable=). os.system, os.popen and getoutput always run a string +// through the shell; the Popen family runs a string through it only with +// shell=True (without, the string is a program name), and a list only +// without it (with, the items past the first are arguments to the shell). +func (a *pyAnalysis) commandLine(name string, pos []*sitter.Node, kw map[string]*sitter.Node) (string, bool) { + // *args and **kwargs may carry anything, cwd= and shell= included. + if len(pos) == 0 || kw["cwd"] != nil || kw["executable"] != nil || hasSplat(pos) { + return "", false + } + alwaysShell := name == "os.system" || name == "os.popen" || strings.HasPrefix(name, "subprocess.get") + shell, hasShell := kw["shell"] + // An escape in the literal reads differently to Python and the shell. + literal := func(n *sitter.Node) (string, bool) { + s, ok := a.stringLiteral(n) + return s, ok && s != "" && !strings.Contains(s, `\`) + } + switch arg := pos[0]; arg.Type() { + case "string", "concatenated_string": + if !alwaysShell && (!hasShell || shell.Type() != "true") { + return "", false + } + return literal(arg) + case "list", "tuple": + if alwaysShell || hasShell && shell.Type() != "false" { + return "", false + } + var words []string + for c := range arg.NamedChildren() { + w, ok := literal(c) + if !ok || len(words) >= 256 { + return "", false + } + words = append(words, shellQuoteWord(w)) + } + return strings.Join(words, " "), len(words) > 0 + } + return "", false +} + +// recordImport counts the names an import binds for realModule: a plain +// `import os` / `import os.path` binds the module itself, and the module +// of a `from module import …` is recorded as bound too. Bare os.O_* flag +// names imported from os are recorded for os.open. +func (a *pyAnalysis) recordImport(n *sitter.Node) { + if n.Type() == "import_statement" { + for c := range n.NamedChildren() { + if c.Type() == "dotted_name" && c.NamedChildCount() > 0 { + a.imports[a.text(c.NamedChild(0))]++ + } + } + return + } + module := n.ChildByFieldName("module_name") + if module == nil { + return + } + a.countFromModule(module) + fromOS := a.text(module) == "os" + for c := range n.NamedChildren() { + switch { + case c.Id() == module.Id(): + case fromOS && c.Type() == "wildcard_import": + a.osStar = true + case fromOS && c.Type() == "dotted_name": + a.osImports[a.text(c)] = true + } + } +} + +// countFromModule counts the names bind records for the module of a +// `from module import …`, a dotted name alone or inside a relative import. +func (a *pyAnalysis) countFromModule(n *sitter.Node) { + for c := range n.NamedChildren() { + switch c.Type() { + case "identifier": + if n.Type() == "dotted_name" { + a.fromNames[a.text(c)]++ + } + case "dotted_name": + a.countFromModule(c) + } + } +} + +// hasSplat reports whether a call's arguments include *args or **kwargs. +func hasSplat(args []*sitter.Node) bool { + for _, arg := range args { + if t := arg.Type(); t == "list_splat" || t == "dictionary_splat" { + return true + } + } + return false +} + +// realModule reports whether name is bound only by plain `import name` +// statements, so name.attr is the module's own: never aliased onto +// another module, assigned, defined, deleted or declared global. +func (a *pyAnalysis) realModule(name string) bool { + return a.imports[name] > 0 && !a.ambiguous[name] && a.bindingCount(name) == a.imports[name]+a.fromNames[name] +} + +// bindingCount returns how many times the script binds name, in any scope. +func (a *pyAnalysis) bindingCount(name string) int { + if a.bindCount == nil { + a.bindCount = make(map[string]int, len(a.bindings)) + for key, bs := range a.bindings { + a.bindCount[key.name] += len(bs) + } + } + return a.bindCount[name] +} + // reboundLater reports whether the name read at pos is bound again further // down a loop that runs the read: on the next pass it holds that value. func (a *pyAnalysis) reboundLater(name string, pos uint32, scope uintptr) bool { diff --git a/internal/indexer/checkout_observe.go b/internal/indexer/checkout_observe.go index f0af1dd59..5e18ec3b3 100644 --- a/internal/indexer/checkout_observe.go +++ b/internal/indexer/checkout_observe.go @@ -19,9 +19,60 @@ import ( const ( checkoutObservationTimeout = 250 * time.Millisecond checkoutObservationWorkTimeout = 5 * time.Second - checkoutObservationCapacity = 32 + // A family runs at most this many discoveries at once. Reaching it refuses + // only that family's new paths. + checkoutObservationFamilyCapacity = 8 + // Workers still running past their work deadline (Git or a mount that + // ignores cancellation) are bounded daemon-wide. Reaching it refuses only + // families that own such a worker. + checkoutObservationStuckCapacity = 32 ) +// Reasons a CheckoutDiscoveryPendingError carries. Every one is retryable. +const ( + checkoutDiscoveryInProgress = "in_progress" + checkoutDiscoveryDraining = "draining" + checkoutDiscoveryFamilyCapacity = "family_capacity" + checkoutDiscoveryStuckCapacity = "stuck_capacity" +) + +// CheckoutDiscoveryPendingError refuses a request whose checkout discovery +// cannot answer yet. It applies to Root, the checkout root holding the +// requested path, in Family; no request outside Root is refused because of it. +type CheckoutDiscoveryPendingError struct { + Root string + Family string + Reason string + cause error +} + +func (e *CheckoutDiscoveryPendingError) Error() string { + msg := fmt.Sprintf("%v: checkout discovery is pending for %s (family %s, reason %s)", ErrCheckoutMutationBusy, e.Root, e.Family, e.Reason) + if e.cause != nil { + msg += ": " + e.cause.Error() + } + return msg +} + +// Unwrap keeps the refusal a retryable busy-lane error and its cause, such as +// the request deadline, inspectable. +func (e *CheckoutDiscoveryPendingError) Unwrap() []error { + if e.cause == nil { + return []error{ErrCheckoutMutationBusy} + } + return []error{ErrCheckoutMutationBusy, e.cause} +} + +// checkoutObservationTarget is what a request learns from its .git marker, +// without running Git, before its discovery may take capacity. +type checkoutObservationTarget struct { + root, familyID string +} + +func (t checkoutObservationTarget) refusal(reason string) error { + return &CheckoutDiscoveryPendingError{Root: t.root, Family: t.familyID, Reason: reason} +} + type checkoutObservationProof struct { inventory *gitstate.FamilyInventory familyID string @@ -41,6 +92,9 @@ type checkoutObservationMarker struct { } type checkoutObservationJob struct { + // The checkout and family the job was started for. Refusals of requests + // that join it name these, whatever those requests' markers read now. + checkoutObservationTarget ctx context.Context cancel context.CancelFunc proofReady chan struct{} @@ -61,8 +115,9 @@ type checkoutObservationJob struct { // up to five seconds across retries. Slow Git startup therefore cannot restart // the same unfinished work indefinitely. Graph builds use existing activation. // -// found=false, err=nil means the path has no known family to serve it from. -// A busy/error outcome carries no permission to fall through to another graph. +// found=false, err=nil means the path has no known family to serve it from; +// that is decided from its .git marker before any discovery work starts. A +// busy/error outcome carries no permission to fall through to another graph. // Explicit selectors must provide an authorizer for the known primary prefix; // it runs outside locks and before any catalog observation or build activation. func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string, authorize ...func(string) error) (checkout store_sqlite.Checkout, found bool, err error) { @@ -77,6 +132,7 @@ func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string } ctx, cancel := context.WithTimeout(ctx, checkoutObservationTimeout) defer cancel() + var target checkoutObservationTarget defer func() { if err != nil && ctx.Err() != nil { err = errors.Join(ctx.Err(), err) @@ -84,14 +140,24 @@ func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string return } } - if err != nil && (errors.Is(err, context.DeadlineExceeded) || retryableCheckoutRefreshError(err)) { - err = fmt.Errorf("%w: selected checkout discovery is pending: %w", ErrCheckoutMutationBusy, err) + var pending *CheckoutDiscoveryPendingError + if err != nil && !errors.As(err, &pending) && (errors.Is(err, context.DeadlineExceeded) || retryableCheckoutRefreshError(err)) { + err = &CheckoutDiscoveryPendingError{Root: target.root, Family: target.familyID, Reason: checkoutDiscoveryInProgress, cause: err} } }() - job, err := l.checkoutObservation(pathkey.CanonicalExistingRoot(path)) + path = pathkey.CanonicalExistingRoot(path) + target, known, err := l.observationTarget(ctx, path) if err != nil { return checkout, false, err } + // A discovery already running for this path still answers it, so a binding + // that changed under its pending proof is reported stale. + job, err := l.checkoutObservation(path, target, known) + if err != nil || job == nil { + return checkout, false, err + } + // Refusals from here on name the joined discovery's checkout. + target = job.checkoutObservationTarget select { case <-ctx.Done(): return checkout, false, ctx.Err() @@ -147,7 +213,24 @@ func (job *checkoutObservationJob) invalidateStaleProof(err error) error { return err } -func (l *CheckoutLifecycle) checkoutObservation(path string) (*checkoutObservationJob, error) { +// observationTarget resolves path's family from its .git marker alone. A path +// outside Git, or in a family the catalog does not know, has nothing to +// observe and never takes a discovery slot; ordinary scope resolution handles +// it. The job's own proof still resolves the family through Git. +func (l *CheckoutLifecycle) observationTarget(ctx context.Context, path string) (checkoutObservationTarget, bool, error) { + root, commonDir, ok := gitstate.MarkerCommonDir(path) + if !ok { + return checkoutObservationTarget{}, false, nil + } + target := checkoutObservationTarget{root: root, familyID: FamilyIDFor(commonDir)} + _, known, err := l.catalog.GetRepositoryFamily(ctx, target.familyID) + return target, known, err +} + +// checkoutObservation joins path's running discovery. Without one it starts +// a discovery when start is set and target's family has capacity, and +// otherwise returns no job. +func (l *CheckoutLifecycle) checkoutObservation(path string, target checkoutObservationTarget, start bool) (*checkoutObservationJob, error) { l.observationMu.Lock() defer l.observationMu.Unlock() if l.observationClosed { @@ -157,22 +240,53 @@ func (l *CheckoutLifecycle) checkoutObservation(path string) (*checkoutObservati if job.ctx.Err() == nil { return job, nil } - return nil, fmt.Errorf("%w: expired checkout discovery is draining", ErrCheckoutMutationBusy) + return nil, job.refusal(checkoutDiscoveryDraining) } - if len(l.observationJobs) >= checkoutObservationCapacity { - return nil, fmt.Errorf("%w: checkout discovery capacity reached", ErrCheckoutMutationBusy) + if !start { + return nil, nil + } + // One family's burst or stall never refuses another family's paths. + inFlight, stuck, familyStuck := 0, 0, false + for _, job := range l.observationJobs { + if !job.holdsSlot() { + continue + } + expired := job.ctx.Err() != nil + if expired { + stuck++ + } + if job.familyID == target.familyID { + inFlight++ + familyStuck = familyStuck || expired + } + } + if inFlight >= checkoutObservationFamilyCapacity { + return nil, target.refusal(checkoutDiscoveryFamilyCapacity) + } + if familyStuck && stuck >= checkoutObservationStuckCapacity { + return nil, target.refusal(checkoutDiscoveryStuckCapacity) } if l.observationJobs == nil { l.observationJobs = make(map[string]*checkoutObservationJob) } ctx, cancel := context.WithTimeout(context.Background(), checkoutObservationWorkTimeout) - job := &checkoutObservationJob{ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} + job := &checkoutObservationJob{checkoutObservationTarget: target, ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} l.observationJobs[path] = job l.observationWG.Add(1) go l.runCheckoutObservation(path, job) return job, nil } +// holdsSlot reports whether job counts against discovery capacity. A finished +// job keeps at most its result, and a proven job that no caller has authorized +// runs no Git work: it only waits for one until its deadline. +func (job *checkoutObservationJob) holdsSlot() bool { + if channelClosed(job.done) { + return false + } + return channelClosed(job.authorized) || !channelClosed(job.proofReady) +} + func (l *CheckoutLifecycle) runCheckoutObservation(path string, job *checkoutObservationJob) { defer l.observationWG.Done() defer job.cancel() diff --git a/internal/indexer/checkout_observe_test.go b/internal/indexer/checkout_observe_test.go index 2dcb634be..4a8069659 100644 --- a/internal/indexer/checkout_observe_test.go +++ b/internal/indexer/checkout_observe_test.go @@ -201,7 +201,7 @@ func TestObserveCheckoutPathRetainsStaleObservationAcrossRetries(t *testing.T) { if job == nil || job.ctx.Err() != nil { t.Fatal("stale refusal did not retain a live discovery job") } - // The retained job owns capacity and a worker until expiry or shutdown. + // The retained job holds a worker until expiry or shutdown. lc.closeCheckoutObservations() if !errors.Is(job.ctx.Err(), context.Canceled) { t.Fatalf("Close did not cancel retained stale discovery: %v", job.ctx.Err()) @@ -264,6 +264,224 @@ func TestObserveCheckoutPathUnknownFamilyAndDeniedScopeHaveNoSideEffects(t *test } } +// One family's stuck discovery bounds only that family. Its own new paths are +// refused, another known family's new worktree still gets its identity, and a +// path outside every known family never takes a discovery slot. +func TestObserveCheckoutPathStuckFamilyLeavesOtherRequestsAlone(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + base := filepath.Dir(primary) + other := filepath.Join(base, "other") + if err := os.Mkdir(other, 0o755); err != nil { + t.Fatal(err) + } + builderGit(t, other, "init", "--initial-branch=main") + builderWriteTree(t, other, builderTreeA()) + builderGit(t, other, "add", "-A") + builderGit(t, other, "commit", "-m", "other") + tracked, err := lc.Register(t.Context(), config.RepoEntry{Path: other, Name: "other"}, TrackSourceCLI) + if err != nil || tracked.CatalogErr != nil { + t.Fatalf("register other family: %v %v", err, tracked.CatalogErr) + } + stuck := filepath.Join(base, "stuck") + builderGit(t, primary, "worktree", "add", "-b", "stuck", stuck) + late := filepath.Join(base, "late") + builderGit(t, other, "worktree", "add", "-b", "late", late) + outside := []string{filepath.Join(base, "plain")} + for i := range 40 { + outside = append(outside, filepath.Join(base, fmt.Sprintf("unknown-%d", i))) + } + for i, dir := range outside { + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if i > 0 { + builderGit(t, dir, "init", "--initial-branch=main") + } + } + release := make(chan struct{}) + defer close(release) + stuckRoot := pathkey.CanonicalExistingRoot(stuck) + var entered atomic.Int32 + lc.observationInventory = func(ctx context.Context, path string) (*gitstate.FamilyInventory, error) { + if pathkey.HasPathPrefix(path, stuckRoot) { + entered.Add(1) + <-release // Git or a mount that ignores cancellation. + return nil, context.Canceled + } + return gitstate.Inventory(ctx, path) + } + activeJobs := func() int { + lc.observationMu.Lock() + defer lc.observationMu.Unlock() + return len(lc.observationJobs) + } + observeOutside := func(phase string, wantJobs int) { + t.Helper() + for _, dir := range outside { + _, found, err := lc.ObserveCheckoutPath(t.Context(), dir) + if jobs := activeJobs(); found || err != nil || jobs != wantJobs { + t.Errorf("%s: %s is in no known family: found=%v err=%v jobs=%d, want %d", phase, dir, found, err, jobs, wantJobs) + return + } + } + } + observeOutside("idle", 0) + + requests := make([]string, 40) + errs := make([]error, len(requests)) + for i := range requests { + requests[i] = filepath.Join(stuck, fmt.Sprintf("request-%d", i)) + if err := os.Mkdir(requests[i], 0o755); err != nil { + t.Fatal(err) + } + } + var wg sync.WaitGroup + for i := range requests { + wg.Go(func() { _, _, errs[i] = lc.ObserveCheckoutPath(t.Context(), requests[i]) }) + } + wg.Wait() + for i, err := range errs { + if !errors.Is(err, ErrCheckoutMutationBusy) { + t.Fatalf("stuck family request %d: %v", i, err) + } + } + lc.observationMu.Lock() + jobs := make([]*checkoutObservationJob, 0, len(lc.observationJobs)) + draining := "" + for path, job := range lc.observationJobs { + jobs = append(jobs, job) + draining = path + } + lc.observationMu.Unlock() + if len(jobs) != checkoutObservationFamilyCapacity { + t.Errorf("one family holds %d discovery jobs, want its own bound of %d", len(jobs), checkoutObservationFamilyCapacity) + } + deadline := time.Now().Add(30 * time.Second) + for int(entered.Load()) < len(jobs) { + if time.Now().After(deadline) { + t.Fatalf("workers did not reach the stalled inventory: %d of %d", entered.Load(), len(jobs)) + } + time.Sleep(time.Millisecond) + } + for _, job := range jobs { + job.cancel() // Past the work deadline; the workers keep running. + } + + observeOutside("stuck", len(jobs)) + // Another family's new worktree may wait for its own discovery only. + lateRoot := pathkey.CanonicalExistingRoot(late) + for deadline := time.Now().Add(checkoutObservationWorkTimeout); ; { + checkout, found, err := lc.ObserveCheckoutPath(t.Context(), late) + var pending *CheckoutDiscoveryPendingError + if !errors.As(err, &pending) { + if err != nil || !found || checkout.FamilyID != tracked.FamilyID || !pathkey.EqualPaths(checkout.RootPath, lateRoot) { + t.Errorf("another family's new worktree was not admitted: %+v found=%v err=%v", checkout, found, err) + } + break + } + if pending.Reason != checkoutDiscoveryInProgress || pending.Family != tracked.FamilyID || !pathkey.EqualPaths(pending.Root, lateRoot) { + t.Fatalf("another family's new worktree was refused: %v", err) + } + if time.Now().After(deadline) { + t.Fatalf("another family's new worktree discovery did not finish: %v", err) + } + time.Sleep(time.Millisecond) + } + extra := filepath.Join(stuck, "extra") + if err := os.Mkdir(extra, 0o755); err != nil { + t.Fatal(err) + } + _, _, err = lc.ObserveCheckoutPath(t.Context(), extra) + var pending *CheckoutDiscoveryPendingError + if !errors.Is(err, ErrCheckoutMutationBusy) || !errors.As(err, &pending) || !pathkey.EqualPaths(pending.Root, stuckRoot) || pending.Family != familyID || pending.Reason != checkoutDiscoveryFamilyCapacity { + t.Errorf("stuck family's new path: %v", err) + } + if _, _, err := lc.ObserveCheckoutPath(t.Context(), draining); !errors.Is(err, ErrCheckoutMutationBusy) || checkoutDiscoveryReason(err) != checkoutDiscoveryDraining { + t.Errorf("stuck path did not keep its draining refusal: %v", err) + } +} + +// A request that joins a running discovery is refused for that discovery's +// checkout, whatever its own .git marker read finds now. Here the marker of a +// worktree nested in the primary checkout is removed, so a fresh read names +// the primary checkout instead. +func TestObserveCheckoutPathJoinedRefusalNamesItsOwnCheckout(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + nested := filepath.Join(primary, "nested") + builderGit(t, primary, "worktree", "add", "-b", "nested", nested) + nestedRoot := pathkey.CanonicalExistingRoot(nested) + release := make(chan struct{}) + defer close(release) + entered := make(chan struct{}) + var once sync.Once + lc.observationInventory = func(context.Context, string) (*gitstate.FamilyInventory, error) { + once.Do(func() { close(entered) }) + <-release // Ignores cancellation, so the job outlives its deadline. + return nil, context.Canceled + } + refused := func(phase, reason string) { + t.Helper() + _, found, err := lc.ObserveCheckoutPath(t.Context(), nested) + var pending *CheckoutDiscoveryPendingError + if found || !errors.As(err, &pending) || pending.Reason != reason || pending.Family != familyID || !pathkey.EqualPaths(pending.Root, nestedRoot) { + t.Fatalf("%s: want a %s refusal for %s in family %s, got found=%v err=%v", phase, reason, nestedRoot, familyID, found, err) + } + } + refused("first request", checkoutDiscoveryInProgress) + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("discovery did not reach the stalled inventory") + } + if err := os.Remove(filepath.Join(nested, ".git")); err != nil { + t.Fatal(err) + } + refused("joined request", checkoutDiscoveryInProgress) + job := checkoutObservationJobForTest(lc, nested) + if job == nil { + t.Fatal("the stalled discovery is not registered") + } + job.cancel() // Past its work deadline; the worker keeps running. + refused("draining request", checkoutDiscoveryDraining) +} + +// A proven discovery that no caller was authorized to use runs no Git work; +// it only waits for an authorized caller until its deadline. Requests denied +// in a family therefore never hold that family's discovery slots. +func TestObserveCheckoutPathDeniedDiscoveriesLeaveFamilyCapacity(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + worktree := filepath.Join(filepath.Dir(primary), "own") + builderGit(t, primary, "worktree", "add", "-b", "own", worktree) + denied := errors.New("outside authorized workspace") + for i := range checkoutObservationFamilyCapacity { + dir := filepath.Join(primary, fmt.Sprintf("denied-%d", i)) + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if _, found, err := observeCheckoutUntilSettled(t, lc, dir, func(string) error { return denied }); found || !errors.Is(err, denied) { + t.Fatalf("denied observation %d: found=%v err=%v", i, found, err) + } + } + lc.observationMu.Lock() + waiting := 0 + for _, job := range lc.observationJobs { + if job.familyID == familyID && job.ctx.Err() == nil && !channelClosed(job.authorized) { + waiting++ + } + } + lc.observationMu.Unlock() + if waiting != checkoutObservationFamilyCapacity { + t.Fatalf("%d denied discoveries still wait for authorization, want %d", waiting, checkoutObservationFamilyCapacity) + } + if _, _, err := lc.ObserveCheckoutPath(t.Context(), worktree); checkoutDiscoveryReason(err) == checkoutDiscoveryFamilyCapacity { + t.Fatalf("denied discoveries held the family's slots: %v", err) + } + checkout, found, err := observeCheckoutUntilSettled(t, lc, worktree) + if err != nil || !found || checkout.FamilyID != familyID || !pathkey.EqualPaths(checkout.RootPath, pathkey.CanonicalExistingRoot(worktree)) { + t.Fatalf("the family's own new worktree was not admitted: %+v found=%v err=%v", checkout, found, err) + } +} + func TestObserveCheckoutPathCancellationAndClosingDoNotAllocate(t *testing.T) { lc, catalog, primary, familyID, _, closeFixture := newCheckoutObservationFixture(t) worktree := filepath.Join(filepath.Dir(primary), "canceled") @@ -390,54 +608,84 @@ func TestObserveCheckoutPathSlowObservationRunsOnceAcrossRetries(t *testing.T) { } } -func TestObserveCheckoutPathExpiredJobsStayBoundedUntilTheyDrain(t *testing.T) { +// checkoutDiscoveryReason is the reason of a pending discovery refusal, "" +// for any other outcome. +func checkoutDiscoveryReason(err error) string { + var pending *CheckoutDiscoveryPendingError + if errors.As(err, &pending) { + return pending.Reason + } + return "" +} + +func TestObserveCheckoutPathStuckJobsStayBoundedWithoutRefusingOtherFamilies(t *testing.T) { lc, catalog, primary, familyID, _, _ := newCheckoutObservationFixture(t) release := make(chan struct{}) defer close(release) - entered := make(chan struct{}) - var started atomic.Int32 + var entered atomic.Int32 lc.observationInventory = func(context.Context, string) (*gitstate.FamilyInventory, error) { - if started.Add(1) == checkoutObservationCapacity { - close(entered) - } + entered.Add(1) <-release // Deliberately model a dependency that ignores cancellation. return nil, context.Canceled } - jobs := make([]*checkoutObservationJob, 0, checkoutObservationCapacity) - for i := 0; i < checkoutObservationCapacity; i++ { - path := filepath.Join(primary, fmt.Sprintf("request-%d", i)) - if err := os.Mkdir(path, 0o755); err != nil { + // Capacity is accounted to the family the pre-check names. Every worker + // proves the fixture's real family through Git, then stalls in Inventory. + observe := func(family string, i int) (*checkoutObservationJob, error) { + path := filepath.Join(primary, fmt.Sprintf("%s-%d", family, i)) + if err := os.MkdirAll(path, 0o755); err != nil { t.Fatal(err) } - job, err := lc.checkoutObservation(path) - if err != nil { - t.Fatal(err) + return lc.checkoutObservation(path, checkoutObservationTarget{root: primary, familyID: family}, true) + } + var jobs []*checkoutObservationJob + for f := 0; f < checkoutObservationStuckCapacity/checkoutObservationFamilyCapacity; f++ { + family := fmt.Sprintf("full-%d", f) + for i := 0; i < checkoutObservationFamilyCapacity; i++ { + job, err := observe(family, i) + if err != nil { + t.Fatal(err) + } + jobs = append(jobs, job) } - jobs = append(jobs, job) + if _, err := observe(family, checkoutObservationFamilyCapacity); checkoutDiscoveryReason(err) != checkoutDiscoveryFamilyCapacity { + t.Fatalf("full family admitted another discovery: %v", err) + } + } + partial, err := observe("partial", 0) + if err != nil { + t.Fatal(err) } + jobs = append(jobs, partial) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() - select { - case <-entered: - case <-ctx.Done(): - t.Fatalf("workers did not enter inventory: started=%d: %v", started.Load(), ctx.Err()) - } - if got := started.Load(); got != checkoutObservationCapacity { - t.Fatalf("workers did not start: %d", got) + for int(entered.Load()) < len(jobs) { + if ctx.Err() != nil { + t.Fatalf("workers did not enter inventory: %d of %d", entered.Load(), len(jobs)) + } + time.Sleep(time.Millisecond) } for _, job := range jobs { job.cancel() // Expiry must not release capacity before the worker exits. } - for i := 0; i < 2*checkoutObservationCapacity; i++ { - if _, err := lc.checkoutObservation(filepath.Join(primary, fmt.Sprintf("extra-%d", i))); !errors.Is(err, ErrCheckoutMutationBusy) { - t.Fatalf("canceled but running jobs released capacity: %v", err) + // Past the daemon-wide stuck bound, a family that owns stuck work is + // refused below its own bound, and a stuck path keeps draining. + for i := 1; i < 2*checkoutObservationStuckCapacity; i++ { + if _, err := observe("partial", i); checkoutDiscoveryReason(err) != checkoutDiscoveryStuckCapacity { + t.Fatalf("family with stuck work escaped the stuck bound: %v", err) } } + if _, err := observe("partial", 0); checkoutDiscoveryReason(err) != checkoutDiscoveryDraining { + t.Fatalf("stuck path was not draining: %v", err) + } + // A family without stuck work is never refused for another family's. + if job, err := observe("fresh", 0); err != nil || job == nil { + t.Fatalf("family without stuck work was refused: %v", err) + } lc.observationMu.Lock() active := len(lc.observationJobs) lc.observationMu.Unlock() - if active != checkoutObservationCapacity || started.Load() != checkoutObservationCapacity { - t.Fatalf("discovery bound escaped: jobs=%d workers=%d", active, started.Load()) + if want := len(jobs) + 1; active != want || int(entered.Load()) > want { + t.Fatalf("discovery bound escaped: jobs=%d workers=%d, want %d", active, entered.Load(), want) } rows, err := catalog.ListCheckouts(t.Context(), familyID) if err != nil || len(rows) != 1 { @@ -455,7 +703,7 @@ func TestObserveCheckoutPathCloseCancelsAndJoinsMetadata(t *testing.T) { close(exited) return nil, ctx.Err() } - if _, err := lc.checkoutObservation(primary); err != nil { + if _, err := lc.checkoutObservation(primary, checkoutObservationTarget{}, true); err != nil { t.Fatal(err) } <-entered @@ -473,7 +721,7 @@ func TestObserveCheckoutPathCloseCancelsAndJoinsMetadata(t *testing.T) { if active != 0 { t.Fatalf("Close retained %d discovery jobs", active) } - if _, err := lc.checkoutObservation(primary); !errors.Is(err, ErrCheckoutRefreshStopped) { + if _, err := lc.checkoutObservation(primary, checkoutObservationTarget{}, true); !errors.Is(err, ErrCheckoutRefreshStopped) { t.Fatalf("closed lifecycle admitted new work: %v", err) } } @@ -504,7 +752,7 @@ func TestObserveCheckoutPathRejectsInventoryWithReboundGitMarker(t *testing.T) { // Judge the job that received the pre-rebind inventory, not a request retry // loop. Slow Git can outlast the request wait, the refused job is then // forgotten, and a retry's fresh inventory legitimately sees the new binding. - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } @@ -531,7 +779,7 @@ func TestObserveCheckoutPathRejectsBindingChangesWhileProofIsPending(t *testing. lc, catalog, primary, familyID, _, _ := newCheckoutObservationFixture(t) worktree := filepath.Join(filepath.Dir(primary), "pending-proof") builderGit(t, primary, "worktree", "add", "-b", "pending-proof", worktree) - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } @@ -630,7 +878,7 @@ func completedCheckoutObservationForTest(t *testing.T) (*CheckoutLifecycle, *che calls.Add(1) return gitstate.Inventory(ctx, path) } - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } diff --git a/internal/indexer/checkout_observe_unknown_family_test.go b/internal/indexer/checkout_observe_unknown_family_test.go index 2a1a25258..d69545bc7 100644 --- a/internal/indexer/checkout_observe_unknown_family_test.go +++ b/internal/indexer/checkout_observe_unknown_family_test.go @@ -56,7 +56,7 @@ func TestObserveCheckoutPathTrackedNonGitUnknownAncestorSkipsInventory(t *testin return gitstate.Inventory(ctx, path) } for range 3 { - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root), checkoutObservationTarget{}, true) require.NoError(t, err) select { case <-job.done: @@ -90,7 +90,7 @@ func TestObserveCheckoutPathTrackedNonGitUnknownAncestorSkipsInventory(t *testin // worktree of the known family. No remembered negative may hide its identity. require.NoError(t, os.Rename(root, root+"-before-worktree")) builderGit(t, primary, "worktree", "add", "-b", "competitor-worktree", root) - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root), checkoutObservationTarget{}, true) require.NoError(t, err) select { case <-job.proofReady: diff --git a/internal/indexer/drain_lane_yield.go b/internal/indexer/drain_lane_yield.go index df25c84b8..4e09a33d6 100644 --- a/internal/indexer/drain_lane_yield.go +++ b/internal/indexer/drain_lane_yield.go @@ -2,8 +2,12 @@ package indexer import ( "context" + "errors" + "fmt" "time" + "go.uber.org/zap" + "github.com/zzet/gortex/internal/graph" ) @@ -33,6 +37,12 @@ var drainYieldRows = 1024 // cooperative drain. var drainSubBatchHook func() +// shadowEvictionBranchHook, when set (tests), observes which eviction a shadow +// drain chose before its first disk write: true for the SQLite +// shadow-replacement retirement, false for the generic current-generation +// eviction. +var shadowEvictionBranchHook func(retirement bool) + type initialDrainCooperationKey struct{} // Only the initial claimed base pauses in place for announced checkout work. @@ -87,11 +97,13 @@ func drainYieldable(ctx context.Context) bool { // claimed drain uses sub-batches and stops on context cancellation. The initial // claimed drain also stands down, boundedly, for interactive demand before each // slice; ordinary and foreground drains keep their single-batch behavior. +// +// A cooperative stop returns the context's own error; anything else is the +// store refusing a write (see drainWrite). drainBatchError tells them apart. func drainAddBatch(ctx context.Context, target graph.Store, nodes []*graph.Node, edges []*graph.Edge) error { wanted := initialDrainDemand(ctx) if !drainYieldable(ctx) && wanted == nil { - target.AddBatch(nodes, edges) - return nil + return drainWrite(ctx, target, nodes, edges) } step := drainYieldRows if step <= 0 { @@ -110,13 +122,63 @@ func drainAddBatch(ctx context.Context, target graph.Store, nodes []*graph.Node, } if len(nodes) > 0 { n := min(len(nodes), step) - target.AddBatch(nodes[:n], nil) + if err := drainWrite(ctx, target, nodes[:n], nil); err != nil { + return err + } nodes = nodes[n:] continue } n := min(len(edges), step) - target.AddBatch(nil, edges[:n]) + if err := drainWrite(ctx, target, nil, edges[:n]); err != nil { + return err + } edges = edges[n:] } return nil } + +// drainCheckedWriter is the error-returning batch write a durable backend +// offers beside the void graph.Store AddBatch. +type drainCheckedWriter interface { + AddBatchCheckedContext(ctx context.Context, nodes []*graph.Node, edges []*graph.Edge) error +} + +// drainWrite writes one batch, through the checked path when target has one, +// so a storage failure (a full volume, an I/O error, a failed deferred-index +// seal) fails the drain as an error instead of panicking out of AddBatch. The +// write itself never observes ctx's cancellation: a drain stops between +// batches, never inside one. +func drainWrite(ctx context.Context, target graph.Store, nodes []*graph.Node, edges []*graph.Edge) error { + if checked, ok := target.(drainCheckedWriter); ok { + return checked.AddBatchCheckedContext(context.WithoutCancel(ctx), nodes, edges) + } + target.AddBatch(nodes, edges) + return nil +} + +// drainBatchError wraps a drainAddBatch error for the build's result: a +// cooperative stop keeps its established wording, and a refused write says the +// graph could not be persisted. +func drainBatchError(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, ctxErr) { + return fmt.Errorf("indexer: cooperative drain stopped: %w", err) + } + return fmt.Errorf("indexer: persist bulk graph batch: %w", err) +} + +// retryBulkFlushAfterPanic retries a FlushBulk that panicked, once, while that +// panic is unwinding. The retry recovers its own panic and only logs what it +// hit, so the original panic is the one that reaches the caller. Under an +// outer coordinated or generation window FlushBulk is a no-op, so this is too. +func retryBulkFlushAfterPanic(bl graph.BulkLoader, logger *zap.Logger, repo string) { + defer func() { + if r := recover(); r != nil { + logger.Error("indexer: bulk flush retry after panic panicked", + zap.String("repo", repo), zap.Any("panic", r)) + } + }() + if err := bl.FlushBulk(); err != nil { + logger.Error("indexer: bulk flush retry after panic failed", + zap.String("repo", repo), zap.Error(err)) + } +} diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index 22bff4e4e..ae8b7c639 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -34,6 +34,11 @@ var _ graph.FileIdentityBatchEvicter = (*graph.DeltaWriter)(nil) // deliberately does not implement, with the reason. The reason names what // the engine does instead. var editDeltaAbsentCapabilities = map[string]string{ + "anonymous{DBStats}": "physical durable-store file-size telemetry; full-shadow drain samples a real store, whereas DeltaWriter stages per-file changes without owning a database or WAL file", + + "anonymous{BulkLoadWindow}": "physical durable-store full-shadow window telemetry; DeltaWriter stages per-file changes and does not own a cold or generation bulk window", + "anonymous{RecordFirstIndexDrain}": "physical durable-store full-shadow completion telemetry; a per-file DeltaWriter has no full-shadow drain to record and must not forward writes to its base", + "graph.BFSAdmissionReader": "optional selected-reader traversal snapshot; the query engine retains its composed point-walk admission when a delta has no snapshot accelerator", "graph.FilteredBFSAdmissionReader": "physical indexed scoped traversal; forwarding it would discard delta ownership, so the engine keeps composed Go admission instead", "graph.KindAdjacencyReader": "optional typed adjacency accelerator; ReadKindAdjacency filters the delta's composed point reads instead of forwarding physical rows past delta ownership", @@ -215,6 +220,7 @@ var editDeltaAbsentCapabilities = map[string]string{ "indexer.repoSemanticStateReleaser": "semantic (go/types) state: the enrichment stage runs over the generation handle after the delta (runEnrichment), not over the DeltaWriter", "indexer.repoSemanticStateRetainer": "semantic (go/types) state: the enrichment stage runs over the generation handle after the delta (runEnrichment), not over the DeltaWriter", "indexer.repositoryCleanupPayload": "whole-repository or administrative operation (full index, repository reset, untrack or cleanup, warm start, vector corpus, bulk load windows, WAL maintenance); not on a file delta's per-save path", + "indexer.drainCheckedWriter": "checked write of the whole-repository shadow drain into the durable store (full index, shadow replacement); not on a file delta's per-save path, and a store without it takes the void AddBatch", "resolver.backendResolveWorkIndicator": "whole-graph pass (ResolveAll, a cold warm-up, a whole-repository derived rebuild); a delta resolves and re-derives by its receipt frontier", "resolver.batchClaimingResolver": "framework synthesizer contract, asserted on synthesizers", "resolver.claimEdgeVocabulary": "framework synthesizer contract, asserted on synthesizers", diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index d6e2e0ba5..30855ebfc 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -6,12 +6,13 @@ import ( "errors" "os" "path/filepath" - "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" @@ -19,18 +20,14 @@ import ( "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" "github.com/zzet/gortex/internal/progress" + "github.com/zzet/gortex/internal/search" ) // Embed the real store so the shadow and its optional persistence capabilities // stay intact. Only the drain boundaries are intercepted. type firstIndexBulkProbe struct { *store_sqlite.Store - beforeBegin func() duringDrain func() - ftsErr error - panicDrain bool - panicFlush bool - flushMu sync.Mutex begins int flushes int batches int @@ -38,41 +35,41 @@ type firstIndexBulkProbe struct { func (s *firstIndexBulkProbe) BeginBulkLoad() { s.begins++ - if s.beforeBegin != nil { - s.beforeBegin() - } s.Store.BeginBulkLoad() } -func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) { +func (s *firstIndexBulkProbe) beforeBatch() { s.batches++ if s.duringDrain != nil { s.duringDrain() } - if s.panicDrain { - panic("injected drain panic") - } +} + +func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) { + s.beforeBatch() s.Store.AddBatch(nodes, edges) } -func (s *firstIndexBulkProbe) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) error { - if s.ftsErr != nil { - return s.ftsErr - } - return s.Store.BatchUpsertSymbolFTS(items) +// The drain writes through the checked path when the target has one. +func (s *firstIndexBulkProbe) AddBatchCheckedContext(ctx context.Context, nodes []*graph.Node, edges []*graph.Edge) error { + s.beforeBatch() + return s.Store.AddBatchCheckedContext(ctx, nodes, edges) } func (s *firstIndexBulkProbe) FlushBulk() error { s.flushes++ - if s.panicFlush { - // Model a FlushBulk panic that leaves its writer lock held. - s.flushMu.Lock() - panic("injected flush panic") - } return s.Store.FlushBulk() } func newFirstIndexBulkFixture(t *testing.T) (*firstIndexBulkProbe, *sql.DB) { + t.Helper() + store, db := newFirstIndexStore(t) + return &firstIndexBulkProbe{Store: store}, db +} + +// newFirstIndexStore opens a fresh on-disk store for the shadow route with a +// read-only pool beside it for schema probes. +func newFirstIndexStore(t *testing.T) (*store_sqlite.Store, *sql.DB) { t.Helper() t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") t.Setenv("GORTEX_SHADOW_MAX_BYTES", "1073741824") @@ -84,7 +81,19 @@ func newFirstIndexBulkFixture(t *testing.T) (*firstIndexBulkProbe, *sql.DB) { db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?mode=ro&_pragma=synchronous(NORMAL)") require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) - return &firstIndexBulkProbe{Store: store}, db + return store, db +} + +// firstIndexWriterSynchronous reads PRAGMA synchronous off the store's writer +// connection. It takes the write gate under a deadline, so it also fails, +// rather than hangs, when a failure path leaked the gate. +func firstIndexWriterSynchronous(t *testing.T, store *store_sqlite.Store) int64 { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + sync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err, "the store's write gate is still held") + return sync } func firstIndexFixtureRepo(t *testing.T) string { @@ -141,33 +150,194 @@ func assertFirstIndexRestored(t *testing.T, store *store_sqlite.Store, db *sql.D require.False(t, firstIndexBulkActive(t, store), "standalone window must be closed before restoration probes") assert.Equal(t, indexes, firstIndexSchemaIndexes(t, db)) // PRAGMA synchronous is connection-local, so this read-only pool cannot - // check writer restoration. TestBulkLoadDropsAndRebuildsIndexes in - // store_sqlite checks the actual writer connection instead. + // check writer restoration; firstIndexWriterSynchronous reads the writer. } -func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { - probe, db := newFirstIndexBulkFixture(t) - indexes := firstIndexSchemaIndexes(t, db) - probe.beforeBegin = func() { assertFirstIndexNoReceipts(t, probe.Store, "repo") } - probe.duringDrain = func() { - assert.True(t, firstIndexBulkActive(t, probe.Store), "cold shadow drain must engage standalone bulk mode") - assertFirstIndexNoReceipts(t, probe.Store, "repo") - assert.NotEqual(t, indexes, firstIndexSchemaIndexes(t, db), "dense indexes must be deferred during drain") - } - idx := newFirstIndexBulkIndexer(t, probe, "repo") - result, err := idx.IndexCtx(context.Background(), firstIndexFixtureRepo(t)) +// sqliteFullError provokes a genuine SQLITE_FULL from the driver on a +// throwaway in-memory database capped at a few pages, so a test can hand a +// store the exact error a full volume produces without filling a disk. +func sqliteFullError(t *testing.T) error { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") require.NoError(t, err) - require.Positive(t, result.NodeCount) - require.Positive(t, probe.batches, "test must observe the shadow drain") - assert.Equal(t, 1, probe.begins) - assert.Equal(t, 1, probe.flushes) - state, found, err := probe.GetRepoIndexState("repo") + defer func() { require.NoError(t, db.Close()) }() + db.SetMaxOpenConns(1) + _, err = db.Exec(`PRAGMA max_page_count = 2`) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE filler(v BLOB)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO filler VALUES (randomblob(65536))`) + require.Error(t, err) + return err +} + +// A drain chunk the store refuses comes back as a typed storage error, for an +// ordinary drain and for the initial claimed one, instead of panicking out of +// the void AddBatch, and none of the refused chunk is committed. +func TestDrainAddBatchReturnsStorageFailure(t *testing.T) { + full := sqliteFullError(t) + for _, tc := range []struct { + name string + ctx context.Context + }{ + {"ordinary", context.Background()}, + {"initial claimed", withInitialDrainCooperation(context.Background(), func() bool { return false })}, + } { + t.Run(tc.name, func(t *testing.T) { + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + defer store.SetBatchWriteHookForTest(func(int, int) error { return full })() + nodes := []*graph.Node{{ + ID: "app.py::fetch", Name: "fetch", Kind: graph.KindFunction, + RepoPrefix: "repo", FilePath: "app.py", Language: "python", + }} + + var drainErr error + require.NotPanics(t, func() { drainErr = drainAddBatch(tc.ctx, store, nodes, nil) }) + var storageErr *store_sqlite.StorageError + require.ErrorAs(t, drainErr, &storageErr) + assert.Contains(t, store_sqlite.SafeStorageFailureReason(drainErr), "SQLite code 13") + assert.Contains(t, drainBatchError(tc.ctx, drainErr).Error(), "indexer: persist bulk graph batch:") + assert.Zero(t, store.NodeCount(), "the refused chunk must roll back") + }) + } +} + +// firstIndexWindowWitness counts a store's batch writes, how many of them +// landed inside an engaged bulk window with the dense indexes deferred, and how +// often each dense index was sealed. fail, when set, decides whether the store +// refuses a batch. +type firstIndexWindowWitness struct { + batches, engaged, deferred int + seals map[string]int + fail func(edgeRows int) error +} + +func watchFirstIndexWindow(t *testing.T, store *store_sqlite.Store, db *sql.DB, indexes []string) *firstIndexWindowWitness { + t.Helper() + w := &firstIndexWindowWitness{seals: map[string]int{}} + t.Cleanup(store.SetBulkSealHookForTest(func(index string) error { + w.seals[index]++ + return nil + })) + t.Cleanup(store.SetBatchWriteHookForTest(func(_, edgeRows int) error { + w.batches++ + if firstIndexBulkActive(t, store) { + w.engaged++ + } + if !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) { + w.deferred++ + } + if w.fail != nil { + return w.fail(edgeRows) + } + return nil + })) + // The eviction runs just before the window opens, so this is the last + // point at which a receipt could wrongly precede the drain. + shadowEvictionBranchHook = func(retirement bool) { + assert.True(t, retirement, "a SQLite target must take the shadow-replacement retirement") + assertFirstIndexNoReceipts(t, store, "repo") + } + t.Cleanup(func() { shadowEvictionBranchHook = nil }) + return w +} + +func (w *firstIndexWindowWitness) assertEngaged(t *testing.T) { + t.Helper() + require.Positive(t, w.batches, "test must observe the shadow drain") + assert.Positive(t, w.engaged, "cold shadow drain must engage standalone bulk mode") + assert.Positive(t, w.deferred, "dense indexes must be deferred during drain") + // One window per drain: a second one would seal every index again. + require.NotEmpty(t, w.seals, "the window must seal the indexes it deferred") + for index, n := range w.seals { + assert.Equal(t, 1, n, "index %s sealed %d times", index, n) + } +} + +func assertFirstIndexReceipts(t *testing.T, store *store_sqlite.Store, result *IndexResult) { + t.Helper() + state, found, err := store.GetRepoIndexState("repo") require.NoError(t, err) require.True(t, found) assert.Equal(t, result.NodeCount, state.NodeCount) assert.Equal(t, result.EdgeCount, state.EdgeCount) - assert.Contains(t, probe.LoadFileMtimes("repo"), "app.py") - assertFirstIndexRestored(t, probe.Store, db, indexes) + assert.NotEmpty(t, store.LoadFileMtimes("repo")) +} + +// A cold first index drains through a standalone bulk window on every route +// into it, and again on a retry after a failed drain left rows behind. +func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { + t.Run("IndexCtx on a fresh store", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + + result, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), firstIndexFixtureRepo(t)) + require.NoError(t, err) + require.Positive(t, result.NodeCount) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Contains(t, store.LoadFileMtimes("repo"), "app.py") + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) + + t.Run("TrackRepoCtx on an empty multi-repo store", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "main.go"), "package app\n\nfunc Fetch(url string) string { return url }\n") + mi := NewMultiIndexer(store, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + + result, err := mi.TrackRepoCtx(t.Context(), config.RepoEntry{Name: "repo", Path: dir}) + require.NoError(t, err) + require.Positive(t, result.NodeCount) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) + + t.Run("a retry after a failed drain engages the window again", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + injected := errors.New("injected edge batch failure") + refused := false + w.fail = func(edgeRows int) error { + if edgeRows > 0 && !refused { + refused = true + return injected + } + return nil + } + dir := firstIndexFixtureRepo(t) + + // The node batches land; the first edge batch is refused. + _, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), dir) + require.ErrorIs(t, err, injected) + require.True(t, refused, "the drain never reached an edge batch") + w.assertEngaged(t) + require.Positive(t, store.NodeCount(), "the failed drain must leave rows for the retry to evict") + assertFirstIndexNoReceipts(t, store, "repo") + assertFirstIndexRestored(t, store, db, indexes) + + // The retry evicts them, which leaves the store cold again. + w.batches, w.engaged, w.deferred, w.seals = 0, 0, 0, map[string]int{} + result, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), dir) + require.NoError(t, err) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) } func TestSecondRepoWarmStoreNoBulkWindow(t *testing.T) { @@ -227,81 +397,138 @@ func (firstIndexPanicReporter) Report(stage string, _, _ int) { } } +// firstIndexDrainFTSTrigger makes every symbol FTS append on store fail, the +// way a storage error inside the drain's FTS writes would. +func firstIndexDrainFTSTrigger(t *testing.T, store *store_sqlite.Store) { + t.Helper() + writer, err := sql.Open("sqlite", "file:"+filepath.ToSlash(store.Path())) + require.NoError(t, err) + _, err = writer.Exec(`CREATE TRIGGER fail_first_index_fts BEFORE INSERT ON symbol_fts_rowid +BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) + require.NoError(t, err) + require.NoError(t, writer.Close()) +} + +// Every way a first index can fail, on the real store: the bulk window must +// close (no leaked write gate, writer durability back, the dense indexes back), +// no success receipt may be written, and the indexer must hand its disk target +// back. Faults inside the store come from its own test hooks, so the eviction +// the drain picks is the production one for a SQLite target. func TestFirstIndexFailureRestoresStore(t *testing.T) { - for _, mode := range []string{"drain error", "cancel before drain", "panic in body", "panic in drain"} { + for _, mode := range []string{ + "drain error", "cancel before drain", "panic in body", "panic in drain", + "storage full", "seal panic", "seal panic after a drain panic", "seal error", + } { t.Run(mode, func(t *testing.T) { - probe, db := newFirstIndexBulkFixture(t) + store, db := newFirstIndexStore(t) indexes := firstIndexSchemaIndexes(t, db) - injected := errors.New("injected drain FTS error") + writerSync := firstIndexWriterSynchronous(t, store) + var evictions []bool + shadowEvictionBranchHook = func(retirement bool) { evictions = append(evictions, retirement) } + t.Cleanup(func() { shadowEvictionBranchHook = nil }) + var fault func() error + batches, engaged := 0, false + defer store.SetBatchWriteHookForTest(func(int, int) error { + batches++ + engaged = engaged || firstIndexBulkActive(t, store) + if fault != nil { + return fault() + } + return nil + })() + seals := 0 + defer store.SetBulkSealHookForTest(func(index string) error { + seals++ + switch { + case (mode == "seal panic" || mode == "seal panic after a drain panic") && seals == 1: + panic("injected seal panic") + case mode == "seal error" && index == "nodes_by_kind": + return errors.New("injected seal failure") + } + return nil + })() ctx, cancel := context.WithCancel(context.Background()) defer cancel() switch mode { case "drain error": - probe.ftsErr = injected - case "panic in drain": - probe.panicDrain = true + firstIndexDrainFTSTrigger(t, store) + case "panic in drain", "seal panic after a drain panic": + fault = func() error { panic("injected drain panic") } + case "storage full": + full := sqliteFullError(t) + fault = func() error { return full } case "cancel before drain": ctx = progress.WithReporter(ctx, firstIndexCancelReporter{cancel: cancel}) case "panic in body": ctx = progress.WithReporter(ctx, firstIndexPanicReporter{}) } - probe.duringDrain = func() { - assert.True(t, firstIndexBulkActive(t, probe.Store), "failure fixture must exercise an engaged bulk window") + idx := newFirstIndexBulkIndexer(t, store, "repo") + if mode == "seal panic" { + idx.indexMemoryAdmission = newIndexMemoryAdmissionBudget(defaultIndexMemoryAdmissionBytes) } - idx := newFirstIndexBulkIndexer(t, probe, "repo") dir := firstIndexFixtureRepo(t) + switch mode { case "panic in body": assert.PanicsWithValue(t, "injected body panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) - assert.Zero(t, probe.NodeCount(), "body panic must not drain shadow nodes") - assert.Zero(t, probe.EdgeCount(), "body panic must not drain shadow edges") + assert.Zero(t, store.NodeCount(), "body panic must not drain shadow nodes") + assert.Zero(t, store.EdgeCount(), "body panic must not drain shadow edges") case "panic in drain": require.PanicsWithValue(t, "injected drain panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + case "seal panic": + require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") + assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") + assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") + case "seal panic after a drain panic": + // The flush the drain panic's unwind runs panics too; it is + // retried the same way, and the later panic reaches the caller. + require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") + case "cancel before drain": + _, err := idx.IndexCtx(ctx, dir) + require.ErrorIs(t, err, context.Canceled) case "drain error": _, err := idx.IndexCtx(ctx, dir) - require.ErrorIs(t, err, injected) - default: + require.ErrorContains(t, err, "injected drain FTS error") + case "storage full": _, err := idx.IndexCtx(ctx, dir) - require.ErrorIs(t, err, context.Canceled) + var storageErr *store_sqlite.StorageError + require.ErrorAs(t, err, &storageErr) + assert.Contains(t, store_sqlite.SafeStorageFailureReason(err), "SQLite code 13") + assert.Equal(t, 1, batches, "a refused write must be the drain's last") + case "seal error": + _, err := idx.IndexCtx(ctx, dir) + require.ErrorContains(t, err, "nodes_by_kind") + // Either still owed or already rebuilt by the maintenance lane, + // which runs as soon as the window closes on a volume with room. + health, healthErr := store.PlannerStatsHealth(context.Background()) + require.NoError(t, healthErr) + if health.Reason == "dense_indexes_missing" { + assert.Equal(t, []string{"nodes_by_kind"}, health.DenseIndexesMissing) + } + deadline := time.Now().Add(10 * time.Second) + for !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } } + if mode == "cancel before drain" || mode == "panic in body" { - assert.Zero(t, probe.begins) - assert.Zero(t, probe.batches) + assert.Empty(t, evictions, "a failure before the drain must not evict") + assert.Zero(t, batches, "a failure before the drain must not write") } else { - require.Positive(t, probe.batches) - assert.Equal(t, 1, probe.begins) + assert.Equal(t, []bool{true}, evictions, "a SQLite target must take the shadow-replacement retirement") + require.Positive(t, batches, "test must observe the shadow drain") + assert.True(t, engaged, "failure fixture must exercise an engaged bulk window") } - assertFirstIndexRestored(t, probe.Store, db, indexes) - assertFirstIndexNoReceipts(t, probe.Store, "repo") - assert.Same(t, probe, idx.graph, "panic and error paths must restore disk target") + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store), "writer durability must be restored") + assertFirstIndexRestored(t, store, db, indexes) + assertFirstIndexNoReceipts(t, store, "repo") + assert.Same(t, store, idx.graph, "panic and error paths must restore disk target") }) } } -func TestFirstIndexFlushPanicPropagates(t *testing.T) { - // Given a FlushBulk implementation that panics while holding its writer lock. - probe, _ := newFirstIndexBulkFixture(t) - probe.panicFlush = true - idx := newFirstIndexBulkIndexer(t, probe, "repo") - idx.indexMemoryAdmission = newIndexMemoryAdmissionBudget(defaultIndexMemoryAdmissionBytes) - // The probe's lock is independent of Store; close the real bulk window at cleanup. - t.Cleanup(func() { require.NoError(t, probe.Store.FlushBulk()) }) - - // When the completed shadow drain flushes, it must not retry that locked call. - require.PanicsWithValue(t, "injected flush panic", func() { - _, _ = idx.IndexCtx(context.Background(), firstIndexFixtureRepo(t)) - }) - - // Then the panic propagates and the backstop does not re-enter FlushBulk. - // This fixture has no failure-ledger rows, so it does not cover a real - // store whose FlushBulk panics with writeMu held while the ledger is dirty. - assert.Equal(t, 1, probe.flushes) - assert.Same(t, probe, idx.graph) - assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") - assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") - assertFirstIndexNoReceipts(t, probe.Store, "repo") -} - type firstIndexDrainPanicReporter struct{} func (firstIndexDrainPanicReporter) Report(stage string, _, _ int) { @@ -327,12 +554,7 @@ func TestFirstIndexFailureRestoresSQLiteReplacement(t *testing.T) { writeFile(t, baseline, "old replacement baseline") ctx := context.Background() if mode == "drain error" { - writer, err := sql.Open("sqlite", "file:"+filepath.ToSlash(probe.Path())) - require.NoError(t, err) - _, err = writer.Exec(`CREATE TRIGGER fail_first_index_fts BEFORE INSERT ON symbol_fts_rowid -BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) - require.NoError(t, err) - require.NoError(t, writer.Close()) + firstIndexDrainFTSTrigger(t, probe.Store) } else { ctx = progress.WithReporter(ctx, firstIndexDrainPanicReporter{}) } @@ -356,3 +578,18 @@ BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) }) } } + +func TestFirstIndexDrainReportsWindowAndSeal(t *testing.T) { + store, _ := newFirstIndexStore(t) + core, observed := observer.New(zap.InfoLevel) + idx := newFirstIndexBulkIndexer(t, store, "repo") + idx.logger = zap.New(core) + result, err := idx.IndexCtx(t.Context(), firstIndexFixtureRepo(t)) + require.NoError(t, err) + entries := observed.FilterMessage("indexer: FlushBulk complete").All() + require.Len(t, entries, 1) + fields := entries[0].ContextMap() + assert.Equal(t, "cold", fields["window"]) + assert.Contains(t, fields, "seal_elapsed") + assert.EqualValues(t, result.NodeCount, fields["nodes"]) +} diff --git a/internal/indexer/go_module_extern_test.go b/internal/indexer/go_module_extern_test.go index e8546be5f..147ee509e 100644 --- a/internal/indexer/go_module_extern_test.go +++ b/internal/indexer/go_module_extern_test.go @@ -153,8 +153,17 @@ func Caller() { library.Hello(); child.Nested() } } case "parallel": errors := make(chan error, len(repos)) + // This is a module-binding test, not an identity-admission + // saturation test. The decoy case has three roots; retain + // parallel tracking within the two native observation slots. + admitted := make(chan struct{}, 2) for _, entry := range repos { - go func() { _, e := mi.TrackRepoCtx(context.Background(), entry); errors <- e }() + go func() { + admitted <- struct{}{} + defer func() { <-admitted }() + _, e := mi.TrackRepoCtx(context.Background(), entry) + errors <- e + }() } for range repos { require.NoError(t, <-errors) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 035ab698c..d4ce2cb8d 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3408,6 +3408,19 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index return } drainStart := time.Now() + var drainWALPeak int64 + var drainWALObserved bool + walReporter, hasWALReporter := diskTarget.(interface{ DBStats() (int64, int64) }) + sampleDrainWAL := func() { + if hasWALReporter { + _, walBytes := walReporter.DBStats() + if walBytes > 0 { + drainWALObserved = true + } + drainWALPeak = max(drainWALPeak, walBytes) + } + } + sampleDrainWAL() shadowNodeCount := inMemShadow.NodeCount() shadowEdgeCount := inMemShadow.EdgeCount() if !shadowEstimateReady { @@ -3438,11 +3451,16 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index restubGoModuleRepoIncoming(diskTarget, idx.RepoPrefix()) } var evictedNodes, evictedEdges int - if sqlite, ok := diskTarget.(*store_sqlite.Store); ok && sqlite.SupportsShadowReplacementRetirement() { + sqliteTarget, isSQLite := diskTarget.(*store_sqlite.Store) + retirement := isSQLite && sqliteTarget.SupportsShadowReplacementRetirement() + if shadowEvictionBranchHook != nil { + shadowEvictionBranchHook(retirement) + } + if retirement { var err error shadowReplacementStarted = true restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix()) - evictedNodes, evictedEdges, err = sqlite.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) + evictedNodes, evictedEdges, err = sqliteTarget.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) if err != nil { retErr = fmt.Errorf("indexer: retire shadow replacement rows: %w", err) return @@ -3461,17 +3479,34 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index return } bl.BeginBulkLoad() + bulkWindow := "unknown" + if reporter, ok := diskTarget.(interface{ BulkLoadWindow() string }); ok { + bulkWindow = reporter.BulkLoadWindow() + } // FlushBulk is a no-op under an outer coordinated window. Otherwise // it must also restore the standalone window if a drain write panics. bulkFlushPending := true bulkFlushInProgress := false defer func() { - // A FlushBulk panic may leave the store's writer lock held. - // Do not re-enter it; let the panic unwind the earlier defers. - if bulkFlushPending && !bulkFlushInProgress { - if err := bl.FlushBulk(); err != nil && retErr == nil { - retErr = fmt.Errorf("indexer: persist bulk graph: %w", err) - } + // A FlushBulk panicked: the explicit one below, or the one the + // next defer runs while a drain panic unwinds. The store releases + // its write gate on every exit and keeps a window whose seal did + // not finish retryable, so one more flush closes it (restoring + // durability, or recording the indexes it could not rebuild) + // instead of leaving a pinned synchronous=OFF writer behind. + if bulkFlushPending && bulkFlushInProgress { + retryBulkFlushAfterPanic(bl, idx.logger, idx.RepoPrefix()) + } + }() + defer func() { + if !bulkFlushPending || bulkFlushInProgress { + return + } + bulkFlushInProgress = true + err := bl.FlushBulk() + bulkFlushPending = false + if err != nil && retErr == nil { + retErr = fmt.Errorf("indexer: persist bulk graph: %w", err) } }() // Transfer the shadow in explicit row/byte-capped batches. The @@ -3506,6 +3541,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // The builtin sentinels the drain is about to move, kept aside for // the re-assert below. See restampedBuiltins. var restampedBuiltins []*graph.Node + nodeDrainStopped := false for nodes := range inMemShadow.DrainNodeBatches(persistChunkRows, persistChunkBytes) { for _, node := range nodes { if node != nil && graph.IsBuiltinStub(node.ID) { @@ -3513,9 +3549,17 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index restampedBuiltins = append(restampedBuiltins, &copied) } } - if err := drainAddBatch(ctx, diskTarget, nodes, nil); err != nil && retErr == nil { - retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", err) + if err := drainAddBatch(ctx, diskTarget, nodes, nil); err != nil { + // A refused write fails every later batch the same way, and + // a cooperative stop is final; either ends the drain, so the + // edge drain below writes nothing for this failed build. + if retErr == nil { + retErr = drainBatchError(ctx, err) + } + nodeDrainStopped = true + break } + sampleDrainWAL() if retErr != nil && drainChecksCancellation(ctx) && ctx.Err() != nil { break } @@ -3536,6 +3580,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index ftsReady = false return false } + sampleDrainWAL() ftsItemCount += len(ftsItems) ftsItems = make([]graph.SymbolFTSItem, 0, min(len(nodes), persistFTSChunkRows)) ftsBytes = 0 @@ -3568,6 +3613,9 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index drainPressure.afterNodeBatch(nodeRows) } for edges := range inMemShadow.DrainEdgeBatches(persistChunkRows, persistChunkBytes) { + if nodeDrainStopped { + break + } if drainChecksCancellation(ctx) && ctx.Err() != nil { if retErr == nil { retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", ctx.Err()) @@ -3576,10 +3624,11 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index } if err := drainAddBatch(ctx, diskTarget, nil, edges); err != nil { if retErr == nil { - retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", err) + retErr = drainBatchError(ctx, err) } break } + sampleDrainWAL() edgeRows := len(edges) drainPressure.afterEdgeBatch(edgeRows) } @@ -3605,10 +3654,17 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // row or no composed view can equal it. The set is one row per // distinct builtin the repository references, so the copy is // bounded by the language's builtin vocabulary, not by corpus size. - if len(restampedBuiltins) > 0 { - diskTarget.AddBatch(restampedBuiltins, nil) + // + // Only after a drain that is still succeeding: a failed one publishes + // nothing and is evicted before its retry, and after a refused write + // this one would only be refused again. + if len(restampedBuiltins) > 0 && retErr == nil { + if err := drainWrite(ctx, diskTarget, restampedBuiltins, nil); err != nil { + retErr = drainBatchError(ctx, err) + } } + sampleDrainWAL() flushStart := time.Now() idx.logger.Info("indexer: FlushBulk start", zap.String("repo", idx.RepoPrefix()), @@ -3621,7 +3677,26 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index if ferr != nil && retErr == nil { retErr = fmt.Errorf("indexer: persist bulk graph: %w", ferr) } + sealElapsed := time.Since(flushStart) + sampleDrainWAL() + drainElapsed := time.Since(drainStart) + if recorder, ok := diskTarget.(interface { + RecordFirstIndexDrain(string, graph.FirstIndexDrain) + }); ok { + observation := graph.FirstIndexDrain{ + Window: bulkWindow, Nodes: shadowNodeCount, Edges: shadowEdgeCount, + DrainMS: float64(drainElapsed) / float64(time.Millisecond), + SealMS: float64(sealElapsed) / float64(time.Millisecond), CompletedAt: time.Now(), + WALPeakBytes: drainWALPeak, WALPeakObserved: drainWALObserved, WALPeakScope: "shared_store_drain_chunk_samples", + } + if retErr != nil { + observation.Error = retErr.Error() + } + recorder.RecordFirstIndexDrain(idx.RepoPrefix(), observation) + } idx.logger.Info("indexer: FlushBulk complete", + zap.String("window", bulkWindow), + zap.Duration("seal_elapsed", sealElapsed), zap.String("repo", idx.RepoPrefix()), zap.Duration("flush_elapsed", time.Since(flushStart)), zap.Duration("total_drain", time.Since(drainStart)), diff --git a/internal/indexer/multi_cold_orchestration_test.go b/internal/indexer/multi_cold_orchestration_test.go index 71f0fa479..345ce554f 100644 --- a/internal/indexer/multi_cold_orchestration_test.go +++ b/internal/indexer/multi_cold_orchestration_test.go @@ -473,6 +473,36 @@ func TestIndexMultiRepoRefFactFailureStopsGlobalConsumers(t *testing.T) { "a failed reference-fact boundary must not publish global completion") } +// TestIndexMultiRepoFailedFinalSealGivesUpTheWindow: when the coordinated +// window's final seal keeps failing, the cold pass reports the failure and then +// gives the window up instead of leaving it open behind the error. The writer +// is back at its own durability, the planner probe no longer sees a live +// window, and the store's repair rebuilds the index the seal could not. +func TestIndexMultiRepoFailedFinalSealGivesUpTheWindow(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + injected := errors.New("injected final seal failure") + restore := store.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + mi := newColdOrchestrationMulti(store, zap.NewNop()) + + _, err := mi.indexFixtureRepos(coldRefFactRepos(t, 2)) + restore() + require.ErrorIs(t, err, injected) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store), "writer durability must be restored") + assert.False(t, firstIndexBulkActive(t, store), "the failed window must not stay open") + deadline := time.Now().Add(10 * time.Second) + for !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + assert.Equal(t, indexes, firstIndexSchemaIndexes(t, db), "the repair must rebuild the index the seal left out") +} + // TestBeginDeferredPassesOverlapSplitsPoolFromTail pins the overlap contract: // BeginDeferredPasses drains the enrichment pool on its own goroutine while // the caller is free to run the resolve phase; contracts and the catch-up diff --git a/internal/indexer/multi_fixture_test.go b/internal/indexer/multi_fixture_test.go index 2b21c49cf..66b3c847d 100644 --- a/internal/indexer/multi_fixture_test.go +++ b/internal/indexer/multi_fixture_test.go @@ -272,7 +272,7 @@ func (mi *MultiIndexer) indexFixtureRepos(repos []config.RepoEntry) (map[string] coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() defer func() { if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil { mi.logger.Error("multi-repo bulk-load cleanup failed", zap.Error(err)) } } @@ -364,7 +364,7 @@ func (mi *MultiIndexer) indexFixtureRepos(repos []config.RepoEntry) (map[string] old.Close() } if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil { return nil, fmt.Errorf("multi-repo bulk-load finalize: %w", err) } coordinatedBulkActive = false diff --git a/internal/indexer/prefix_observation_publication_test.go b/internal/indexer/prefix_observation_publication_test.go new file mode 100644 index 000000000..26df8a0ec --- /dev/null +++ b/internal/indexer/prefix_observation_publication_test.go @@ -0,0 +1,71 @@ +package indexer + +import ( + "context" + "os" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" +) + +// Publication must mean the native worker has returned its allowance. A caller +// often starts another observation immediately; announcing completion before +// permit return can spuriously exhaust both slots under parallel tracking. +func TestIdentityObservationPublishesOnlyAfterPermitReturn(t *testing.T) { + root := t.TempDir() + for _, operation := range []string{"stat", "naming"} { + t.Run(operation, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var allowance prefixStatAllowance + allowance.once.Do(func() { allowance.slots = make(chan struct{}, 2) }) + nativeDone := make(chan struct{}) + // Temporarily intercept this worker's token so its release must + // wait for the test. This makes publication ordering observable + // without scheduler timing, polling or an extra production hook. + finishNative := func() { + <-allowance.slots + close(nativeDone) + } + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + result := make(chan error, 1) + if operation == "stat" { + allowance.stat = func(path string) (os.FileInfo, error) { + info, err := os.Stat(path) + finishNative() + return info, err + } + go func() { result <- allowance.observe(ctx, root).err }() + } else { + go func() { + _, err := allowance.observeNaming(ctx, root, func(context.Context, string) trackNamingFacts { + finishNative() + return trackNamingFacts{canonical: root} + }) + result <- err + }() + } + <-nativeDone + synctest.Wait() + var publishedEarly bool + var err error + select { + case err = <-result: + publishedEarly = true + default: + } + // Always let the worker finish before assertions, including RED. + allowance.slots <- struct{}{} + synctest.Wait() + if !publishedEarly { + err = <-result + } + require.NoError(t, err) + require.Zero(t, len(allowance.slots)) + require.False(t, publishedEarly, "completed identity result escaped before its permit was returned") + }) + }) + } +} diff --git a/internal/indexer/prefix_ownership.go b/internal/indexer/prefix_ownership.go index a8da85371..75beaf395 100644 --- a/internal/indexer/prefix_ownership.go +++ b/internal/indexer/prefix_ownership.go @@ -60,9 +60,14 @@ func (a *prefixStatAllowance) observe(ctx context.Context, path string) prefixSt slots := a.slots done := make(chan prefixStatResult, 1) go func() { - defer func() { <-slots }() - info, err := stat(path) - done <- prefixStatResult{info: info, err: err} + result := func() prefixStatResult { + defer func() { <-slots }() + info, err := stat(path) + return prefixStatResult{info: info, err: err} + }() + // A completed result authorizes the caller's next observation. + // Return this permit first, even when the receiver runs immediately. + done <- result }() select { case <-ctx.Done(): diff --git a/internal/indexer/track_naming.go b/internal/indexer/track_naming.go index 3505de97f..72b9dfb9b 100644 --- a/internal/indexer/track_naming.go +++ b/internal/indexer/track_naming.go @@ -86,8 +86,12 @@ func (a *prefixStatAllowance) observeNaming(ctx context.Context, root string, re workerCtx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() go func() { - defer func() { <-slots }() - done <- read(workerCtx, root) + facts := func() trackNamingFacts { + defer func() { <-slots }() + return read(workerCtx, root) + }() + // Completion includes permit return, as for root Stat observations. + done <- facts }() select { case <-ctx.Done(): diff --git a/internal/mcp/change_contract.go b/internal/mcp/change_contract.go index 81d47c99e..325397fba 100644 --- a/internal/mcp/change_contract.go +++ b/internal/mcp/change_contract.go @@ -897,6 +897,9 @@ func (s *Server) handleChangeContract(ctx context.Context, req mcp.CallToolReque if s == nil || s.graph == nil { return mcp.NewToolResultError("change_contract: server not fully initialised"), nil } + // Snapshot the unpublished edits before the walk; the changed symbols + // that scope the label are known only once it has run. + gaps := s.laggingMutationReceipts(ctx) p, err := s.lowerChange(ctx, req) if err != nil { if ctxErr := requestContextError(ctx, err); ctxErr != nil { @@ -909,5 +912,7 @@ func (s *Server) handleChangeContract(ctx context.Context, req mcp.CallToolReque } violations := s.evaluateChange(ctx, p) env := s.assembleEnvelope(ctx, p, violations) - return s.respondJSONOrTOON(ctx, req, env) + label := s.labelRelations(ctx, gaps, p.changedIDs, nil) + res, err := s.respondJSONOrTOON(ctx, req, label.labelled(env)) + return label.decorate(res), err } diff --git a/internal/mcp/checkout_binding.go b/internal/mcp/checkout_binding.go index baa0fcfd9..9e88a9aa8 100644 --- a/internal/mcp/checkout_binding.go +++ b/internal/mcp/checkout_binding.go @@ -759,6 +759,11 @@ func exactnessWithdrawnRefusal(reason string) error { // session scope and request view selection. A cache miss observes only this // checkout of an already known Git family; it never tracks a new repository or // waits for parsing. Existing identities take the catalog-only fast path. +// +// A path inside a registered root, its subdirectories included, is answered +// by that root and is never refused for a discovery pending elsewhere. The one +// exception is a path inside a nested worktree that discovery has not +// registered yet: only such a path is observed, so only it can be refused. func (s *Server) checkoutForRequestPath(ctx context.Context, path string) (store_sqlite.Checkout, bool, error) { checkout, found, err := s.registeredCheckoutForPath(ctx, path) if err != nil { diff --git a/internal/mcp/contract_core_capability_table_test.go b/internal/mcp/contract_core_capability_table_test.go index cdd3d5f34..798bfd7ee 100644 --- a/internal/mcp/contract_core_capability_table_test.go +++ b/internal/mcp/contract_core_capability_table_test.go @@ -118,6 +118,9 @@ type ( capBFSContext interface { BFSContext(context.Context, []string, graph.Direction, []graph.EdgeKind, int, int) ([]graph.BFSHop, error) } + capLastFirstIndexDrains interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + } capRepoPrefixes interface{ RepoPrefixes() []string } ) @@ -218,6 +221,7 @@ func contractCoreCapabilityTable() []contractCoreCapRow { {"NodeLightSequencer", capOf[graph.NodeLightSequencer], capDropped, "node-only streaming; graph.NodesLightSeq falls back to the AllNodesLight forwarder, the same rows materialised"}, {"Store", capOf[graph.Store], capDropped, "the whole backend contract; a request reader is never the store"}, {"local:WALReclaimStats", capOf[walReclaimSnapshotReader], capDropped, capReasonBackend + " (physical store counters, never selected graph state)"}, + {"local:LastFirstIndexDrains", capOf[capLastFirstIndexDrains], capDropped, capReasonBackend + " (shared-store drain observations, never selected graph state)"}, {"AnalysisGenerationStore", capOf[graph.AnalysisGenerationStore], capDropped, capReasonBackend}, {"AnalysisQueryStore", capOf[graph.AnalysisQueryStore], capDropped, capReasonBackend}, {"TryAnalysisQueryStore", capOf[graph.TryAnalysisQueryStore], capDropped, capReasonBackend}, diff --git a/internal/mcp/gcx.go b/internal/mcp/gcx.go index ef2b16ae0..2a4caddb3 100644 --- a/internal/mcp/gcx.go +++ b/internal/mcp/gcx.go @@ -1937,8 +1937,13 @@ func encodeChangeImpact(result map[string]any) ([]byte, error) { communityNote, _ := result["community_note"].(string) contractRiskUpgrade, _ := result["contract_risk_upgrade"].(string) + var meta []string + if waited, ok := result["barrier_wait_ms"].(int64); ok { + meta = append(meta, "barrier_wait_ms", strconv.FormatInt(waited, 10)) + } sumEnc := newGCX(&buf, "explain_change_impact.summary", []string{"risk", "summary", "total_affected", "cross_repo", "processes", "communities", "test_files", "warning", "note", "risk_upgrade"}, + meta..., ) if err := sumEnc.WriteRow( string(risk), diff --git a/internal/mcp/index_health_planner_stats_test.go b/internal/mcp/index_health_planner_stats_test.go index b89263823..7ce1553bf 100644 --- a/internal/mcp/index_health_planner_stats_test.go +++ b/internal/mcp/index_health_planner_stats_test.go @@ -199,6 +199,30 @@ func TestIndexHealth_SurfacesPlannerStats(t *testing.T) { "published a receiver cardinality for an index that is not in the schema") }) + // Dense indexes an interrupted bulk load left out are named, with the + // sentence that says what they cost and how they come back. The real + // state repairs itself on the store's maintenance lane as soon as the + // volume has room, so the verdict is fixed here rather than raced. + t.Run("missing dense indexes are named", func(t *testing.T) { + _, store := newEmptySQLiteBackedServer(t) + missing := []string{"nodes_by_kind", "edges_by_kind"} + wrapped := plannerHealthOverride{Store: store, health: graph.PlannerStatsFreshness{ + Reason: "dense_indexes_missing", DenseIndexesMissing: missing, + }} + idx := indexer.New(wrapped, testRegistry(), config.Default().Index, zap.NewNop()) + missingSrv := NewServer(query.NewEngine(wrapped), wrapped, idx, nil, zap.NewNop(), nil) + + payload, err := missingSrv.buildIndexHealthPayloadCtx(context.Background()) + require.NoError(t, err) + stats, ok := payload["planner_stats"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dense_indexes_missing", stats["reason"]) + assert.Equal(t, missing, stats["dense_indexes_missing"]) + recommendation, _ := payload["recommendation"].(string) + assert.Contains(t, recommendation, "nodes_by_kind, edges_by_kind") + assert.NotContains(t, recommendation, "planner statistics are stale") + }) + // The window between a deferred cooperative pass and the boundary that // finishes it, seen from the side an agent reads. // @@ -304,6 +328,16 @@ func TestIndexHealth_SurfacesPlannerStats(t *testing.T) { }) } +// plannerHealthOverride reports a fixed planner verdict over a real store. +type plannerHealthOverride struct { + *store_sqlite.Store + health graph.PlannerStatsFreshness +} + +func (s plannerHealthOverride) PlannerStatsHealth(context.Context) (graph.PlannerStatsFreshness, error) { + return s.health, nil +} + // A backend without the capability must omit the key rather than publish a // struct of zeros, which would read as "the planner believes nothing". func TestIndexHealth_PlannerStatsAbsentOnBackendWithoutCapability(t *testing.T) { @@ -314,3 +348,26 @@ func TestIndexHealth_PlannerStatsAbsentOnBackendWithoutCapability(t *testing.T) assert.NotContains(t, payload, "planner_stats", "the in-memory graph has no planner statistics; reporting zeros would invent a defect") } + +func TestIndexHealthReportsLastFirstIndexDrain(t *testing.T) { + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_SHADOW_MAX_BYTES", "1073741824") + srv, _ := newSQLiteBackedServer(t) + payload, err := srv.buildIndexHealthPayloadCtx(t.Context()) + require.NoError(t, err) + observations, ok := payload["last_first_index_drain"].(map[string]graph.FirstIndexDrain) + require.True(t, ok, "a completed first index must publish its actual drain observation") + require.Len(t, observations, 1) + for _, observation := range observations { + assert.Equal(t, "cold", observation.Window) + assert.Greater(t, observation.Nodes, 0) + assert.GreaterOrEqual(t, observation.DrainMS, observation.SealMS) + assert.False(t, observation.CompletedAt.IsZero()) + assert.Equal(t, "shared_store_drain_chunk_samples", observation.WALPeakScope) + } + empty, _ := newEmptySQLiteBackedServer(t) + noDrain, err := empty.buildIndexHealthPayloadCtx(t.Context()) + require.NoError(t, err) + assert.NotContains(t, noDrain, "last_first_index_drain", "absence is not a fabricated completed zero drain") +} diff --git a/internal/mcp/mutation_freshness_hints_test.go b/internal/mcp/mutation_freshness_hints_test.go index eb5f96d73..9e1cd9ffb 100644 --- a/internal/mcp/mutation_freshness_hints_test.go +++ b/internal/mcp/mutation_freshness_hints_test.go @@ -105,7 +105,7 @@ func TestFreshnessRefusalAnnotationBoundsAndScope(t *testing.T) { // Exercise the public call shapes against real handlers and an owned working // tree. The existing watcher seam supplies a failed publication result after // the edit acknowledges; this does not inject a real store patch failure. -func TestFreshnessRefusalRecoveryFlow(t *testing.T) { +func TestRelationFreshnessRecoveryFlow(t *testing.T) { s, root := setupTestServer(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -144,14 +144,10 @@ func TestFreshnessRefusalRecoveryFlow(t *testing.T) { waitCheckoutReceipt(t, s, id) detectArgs := map[string]any{"operation": "detect", "options": map[string]any{"scope": "unstaged"}} - refused := call("change", detectArgs) - require.True(t, refused.IsError) - for _, want := range []string{ - "graph freshness unavailable", "change.receipt options.receipt=" + id, - "workspace_admin operation=reindex", "paths=[P]", `mutation_id="recovery-flow"`, - } { - require.Contains(t, resultText(refused), want) - } + partial := resultJSON(t, call("change", detectArgs)) + require.Equal(t, false, partial["complete"]) + require.Equal(t, "failed", partial["graph_freshness"]) + require.NotEmpty(t, partial["recovery_hint"]) status := resultJSON(t, call("change", map[string]any{ "operation": "receipt", "options": map[string]any{"mutation_id": id}, })) diff --git a/internal/mcp/mutation_freshness_test.go b/internal/mcp/mutation_freshness_test.go index d9dd4b21c..011c5e5fa 100644 --- a/internal/mcp/mutation_freshness_test.go +++ b/internal/mcp/mutation_freshness_test.go @@ -434,41 +434,44 @@ func TestMutationReposForSymbolIDsUnresolvedWidensBarrier(t *testing.T) { } } -func TestChangeImpactFreshnessGuardAndRetry(t *testing.T) { +func TestChangeImpactLabelsLaggingReceiptsInScope(t *testing.T) { g := graph.New() g.AddNode(&graph.Node{ ID: "repo-a::target", Kind: graph.KindFunction, Name: "target", - FilePath: "target.go", + FilePath: "/repo-a/target.go", StartLine: 1, EndLine: 1, Language: "go", RepoPrefix: "repo-a", }) s := &Server{ - graph: g, - engine: query.NewEngine(g), - session: newSessionState(), - mutationSafetyWait: time.Millisecond, + graph: g, + engine: query.NewEngine(g), + session: newSessionState(), } receipt := pendingFreshnessReceipt(s, "receipt-impact", "repo-a", "/repo-a/target.go", 11) req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{ Name: "explain_change_impact", Arguments: map[string]any{"ids": "repo-a::target", "format": "json"}, }} - - result, err := s.handleEnhancedChangeImpact(context.Background(), req) - if err != nil { - t.Fatalf("pending impact handler error: %v", err) + impact := func() map[string]any { + t.Helper() + result, err := s.handleEnhancedChangeImpact(context.Background(), req) + if err != nil { + t.Fatalf("impact handler error: %v", err) + } + return lifecycleResultPayload(t, result) } - if result == nil || !result.IsError { - t.Fatalf("pending impact result = %#v, want MCP error", result) + + // A pending edit to the symbol's own file is labelled, never refused. + payload := impact() + if payload["graph_freshness"] != "pending" || payload["complete"] != false { + t.Fatalf("pending impact = %v, want a pending partial", payload) } - for _, want := range []string{"change impact refused a stale graph", "receipt-impact", "generation=11"} { - if !strings.Contains(freshnessToolResultText(result), want) { - t.Fatalf("pending impact response %q does not contain %q", freshnessToolResultText(result), want) - } + if files, _ := payload["pending_files"].([]any); len(files) != 1 || files[0] != "/repo-a/target.go" { + t.Fatalf("pending impact files = %v", payload["pending_files"]) } completeFreshnessReceipt(receipt, indexer.MutationResult{ @@ -476,22 +479,31 @@ func TestChangeImpactFreshnessGuardAndRetry(t *testing.T) { AppliedGeneration: 11, Reindexed: true, }) - result, err = s.handleEnhancedChangeImpact(context.Background(), req) - if err != nil { - t.Fatalf("completed impact handler error: %v", err) + if payload = impact(); payload["graph_freshness"] != nil { + t.Fatalf("published edit still labelled: %v", payload) + } + + // A terminally failed edit in scope is a labelled partial with a + // recovery hint, not a tool error. + failed := pendingFreshnessReceipt(s, "receipt-failed", "repo-a", "/repo-a/target.go", 12) + completeFreshnessReceipt(failed, indexer.MutationResult{ + RequestedGeneration: 12, + AppliedGeneration: 12, + Err: errors.New("syntax patch failed"), + }) + payload = impact() + if payload["graph_freshness"] != "failed" || payload["complete"] != false { + t.Fatalf("failed impact = %v, want a failed partial", payload) } - if result == nil || result.IsError { - t.Fatalf("completed impact remained blocked: %q", freshnessToolResultText(result)) + if hint, _ := payload["recovery_hint"].(string); !strings.Contains(hint, "reindex") { + t.Fatalf("failed impact recovery hint = %q", hint) } } -func TestDetectChangesFreshnessGuard(t *testing.T) { - s := &Server{ - graph: graph.New(), - session: newSessionState(), - mutationSafetyWait: time.Millisecond, - } - pendingFreshnessReceipt(s, "receipt-detect", "repo-a", "/repo-a/changed.go", 12) +func TestDetectChangesLabelsPendingChangedFile(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-detect", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 12) req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{ Name: "detect_changes", Arguments: map[string]any{"format": "json"}, @@ -501,12 +513,54 @@ func TestDetectChangesFreshnessGuard(t *testing.T) { if err != nil { t.Fatalf("detect handler error: %v", err) } - if result == nil || !result.IsError { - t.Fatalf("pending detect result = %#v, want MCP error", result) + payload := lifecycleResultPayload(t, result) + for key, want := range map[string]any{ + "complete": false, + "graph_freshness": "pending", + "pending_file_count": float64(1), + } { + if payload[key] != want { + t.Fatalf("detect %s = %v, want %v: %v", key, payload[key], want, payload) + } + } + if files, _ := payload["changed_files"].([]any); len(files) != 1 || files[0] != "caller.go" { + t.Fatalf("detect changed_files = %v", payload["changed_files"]) + } + // The walk still ran: the changed symbol is mapped and risk is scored. + if symbols, _ := payload["changed_symbols"].([]any); len(symbols) == 0 || payload["risk"] == "UNKNOWN" { + t.Fatalf("labelled detect skipped the walk: %v", payload) } - for _, want := range []string{"change detection refused a stale graph", "receipt-detect", "generation=12"} { - if !strings.Contains(freshnessToolResultText(result), want) { - t.Fatalf("pending detect response %q does not contain %q", freshnessToolResultText(result), want) + + // The Stop hook bounds the wait with an earlier wait_deadline, which can + // pass between admission and the handler reading it. Either way the + // answer is labelled without sitting out the default budget. + for _, offset := range []time.Duration{50 * time.Millisecond, -time.Second} { + req.Params.Arguments = map[string]any{"format": "json", + "wait_deadline": time.Now().Add(offset).Format(time.RFC3339Nano)} + result, err = s.handleDetectChanges(context.Background(), req) + if err != nil { + t.Fatalf("detect handler error: %v", err) + } + payload = lifecycleResultPayload(t, result) + if waited, _ := payload["barrier_wait_ms"].(float64); payload["graph_freshness"] != "pending" || waited >= 200 { + t.Fatalf("detect with wait_deadline %v: graph_freshness %v after %v ms", offset, payload["graph_freshness"], waited) } } + + // A staged diff leaves the pending caller.go out of the changed files, + // but the walk still reads it as a caller of the changed Target. + writeUnindexed(t, root, "target.go", "package p\n\nfunc Target() { _ = 0 }\n") + checkoutMutationGit(t, root, "add", "target.go") + req.Params.Arguments = map[string]any{"format": "json", "scope": "staged"} + result, err = s.handleDetectChanges(context.Background(), req) + if err != nil { + t.Fatalf("detect handler error: %v", err) + } + payload = lifecycleResultPayload(t, result) + if files, _ := payload["changed_files"].([]any); len(files) != 1 || files[0] != "target.go" { + t.Fatalf("staged detect changed_files = %v", payload["changed_files"]) + } + if pending, _ := payload["pending_files"].([]any); payload["graph_freshness"] != "pending" || len(pending) != 1 || pending[0] != "caller.go" { + t.Fatalf("staged detect left the pending caller unlabelled: %v", payload) + } } diff --git a/internal/mcp/refs_retry_test.go b/internal/mcp/refs_retry_test.go new file mode 100644 index 000000000..0dac0d8f1 --- /dev/null +++ b/internal/mcp/refs_retry_test.go @@ -0,0 +1,329 @@ +package mcp + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/lspuri" + "github.com/zzet/gortex/internal/semantic" + "github.com/zzet/gortex/internal/semantic/lsp" +) + +// Exercise the real provider and output authority without an installed language +// server. The first hierarchy response models an indexing server's empty answer. +func newReferenceRetryStack(t *testing.T, first string, beforeIncoming func()) (*viewStack, *graph.Node, *atomic.Int32, *atomic.Int32) { + t.Helper() + stack := newViewStack(t) + source := "fn target() {}\nfn caller() { target(); }\n" + path := filepath.Join(stack.repoRoot, "retry.rs") + if err := os.WriteFile(path, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + // A Windows drive must be in the URI path, never its authority. + uri := lspuri.PathToURI(path) + if got := lspuri.URIToAbsPath(uri); got != path { + t.Fatalf("fixture URI %q resolves to %q, want %q", uri, got, path) + } + target := viewRepoNode("repo/retry.rs::target", "target", graph.KindFunction, "repo/retry.rs", 1) + caller := viewRepoNode("repo/retry.rs::caller", "caller", graph.KindFunction, "repo/retry.rs", 2) + target.Language, caller.Language = "rust", "rust" + stack.store.AddBatch([]*graph.Node{target, caller}, []*graph.Edge{{ + From: caller.ID, To: target.ID, Kind: graph.EdgeCalls, + FilePath: caller.FilePath, Line: 2, Origin: graph.OriginASTResolved, + }}) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + calls, connections := &atomic.Int32{}, &atomic.Int32{} + done := make(chan struct{}) + var active net.Conn + accepted := make(chan struct{}) + go func() { + defer close(done) + conn, err := ln.Accept() + if err != nil { + close(accepted) + return + } + active = conn + connections.Add(1) + close(accepted) + defer conn.Close() + reader := bufio.NewReader(conn) + for { + size := 0 + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + if line == "\r\n" { + break + } + if strings.HasPrefix(line, "Content-Length:") { + size, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Content-Length:"))) + } + } + body := make([]byte, size) + if _, err := io.ReadFull(reader, body); err != nil { + return + } + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + } + if json.Unmarshal(body, &req) != nil { + return + } + if len(req.ID) == 0 { + continue + } + var result any + var rpcError any + switch req.Method { + case "initialize": + result = map[string]any{"capabilities": map[string]any{"callHierarchyProvider": true}} + case "textDocument/prepareCallHierarchy": + attempt := calls.Add(1) + if first == "error" && attempt == 1 { + rpcError = map[string]any{"code": -32603, "message": "workspace loading"} + } + if first != "zero" && (attempt > 1 || first == "positive" || first == "incoming-error") { + result = []lsp.CallHierarchyItem{{Name: "target", Kind: 12, URI: uri, Range: lsp.Range{Start: lsp.Position{Line: 0}, End: lsp.Position{Line: 0, Character: 14}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 0, Character: 3}, End: lsp.Position{Line: 0, Character: 9}}}} + } else { + result = []any{} + } + case "callHierarchy/incomingCalls": + if first == "incoming-error" && calls.Load() == 1 { + rpcError = map[string]any{"code": -32603, "message": "workspace loading"} + } + if beforeIncoming != nil { + beforeIncoming() + } + result = []lsp.CallHierarchyIncomingCall{{From: lsp.CallHierarchyItem{Name: "caller", Kind: 12, URI: uri, Range: lsp.Range{Start: lsp.Position{Line: 1}, End: lsp.Position{Line: 1, Character: 25}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 1, Character: 3}, End: lsp.Position{Line: 1, Character: 9}}}, FromRanges: []lsp.Range{{Start: lsp.Position{Line: 1, Character: 14}, End: lsp.Position{Line: 1, Character: 20}}}}} + } + reply := map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result} + if rpcError != nil { + delete(reply, "result") + reply["error"] = rpcError + } + response, _ := json.Marshal(reply) + if _, err := fmt.Fprintf(conn, "Content-Length: %d\r\n\r\n%s", len(response), response); err != nil { + return + } + } + }() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + provider := lsp.NewProviderFromSpec(&lsp.ServerSpec{ + Name: "reference-fixture", Command: executable, Languages: []string{"rust"}, + Daemon: true, NoHeavyRequests: first == "optout", + Connect: &lsp.ConnectSpec{Network: "tcp", Address: ln.Addr().String()}, + }, zap.NewNop()) + stack.srv.semanticMgr = semantic.NewManager(semantic.Config{}, zap.NewNop()) + stack.srv.semanticMgr.RegisterProvider(provider) + t.Cleanup(func() { + _ = provider.Close() + _ = ln.Close() + <-accepted + if active != nil { + _ = active.Close() + } + <-done + }) + return stack, target, calls, connections +} + +func TestColdReferenceConfirmationIsNotPermanent(t *testing.T) { + stack, node, calls, _ := newReferenceRetryStack(t, "empty", nil) + stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) + if calls.Load() != 1 { + t.Fatalf("hierarchy requests = %d, want 1", calls.Load()) + } + if entry, ok := stack.srv.refsConfirmed.Load(node.ID); ok { + if entry.(*referenceConfirmation).confirmed { + t.Fatal("cold empty response permanently confirmed the symbol") + } + } +} + +func TestReferenceConfirmationRecoversAfterCooldown(t *testing.T) { + for _, first := range []string{"empty", "error", "incoming-error", "positive", "zero"} { + t.Run(first, func(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, first, nil) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), node, now) + for range 3 { + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown-time.Nanosecond)) + } + if calls.Load() != 1 { + t.Fatalf("cooldown requests = %d, want 1", calls.Load()) + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + want := int32(2) + if first == "positive" { + want = 1 + } + if calls.Load() != want { + t.Fatalf("post-cooldown requests = %d, want %d", calls.Load(), want) + } + if first != "zero" { + edges := stack.store.GetInEdges(node.ID) + found := false + for _, edge := range edges { + if edge.From == "repo/retry.rs::caller" && edge.Kind == graph.EdgeCalls && edge.Origin == graph.OriginLSPResolved { + found = true + } + } + if !found { + t.Fatalf("retry did not promote caller edge: %+v", edges) + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(2*referenceRetryCooldown)) + if calls.Load() != want { + t.Fatal("positive confirmation was not cached") + } + } + if first == "zero" { + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(2*referenceRetryCooldown)) + if calls.Load() != 3 { + t.Fatal("valid zero callers lost eventual retry eligibility") + } + } + if connections.Load() != 1 { + t.Fatalf("provider connections = %d, want 1", connections.Load()) + } + }) + } +} + +func TestReferenceConfirmationClaimBoundsConcurrentQueries(t *testing.T) { + stack, node, calls, _ := newReferenceRetryStack(t, "empty", nil) + now := time.Now() + for _, stamp := range []time.Time{now, now.Add(referenceRetryCooldown)} { + var queries sync.WaitGroup + for range 16 { + queries.Go(func() { stack.srv.confirmSymbolRefsAt(context.Background(), node, stamp) }) + } + queries.Wait() + } + if calls.Load() != 2 { + t.Fatalf("concurrent queries made %d hierarchy requests, want 2", calls.Load()) + } +} + +func TestReferenceConfirmationRefusedCompletionRetries(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + stack, node, calls, _ := newReferenceRetryStack(t, "positive", func() { once.Do(func() { close(started); <-release }) }) + now := time.Now() + done := make(chan struct{}) + go func() { defer close(done); stack.srv.confirmSymbolRefsAt(context.Background(), node, now) }() + <-started + out, err := BeginBaseEnrichment(context.Background(), stack.srv.outputGenerationAuthority(), stack.srv.graph, EnrichProducerSymbolRefs, node.RepoPrefix, stack.repoRoot) + close(release) + <-done + if err != nil { + t.Fatal(err) + } + out.Abandon() + entry, _ := stack.srv.refsConfirmed.Load(node.ID) + if entry.(*referenceConfirmation).confirmed { + t.Fatal("superseded output was permanently confirmed") + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + if calls.Load() != 2 { + t.Fatalf("refused completion made %d requests, want 2", calls.Load()) + } +} + +func TestReferenceConfirmationResetCannotReviveInflightEntry(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + stack, node, _, _ := newReferenceRetryStack(t, "positive", func() { close(started); <-release }) + done := make(chan struct{}) + go func() { defer close(done); stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) }() + <-started + stack.srv.resetConfirmedRefs() + // A new epoch's entry must survive the previous attempt finishing. + replacement := stack.srv.claimReferenceConfirmation(node.ID, time.Now()) + close(release) + <-done + if entry, ok := stack.srv.refsConfirmed.Load(node.ID); !ok || entry != replacement { + t.Fatal("old completion replaced the new epoch's claim") + } +} + +func TestReferenceConfirmationClaimsAreIndependent(t *testing.T) { + srv := &Server{} + now := time.Now() + first := srv.claimReferenceConfirmation("first", now) + if first == nil || srv.claimReferenceConfirmation("second", now) == nil { + t.Fatal("different symbol could not claim") + } + if srv.claimReferenceConfirmation("first", now) != nil { + t.Fatal("active symbol claimed twice") + } + srv.resetConfirmedRefs() + if srv.refsConfirmed.CompareAndSwap("first", first, &referenceConfirmation{confirmed: true}) { + t.Fatal("old claim revived after reset") + } + if srv.claimReferenceConfirmation("first", now) == nil { + t.Fatal("reset did not allow a new attempt") + } +} + +func TestReferenceConfirmationRoutedRefusalLeavesPrimaryEligible(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, "empty", nil) + stack.srv.confirmSymbolRefsOnDemand(routedEnrichmentCtx(t, stack), node) + if calls.Load() != 0 || connections.Load() != 0 { + t.Fatal("routed refusal reached the language server") + } + if _, ok := stack.srv.refsConfirmed.Load(node.ID); ok { + t.Fatal("routed refusal retained a primary claim") + } + stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) + if calls.Load() != 1 { + t.Fatalf("primary request made %d calls, want 1", calls.Load()) + } +} + +func TestReferenceConfirmationHonorsProviderOptOut(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, "optout", nil) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), node, now) + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + if calls.Load() != 0 { + t.Fatalf("opted-out provider made %d hierarchy requests", calls.Load()) + } + if connections.Load() != 1 { + t.Fatalf("opted-out provider connections = %d, want 1", connections.Load()) + } +} + +func TestReferenceConfirmationProviderFailureBacksOff(t *testing.T) { + stack := newLSPEnrichmentStack(t) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), stack.node, now) + standing := stack.standingReceipt(t, EnrichProducerSymbolRefs) + stack.srv.confirmSymbolRefsAt(context.Background(), stack.node, now.Add(referenceRetryCooldown-time.Nanosecond)) + if err := standing.Complete(); err != nil { + t.Fatalf("provider failure retried inside cooldown: %v", err) + } +} diff --git a/internal/mcp/relation_freshness.go b/internal/mcp/relation_freshness.go new file mode 100644 index 000000000..b586cf68b --- /dev/null +++ b/internal/mcp/relation_freshness.go @@ -0,0 +1,376 @@ +package mcp + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "time" + + wire "github.com/gortexhq/gcx-go" + "github.com/mark3labs/mcp-go/mcp" + + "github.com/zzet/gortex/internal/graph" +) + +// relationFreshnessWait bounds how long a change tool waits for the edits in +// its relation scope before it answers with a label instead. +const relationFreshnessWait = 250 * time.Millisecond + +// maxLabelledPendingFiles caps the pending_files list of a label; +// pending_file_count always carries the full count. +const maxLabelledPendingFiles = 16 + +// relationFreshness labels an answer read from graph relations while edits in +// its scope are still being published. The zero value is a fresh scope. +type relationFreshness struct { + state string // "", "pending" or "failed" + files []string // graph-keyed paths of the lagging in-scope edits, sorted + waited time.Duration +} + +// relationWaitDeadline is when a change tool stops waiting for its relation +// scope: relationFreshnessWait from now, or an earlier wait_deadline. With +// require_fresh it is the request's own freshness deadline instead. The +// middleware has already refused a malformed or past wait_deadline, so one +// the handler cannot take passed after admission and ends the wait now. +func relationWaitDeadline(ctx context.Context, freshness requestFreshness) time.Time { + now := time.Now() + if freshness.err != nil { + return now + } + if freshness.requireFresh { + return freshness.effectiveDeadline(now, ctx) + } + deadline := now.Add(relationFreshnessWait) + if freshness.hasDeadline && freshness.deadline.Before(deadline) { + return freshness.deadline + } + return deadline +} + +// awaitRelationFreshness waits until deadline for the edits in the relation +// scope of ids and files, then labels whatever in that scope is still pending +// or failed. A deadline already passed labels without waiting. Edits outside +// the scope are neither waited on nor reported. +func (s *Server) awaitRelationFreshness(ctx context.Context, ids, files []string, deadline time.Time) relationFreshness { + started := time.Now() + lagging := s.laggingMutationReceipts(ctx) + if len(lagging) == 0 { + return relationFreshness{waited: time.Since(started)} + } + inScope := s.relationScope(ctx, ids, files) + if remaining := time.Until(deadline); remaining > 0 { + timer := time.NewTimer(remaining) + defer timer.Stop() + wait: + for _, receipt := range lagging { + if !inScope(receipt.path) { + continue + } + select { + case <-receipt.done: + case <-timer.C: + break wait + case <-ctx.Done(): + break wait + } + } + lagging = s.laggingMutationReceipts(ctx) + } + label := s.relationLabel(lagging, inScope) + label.waited = time.Since(started) + return label +} + +// labelRelations labels an answer that read relations without waiting. gaps +// is the laggingMutationReceipts snapshot taken before the walk; the scope of +// ids and files, known only once the walk has run, filters it afterwards. An +// edit that published during the walk still labels the answer, which may +// have read around it. +func (s *Server) labelRelations(ctx context.Context, gaps []*mutationReceipt, ids, files []string) relationFreshness { + if len(gaps) == 0 { + return relationFreshness{} + } + return s.relationLabel(gaps, s.relationScope(ctx, ids, files)) +} + +// relationLabel labels the receipts that fall in scope: failed when the edit +// failed to publish, pending otherwise. One pending receipt for a path +// outweighs an older failed one, and any failed path makes the label failed. +func (s *Server) relationLabel(receipts []*mutationReceipt, inScope func(absPath string) bool) relationFreshness { + states := make(map[string]string) + for _, receipt := range receipts { + if !inScope(receipt.path) { + continue + } + key := s.repoRelative(receipt.path) + outcome := receipt.outcome(true) + if outcome.Pending || (outcome.Err == nil && outcome.Reindexed) { + states[key] = "pending" + } else if states[key] == "" { + states[key] = "failed" + } + } + var label relationFreshness + for path, state := range states { + label.files = append(label.files, path) + if label.state != "failed" { + label.state = state + } + } + sort.Strings(label.files) + return label +} + +// laggingMutationReceipts returns the request's mutation receipts whose edit +// the graph has not published: still pending, or terminally failed. +func (s *Server) laggingMutationReceipts(ctx context.Context) []*mutationReceipt { + var lagging []*mutationReceipt + for _, receipt := range s.mutationReceiptsForRepos(ctx) { + if outcome := receipt.outcome(true); outcome.Pending || outcome.Err != nil || !outcome.Reindexed { + lagging = append(lagging, receipt) + } + } + return lagging +} + +// relationScope reports whether an edit to a file can change the relations +// of ids: the file defines one of them, holds an edge into one of them, is one +// of files (graph-keyed), or its current bytes name one of them as a whole +// identifier, which is how a call added by an edit the graph has not read yet +// shows up. An id that does not resolve contributes the file and name its +// spelling carries, so it narrows the scope instead of widening it. +func (s *Server) relationScope(ctx context.Context, ids, files []string) func(absPath string) bool { + scoped := make(map[string]bool, len(files)+len(ids)) + for _, file := range files { + scoped[file] = true + } + var names []string + reader := s.readerFor(ctx) + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + node := reader.GetNode(id) + if node == nil { + if file := graph.IDFile(id); file != "" { + scoped[file] = true + } + names = append(names, identifierName(id[strings.LastIndex(id, ":")+1:])) + continue + } + scoped[node.FilePath] = true + names = append(names, identifierName(node.Name)) + for _, edge := range reader.GetInEdges(id) { + file := edge.FilePath + if file == "" { + file = graph.IDFile(edge.From) + } + if file != "" { + scoped[file] = true + } + } + } + named := make(map[string]bool) + return func(absPath string) bool { + if scoped[s.repoRelative(absPath)] { + return true + } + hit, seen := named[absPath] + if !seen && len(names) > 0 { + data, err := os.ReadFile(absPath) + hit = err == nil && namesAnyIdentifier(data, names) + named[absPath] = hit + } + return hit + } +} + +// identifierName is the last identifier of a possibly qualified name, the +// spelling a call site uses: Recv.Method is called as Method. +func identifierName(name string) string { + return name[strings.LastIndex(name, ".")+1:] +} + +// namesAnyIdentifier reports whether data holds one of names as a whole +// identifier rather than as part of a longer one. +func namesAnyIdentifier(data []byte, names []string) bool { + for _, name := range names { + if name == "" { + continue + } + needle := []byte(name) + for offset := 0; offset < len(data); { + i := bytes.Index(data[offset:], needle) + if i < 0 { + break + } + start, end := offset+i, offset+i+len(needle) + if (start == 0 || !isIdentByte(data[start-1])) && (end == len(data) || !isIdentByte(data[end])) { + return true + } + offset = start + 1 + } + } + return false +} + +// apply writes the label into an answer's fields. A fresh scope writes +// nothing, so an answer with nothing lagging keeps its shape. +func (f relationFreshness) apply(result map[string]any) { + if f.state == "" { + return + } + result["complete"] = false + result["graph_freshness"] = f.state + result["freshness_scope"] = "relations" + result["pending_file_count"] = len(f.files) + result["pending_files"] = f.files[:min(maxLabelledPendingFiles, len(f.files))] + if f.state == "failed" { + result["recovery_hint"] = mutationFreshnessRecoveryHint() + } +} + +// labelled is payload with the label's fields merged in, a struct payload +// being re-read as its JSON object first. A fresh scope returns payload +// unchanged. +func (f relationFreshness) labelled(payload any) any { + if f.state == "" { + return payload + } + data, err := json.Marshal(payload) + if err != nil { + return payload + } + var fields map[string]any + if json.Unmarshal(data, &fields) != nil || fields == nil { + return payload + } + f.apply(fields) + return fields +} + +// line is the relations line a text answer leads with. +func (f relationFreshness) line() string { + return fmt.Sprintf("relations: %s (%d files)", f.state, len(f.files)) +} + +// requireFreshError is the answer to a require_fresh call whose wait ended +// with relations in its scope still lagging. +func (f relationFreshness) requireFreshError() *mcp.CallToolResult { + message := fmt.Sprintf("require_fresh: %s after waiting %d ms: %s", + f.line(), f.waited.Milliseconds(), strings.Join(f.files[:min(maxLabelledPendingFiles, len(f.files))], ", ")) + if f.state == "failed" { + message += "; " + mutationFreshnessRecoveryHint() + } + return mcp.NewToolResultError(message) +} + +// merge folds the label of another scope the same answer read into f: the +// files of both, failed over pending, and the time spent waiting on each. +func (f relationFreshness) merge(other relationFreshness) relationFreshness { + f.waited += other.waited + if other.state == "" { + return f + } + if f.state != "failed" { + f.state = other.state + } + seen := make(map[string]bool, len(f.files)) + for _, file := range f.files { + seen[file] = true + } + for _, file := range other.files { + if !seen[file] { + f.files = append(f.files, file) + } + } + sort.Strings(f.files) + return f +} + +// decorate makes a rendered text answer lead with its label: TOON and plain +// text start with the relations line; GCX keeps GCX1 as line 1, carries the +// label as header meta and the relations line as a comment on line 2. JSON +// already carries the label in its fields and passes through unchanged. +func (f relationFreshness) decorate(res *mcp.CallToolResult) *mcp.CallToolResult { + if f.state == "" || res == nil || res.IsError || len(res.Content) == 0 { + return res + } + text, ok := res.Content[0].(mcp.TextContent) + if !ok || strings.HasPrefix(strings.TrimSpace(text.Text), "{") { + return res + } + body := text.Text + if strings.HasPrefix(body, wire.Tag) { + body, _ = injectGCXHeaderMeta(body, map[string]any{ + "graph_freshness": f.state, + "freshness_scope": "relations", + "pending_file_count": len(f.files), + }) + head, rows, _ := strings.Cut(body, wire.RowSep) + body = head + wire.RowSep + wire.CommentPrefix + " " + f.line() + wire.RowSep + rows + } else { + body = f.line() + "\n" + body + } + return rebuildTextResult(res, body) +} + +// provisionalSite is a line of a file whose edit the graph has not published +// that names a symbol as a whole identifier: a reference the graph cannot show +// yet, reported so a mutator neither misses it nor rewrites it. Line is 0 for +// the symbol's own file, which is provisional as a whole. +type provisionalSite struct { + File string `json:"file"` + Line int `json:"line"` + Text string `json:"text"` +} + +// provisionalSites scans the files of gaps for lines naming node, leaving out +// the lines where g already holds a reference to it. When the node's own file +// is among them it is one site without a line: until that edit publishes, the +// graph's span of the node may not hold, and a line that names the node there +// may be a new caller or the definition itself. +func (s *Server) provisionalSites(g graph.Reader, gaps []*mutationReceipt, node *graph.Node) []provisionalSite { + names := []string{identifierName(node.Name)} + known := make(map[string]bool) + for _, e := range g.GetInEdges(node.ID) { + if caller := g.GetNode(e.From); caller != nil && isReferencingEdgeKind(e.Kind) { + known[fmt.Sprintf("%s:%d", caller.FilePath, e.Line)] = true + } + } + seen := make(map[string]bool) + var sites []provisionalSite + for _, receipt := range gaps { + file := s.repoRelative(receipt.path) + if seen[file] { + continue + } + seen[file] = true + if file == node.FilePath { + sites = append(sites, provisionalSite{File: file}) + continue + } + data, err := os.ReadFile(receipt.path) + if err != nil || !namesAnyIdentifier(data, names) { + continue + } + for i, line := range strings.Split(string(data), "\n") { + if namesAnyIdentifier([]byte(line), names) && !known[fmt.Sprintf("%s:%d", file, i+1)] { + sites = append(sites, provisionalSite{File: file, Line: i + 1, Text: strings.TrimSpace(line)}) + } + } + } + sort.Slice(sites, func(i, j int) bool { + if sites[i].File != sites[j].File { + return sites[i].File < sites[j].File + } + return sites[i].Line < sites[j].Line + }) + return sites +} diff --git a/internal/mcp/relation_freshness_test.go b/internal/mcp/relation_freshness_test.go new file mode 100644 index 000000000..368e8b28a --- /dev/null +++ b/internal/mcp/relation_freshness_test.go @@ -0,0 +1,321 @@ +package mcp + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/query" +) + +// newRelationFixture indexes a committed git repository in which Target +// (target.go) is called from caller.go, beside the unrelated other.go and +// namer.go. It returns the server and the indexed root receipts are keyed by. +func newRelationFixture(t *testing.T) (*Server, string) { + t.Helper() + dir := t.TempDir() + for name, src := range map[string]string{ + "target.go": "package p\n\nfunc Target() {}\n", + "caller.go": "package p\n\nfunc Caller() {\n\tTarget()\n}\n", + "other.go": "package p\n\nfunc Other() {}\n", + "namer.go": "package p\n\nfunc Namer() {}\n", + } { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(src), 0o644)) + } + checkoutMutationGit(t, dir, "init", "-q") + checkoutMutationGit(t, dir, "add", ".") + checkoutMutationGit(t, dir, "commit", "-q", "-m", "base") + + g := graph.New() + idx := indexer.New(g, testRegistry(), config.Default().Index, zap.NewNop()) + _, err := idx.Index(dir) + require.NoError(t, err) + return NewServer(query.NewEngine(g), g, idx, nil, zap.NewNop(), nil), idx.RootPath() +} + +// writeUnindexed changes a file on disk without telling the graph, the state +// a pending edit leaves behind. +func writeUnindexed(t *testing.T, root, name, src string) string { + t.Helper() + path := filepath.Join(root, name) + require.NoError(t, os.WriteFile(path, []byte(src), 0o644)) + return path +} + +func targetID(t *testing.T, s *Server) string { + t.Helper() + for _, n := range s.graph.AllNodes() { + if n.Name == "Target" && n.Kind == graph.KindFunction { + return n.ID + } + } + t.Fatal("Target is not indexed") + return "" +} + +func callChangeTool(t *testing.T, handler func(context.Context, mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error), name string, args map[string]any) *mcpgo.CallToolResult { + t.Helper() + req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{Name: name, Arguments: args}} + result, err := handler(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, result) + return result +} + +func TestChangeToolsIgnorePendingReceiptOutsideScope(t *testing.T) { + s, root := newRelationFixture(t) + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n") + pendingFreshnessReceipt(s, "receipt-other", "", filepath.Join(root, "other.go"), 1) + + impact := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json"})) + detect := lifecycleResultPayload(t, callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"format": "json"})) + for tool, payload := range map[string]map[string]any{"impact": impact, "detect": detect} { + require.NotContains(t, payload, "graph_freshness", "%s labelled an edit outside its scope: %v", tool, payload) + require.Less(t, payload["barrier_wait_ms"], float64(100), "%s waited on an edit outside its scope", tool) + } + require.Equal(t, []any{"caller.go"}, detect["changed_files"]) + require.NotEmpty(t, detect["changed_symbols"]) +} + +func TestChangeToolsScopeIncludesFilesNamingTheSymbol(t *testing.T) { + s, root := newRelationFixture(t) + // namer.go now calls Target, but the graph has no edge for it yet; + // other.go names only a longer identifier. + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + pendingFreshnessReceipt(s, "receipt-other", "", + writeUnindexed(t, root, "other.go", "package p\n\nfunc Other() {\n\tTargetX()\n}\n"), 2) + + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json"})) + require.Equal(t, "pending", payload["graph_freshness"]) + require.Equal(t, "relations", payload["freshness_scope"]) + require.Equal(t, false, payload["complete"]) + require.Equal(t, []any{"namer.go"}, payload["pending_files"]) + require.Equal(t, float64(1), payload["pending_file_count"]) + require.Less(t, payload["barrier_wait_ms"], float64(500)) +} + +func TestChangeToolsUnresolvedIDDoesNotWidenBarrier(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-other", "", filepath.Join(root, "other.go"), 1) + pendingFreshnessReceipt(s, "receipt-fresh", "", + writeUnindexed(t, root, "fresh.go", "package p\n\nfunc Fresh() {}\n"), 2) + + // Defined by a pending edit the graph has not read: labelled, and bounded + // by the budget rather than by every receipt in the repository. + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": "fresh.go::Fresh", "format": "json"})) + require.Equal(t, "pending", payload["graph_freshness"]) + require.Equal(t, []any{"fresh.go"}, payload["pending_files"]) + require.Less(t, payload["barrier_wait_ms"], float64(500)) + + // No pending edit relates to it: answered at once, unlabelled. + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": "gone.go::Gone", "format": "json"})) + require.NotContains(t, payload, "graph_freshness") + require.Less(t, payload["barrier_wait_ms"], float64(100)) +} + +func TestChangeToolsRequireFreshWaitsUntilWaitDeadline(t *testing.T) { + s, root := newRelationFixture(t) + target := pendingFreshnessReceipt(s, "receipt-target", "", filepath.Join(root, "target.go"), 1) + caller := pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 2) + deadline := func(d time.Duration) string { return time.Now().Add(d).Format(time.RFC3339Nano) } + + // The receipt never lands: the wait runs past the default budget to the + // caller's deadline, then refuses. + started := time.Now() + result := callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "require_fresh": true, "wait_deadline": deadline(700 * time.Millisecond)}) + require.True(t, result.IsError) + require.GreaterOrEqual(t, time.Since(started), 600*time.Millisecond) + require.Contains(t, freshnessToolResultText(result), "relations: pending (2 files)") + require.Contains(t, freshnessToolResultText(result), "target.go") + + result = callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"require_fresh": true, "wait_deadline": deadline(500 * time.Millisecond)}) + require.True(t, result.IsError) + require.Contains(t, freshnessToolResultText(result), "caller.go") + + // The receipts land after the default budget but before the deadline: + // the answer waits for them and carries no label. + go func() { + time.Sleep(400 * time.Millisecond) + completeFreshnessReceipt(target, indexer.MutationResult{RequestedGeneration: 1, AppliedGeneration: 1, Reindexed: true}) + completeFreshnessReceipt(caller, indexer.MutationResult{RequestedGeneration: 2, AppliedGeneration: 2, Reindexed: true}) + }() + started = time.Now() + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json", "require_fresh": true, "wait_deadline": deadline(5 * time.Second)})) + require.GreaterOrEqual(t, time.Since(started), 350*time.Millisecond) + require.NotContains(t, payload, "graph_freshness") +} + +func TestChangeAnswersStartWithRelationsLine(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 1) + const line = "relations: pending (1 files)" + id := targetID(t, s) + + for _, format := range []string{"toon", "gcx"} { + impact := freshnessToolResultText(callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": id, "format": format})) + lines := strings.SplitN(impact, "\n", 3) + require.GreaterOrEqual(t, len(lines), 2, impact) + if format == "gcx" { + require.True(t, strings.HasPrefix(lines[0], "GCX1 "), impact) + require.Contains(t, lines[0], " graph_freshness=pending") + require.Contains(t, lines[0], " pending_file_count=1") + require.Contains(t, lines[0], " barrier_wait_ms=") + require.Equal(t, "# "+line, lines[1]) + } else { + require.Equal(t, line, lines[0], impact) + } + } + + detect := freshnessToolResultText(callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"format": "toon"})) + require.Equal(t, line, strings.SplitN(detect, "\n", 2)[0], detect) +} + +func nodeID(t *testing.T, s *Server, name string) string { + t.Helper() + for _, n := range s.graph.AllNodes() { + if n.Name == name && n.Kind == graph.KindFunction { + return n.ID + } + } + t.Fatalf("%s is not indexed", name) + return "" +} + +// A call to a symbol that an edit added and the graph has not read yet must +// block safe_delete like a graph reference, in plain and propagate mode. An +// edit to the symbol's own file blocks it as a whole: it may add a caller +// there, and the graph's span of the symbol may no longer hold. +func TestSafeDeleteRejectsTargetCalledFromPendingFile(t *testing.T) { + for name, edit := range map[string]struct { + file, src string + ref map[string]any + }{ + "caller file": {"namer.go", "package p\n\nfunc Namer() {\n\tOther()\n}\n", + map[string]any{"kind": "provisional", "file_path": "namer.go", "line": float64(4)}}, + "definition file": {"other.go", "package p\n\nfunc Extra() {\n\tOther()\n}\n\nfunc Other() {}\n", + map[string]any{"kind": "provisional", "file_path": "other.go"}}, + } { + t.Run(name, func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-"+edit.file, "", writeUnindexed(t, root, edit.file, edit.src), 1) + otherPath := filepath.Join(root, "other.go") + before, err := os.ReadFile(otherPath) + require.NoError(t, err) + id := nodeID(t, s, "Other") + + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": id, "dry_run": false, "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"]) + require.Equal(t, []any{edit.ref}, payload["references"]) + require.Equal(t, "pending", payload["graph_freshness"]) + + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": id, "propagate": true, "dry_run": false, "format": "json"})) + require.Equal(t, "propagation_blocked_manual", payload["status"]) + require.Equal(t, float64(1), payload["manual"]) + + after, err := os.ReadFile(otherPath) + require.NoError(t, err) + require.Equal(t, string(before), string(after), "the target was deleted with a pending caller") + }) + } + + // A call the graph already holds is listed once, as the graph's + // reference, when its file has an unrelated pending edit; propagate can + // still remove it. + t.Run("graph reference", func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n}\n\nfunc More() {}\n"), 1) + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": targetID(t, s), "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"]) + require.Equal(t, float64(1), payload["reference_count"], "%v", payload["references"]) + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": targetID(t, s), "propagate": true, "format": "json"})) + require.Equal(t, "propagation_plan", payload["status"]) + require.Equal(t, float64(0), payload["manual"], "%v", payload["caller_patches"]) + }) + + // Target is orphaned with Caller by the graph's edges, but a pending edit + // now calls it: the cascade must not delete it with Caller. + t.Run("cascade member", func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + sources := map[string]string{} + for _, name := range []string{"caller.go", "target.go"} { + data, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + sources[name] = string(data) + } + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": nodeID(t, s, "Caller"), "cascade": "apply", "dry_run": false, "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"], "%v", payload) + require.Equal(t, []any{map[string]any{"kind": "provisional", "file_path": "namer.go", "line": float64(4)}}, payload["references"]) + for name, src := range sources { + data, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + require.Equal(t, src, string(data), "%s changed although the cascade was refused", name) + } + }) +} + +// The relation-reading mutators answer at once and label the answer while an +// edit in their scope never publishes; they never wait it out or refuse. +func TestRelationToolsLabelAndNeverRefuseWithUnresolvedReceipt(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + id := targetID(t, s) + rename := `{"changes":{"target.go":[{"range":{"start":{"line":2,"character":5},"end":{"line":2,"character":11}},"newText":"Renamed"}]}}` + comment := `{"changes":{"` + nativeFileURI(filepath.Join(root, "namer.go")) + + `":[{"range":{"start":{"line":0,"character":0},"end":{"line":0,"character":0}},"newText":"// Namer names Target.\n"}]}}` + + for tool, call := range map[string]func() *mcpgo.CallToolResult{ + "change_contract": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handleChangeContract, "change_contract", + map[string]any{"source": "symbols", "symbols": id, "format": "json"}) + }, + "preview_edit": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handlePreviewEdit, "preview_edit", + map[string]any{"workspace_edit": rename, "diagnostics": false}) + }, + // No symbol is removed or renamed: only the touched file, named by + // an absolute file URI, brings the pending edit into scope. + "preview_edit file uri": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handlePreviewEdit, "preview_edit", + map[string]any{"workspace_edit": comment, "diagnostics": false}) + }, + } { + started := time.Now() + payload := lifecycleResultPayload(t, call()) + require.Less(t, time.Since(started), 2*time.Second, "%s waited on a receipt that never publishes", tool) + require.Equal(t, "pending", payload["graph_freshness"], "%s: %v", tool, payload) + require.Equal(t, []any{"namer.go"}, payload["pending_files"], tool) + require.Equal(t, false, payload["complete"], tool) + } +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index d0e52bdc7..79ca8fb10 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -374,12 +374,10 @@ type Server struct { equivalence *search.EquivalenceTable contractRegistry *contracts.Registry semanticMgr *semantic.Manager - // refsConfirmed is the lazy-enrichment ledger: symbol IDs whose incoming - // references have been confirmed on demand (via ConfirmSymbolRefs) this - // daemon session, so a repeat usages query serves from the confirmed graph - // instead of re-spawning the language server. An entry is recorded after - // the first attempt (success or empty) to bound per-query LSP work. - refsConfirmed sync.Map // symbolID → struct{} + // refsConfirmed bounds on-demand incoming-reference work per symbol. Entries + // hold an active claim, a retry deadline, or an accepted positive result for + // this analysis epoch. Failed/empty attempts never permanently suppress retry. + refsConfirmed sync.Map // symbolID → *referenceConfirmation feedback *feedbackManager notes *notesManager memories *memoryManager @@ -3229,22 +3227,16 @@ func ScheduleMemoryReleaseAfterBurst(logger *zap.Logger, reason string) { go runScheduledMCPMemoryRelease(logger, reason) } -// resetConfirmedRefs clears the lazy-enrichment ledger (see the -// refsConfirmed field). sync.Map has no clear-all, so this ranges and -// deletes; it is safe against concurrent confirmSymbolRefsOnDemand -// readers because a racing miss just re-confirms the symbol, which is -// idempotent — it re-lands the same lsp_resolved edges. +// resetConfirmedRefs starts a new reference-confirmation epoch. Finishers use +// pointer CAS, so an attempt admitted before Clear cannot revive its old entry. func (s *Server) resetConfirmedRefs() { - s.refsConfirmed.Range(func(k, _ any) bool { - s.refsConfirmed.Delete(k) - return true - }) + s.refsConfirmed.Clear() } // resetTestIndexProbe clears the per-repo-prefix test-symbol probe (see the -// testIndexProbe field) so the next review re-scans the rebuilt graph. Like -// resetConfirmedRefs it ranges-and-deletes because sync.Map has no clear-all, -// and it is safe against a concurrent testLangsIndexed reader: a racing miss +// testIndexProbe field) so the next review re-scans the rebuilt graph. +// Ranging and deleting is safe against a concurrent testLangsIndexed reader: +// a racing miss // just re-scans, which is idempotent. func (s *Server) resetTestIndexProbe() { s.testIndexProbe.Range(func(k, _ any) bool { diff --git a/internal/mcp/tools_analysis.go b/internal/mcp/tools_analysis.go index feae760a1..bec7687a4 100644 --- a/internal/mcp/tools_analysis.go +++ b/internal/mcp/tools_analysis.go @@ -352,12 +352,43 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques if scopeErr := s.repoPrefixInSessionScope(ctx, repoPrefix, repoPrefix); scopeErr != nil { return mcp.NewToolResultError(scopeErr.Error()), nil } + checkoutScoped := control != nil && control.CheckoutScoped var pending bool var freshnessErr error - if control != nil && control.CheckoutScoped { + var label relationFreshness + var diff *analysis.DiffResult + var freshness requestFreshness + var keys []string + var deadline time.Time + if checkoutScoped { pending, freshnessErr = s.mutationFreshnessSummaryForRepos(ctx, repoPrefix) - } else if err := s.awaitMutationFreshnessForRepos(ctx, repoPrefix); err != nil { - return mcp.NewToolResultError("change detection refused a stale graph: " + err.Error()), nil + } else { + // Diff the files first: they are the relation scope the wait below + // is bounded to, so an edit elsewhere in the repository neither + // delays nor labels this answer. + files, err := analysis.MapGitDiffContext(ctx, nil, repoRoot, repoPrefix, scope, baseRef) + if err != nil { + return mcp.NewToolResultError(err.Error()), nil + } + keys = make([]string, len(files.ChangedFiles)) + for i, file := range files.ChangedFiles { + keys[i] = analysis.GraphKey(repoPrefix, file, analysis.RepoRelativePath) + } + freshness = takeRequestFreshness(&req) + deadline = relationWaitDeadline(ctx, freshness) + label = s.awaitRelationFreshness(ctx, nil, keys, deadline) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil + } + diff = files + } + respond := func(result map[string]any) (*mcp.CallToolResult, error) { + if !checkoutScoped { + result["barrier_wait_ms"] = label.waited.Milliseconds() + label.apply(result) + } + res, err := s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err } reader := s.readerFor(ctx) graphStatus, graphDetail := "ready", "" @@ -381,7 +412,12 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques reader = nil } - diff, err := analysis.MapGitDiffContext(ctx, reader, repoRoot, repoPrefix, scope, baseRef) + var err error + if diff == nil { + diff, err = analysis.MapGitDiffContext(ctx, reader, repoRoot, repoPrefix, scope, baseRef) + } else { + diff, err = analysis.JoinDiffSymbolsContext(ctx, reader, repoPrefix, diff) + } if err != nil { return mcp.NewToolResultError(err.Error()), nil } @@ -398,7 +434,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques fileChanges = []analysis.FileChange{} } if graphStatus != "ready" { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "changed_symbols": []any{}, "changed_files": changedFiles, "file_changes": fileChanges, "risk": "UNKNOWN", "complete": false, "graph_status": graphStatus, "summary": "Git file changes are available; symbol mapping and impact await an exact published graph", @@ -407,7 +443,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques } if len(diff.ChangedSymbols) == 0 { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "changed_symbols": []any{}, "changed_files": changedFiles, "file_changes": fileChanges, @@ -432,6 +468,15 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques for i, cs := range diff.ChangedSymbols { symbolIDs[i] = cs.ID } + if !checkoutScoped { + // The walk reads the changed symbols' callers, and an edit outside + // the diff (a staged or compare scope leaves the working tree out) + // can be changing those. They share what is left of the budget. + label = label.merge(s.awaitRelationFreshness(ctx, symbolIDs, keys, deadline)) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil + } + } impact, err := s.analyzeDiffImpact(ctx, symbolIDs) if err != nil { @@ -459,7 +504,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques req.GetBool("summary_only", false), req.GetInt("offset", 0), req.GetInt("limit", 100)) - return s.respondJSONOrTOON(ctx, req, detectResult) + return respond(detectResult) } // tryImpactAnalysisSnapshots reads optional cached enrichments without waiting @@ -487,8 +532,12 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo ids[i] = strings.TrimSpace(ids[i]) } - if freshnessErr := s.awaitMutationFreshnessForRepos(ctx, s.mutationReposForSymbolIDs(ctx, ids)...); freshnessErr != nil { - return mcp.NewToolResultError("change impact refused a stale graph: " + freshnessErr.Error()), nil + // Wait briefly for the edits these symbols' relations depend on, then + // answer; whatever is still lagging is labelled, not refused. + freshness := takeRequestFreshness(&req) + label := s.awaitRelationFreshness(ctx, ids, nil, relationWaitDeadline(ctx, freshness)) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil } // Keep the mandatory pre-edit safety gate well below host transport @@ -511,7 +560,9 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo "test_files": impact.TestFiles, "total_affected": impact.TotalAffected, "cross_repo_impact": impact.CrossRepoImpact, + "barrier_wait_ms": label.waited.Milliseconds(), } + label.apply(result) // GNX-3: by_depth_counts is the headline; the heavy by_depth rows are // paged (offset / limit) or dropped (summary_only) so the agent gets the @@ -605,18 +656,20 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo } } - if s.isGCX(ctx, req) { + var res *mcp.CallToolResult + switch { + case s.isGCX(ctx, req): // encodeChangeImpact reads the same map shape we'd return as // JSON; routing through it keeps a single source of truth for // field names and avoids divergence on the next analyzer // addition. - return s.gcxResponseWithBudget(req)(encodeChangeImpact(result)) - } - if s.isTOON(ctx, req) { - return returnTOON(result) + res, err = s.gcxResponseWithBudget(req)(encodeChangeImpact(result)) + case s.isTOON(ctx, req): + res, err = returnTOON(result) + default: + res, err = s.respondJSONOrTOON(ctx, req, result) } - - return s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err } func impactComplete(impact *analysis.ImpactResult) bool { diff --git a/internal/mcp/tools_enhancements.go b/internal/mcp/tools_enhancements.go index 65586d88f..5132d62ee 100644 --- a/internal/mcp/tools_enhancements.go +++ b/internal/mcp/tools_enhancements.go @@ -3698,6 +3698,19 @@ func (s *Server) buildIndexHealthBasePayloadCtx(ctx context.Context) (map[string plannerStats["last_refresh_at"] = planner.LastRefreshAt.UTC().Format(time.RFC3339) plannerStats["last_refresh_reason"] = planner.LastRefreshReason } + // Named, not counted: they are what an operator checks the schema for, + // and the reason above alone does not say which lookups are scanning. + if len(planner.DenseIndexesMissing) > 0 { + plannerStats["dense_indexes_missing"] = planner.DenseIndexesMissing + msg := "A bulk index load ended without rebuilding these SQLite indexes: " + strings.Join(planner.DenseIndexesMissing, ", ") + + ". Queries that use them scan instead until they are back. The store rebuilds them in the background once " + + "its volume has free space at least the size of the store, and a daemon restart rebuilds them too." + if recommendation == "" { + recommendation = msg + } else { + recommendation = msg + " " + recommendation + } + } result["planner_stats"] = plannerStats if planner.Stale { msg := "SQLite planner statistics are stale (" + planner.Reason + "): the graph has grown past what the query " + @@ -3710,6 +3723,13 @@ func (s *Server) buildIndexHealthBasePayloadCtx(ctx context.Context) (map[string } } } + if reporter, ok := s.graph.(interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + }); ok { + if observations := reporter.LastFirstIndexDrains(); len(observations) > 0 { + result["last_first_index_drain"] = observations + } + } if prefixAuditOK { ownership := map[string]any{ "owned_code_nodes": prefixAudit.OwnedCodeNodes, diff --git a/internal/mcp/tools_lsp.go b/internal/mcp/tools_lsp.go index f525fb29b..b340d8752 100644 --- a/internal/mcp/tools_lsp.go +++ b/internal/mcp/tools_lsp.go @@ -64,7 +64,8 @@ func (s *Server) enrichNodeOnDemand(ctx context.Context, node *graph.Node) { // point: a request reading a checkout of its own has no writable output, so it // must not spawn a server or hover any bytes at all. Returns a nil output for // that case, for a path that cannot be placed on disk, and when no server -// serves the language. +// serves the language. A nonempty path with nil output means admission succeeded +// but provider resolution failed; callers can back off that failure separately. func (s *Server) lspEnrichmentTarget( ctx context.Context, producer string, node *graph.Node, ) (string, *EnrichmentOutput, *lsp.Provider) { @@ -95,7 +96,7 @@ func (s *Server) lspEnrichmentTarget( provider, _, err := s.lspProviderForPath(absPath) if err != nil || provider == nil { out.Abandon() - return "", nil, nil + return absPath, nil, nil } return absPath, out, provider } @@ -110,37 +111,79 @@ func lspEnrichmentCapability(producer string) graphview.CapabilityID { return graphview.CapLSPHover } -// confirmSymbolRefsOnDemand faults in a callable symbol's INCOMING references -// on a usages-shaped query (find_usages / get_callers) when the synchronous -// LSP sweep is off. It lazy-spawns the language server, confirms this one -// symbol's callers via call hierarchy, and lands the lsp_resolved edges into -// the graph so the query answer — and every repeat — reflects compiler-grade -// usages accuracy. Idempotent per session via the refsConfirmed ledger; a -// no-op for non-callable nodes, when no call-hierarchy server serves the -// language, or when already confirmed. +// referenceConfirmation is immutable once published. An entry with neither a +// deadline nor confirmed set is an active claim; its pointer is the finish token. +// Replacing that exact token prevents an old attempt from restoring a reset entry. +type referenceConfirmation struct { + retryAfter time.Time + confirmed bool +} + +const referenceRetryCooldown = time.Minute + +func (s *Server) claimReferenceConfirmation(id string, now time.Time) *referenceConfirmation { + for { + previous, loaded := s.refsConfirmed.Load(id) + if loaded { + entry := previous.(*referenceConfirmation) + if entry.confirmed || entry.retryAfter.IsZero() || now.Before(entry.retryAfter) { + return nil + } + claim := &referenceConfirmation{} + if s.refsConfirmed.CompareAndSwap(id, entry, claim) { + return claim + } + } else { + claim := &referenceConfirmation{} + if _, loaded := s.refsConfirmed.LoadOrStore(id, claim); !loaded { + return claim + } + } + } +} + +// confirmSymbolRefsOnDemand faults in compiler-confirmed incoming calls for a +// callable symbol. Positive, accepted results last for the analysis epoch. Empty +// results (including a still-indexing server), errors, and refused completions +// retry after a short cooldown, reusing the provider's existing client. func (s *Server) confirmSymbolRefsOnDemand(ctx context.Context, node *graph.Node) { + s.confirmSymbolRefsAt(ctx, node, time.Now()) +} + +func (s *Server) confirmSymbolRefsAt(ctx context.Context, node *graph.Node, now time.Time) { if node == nil || s.semanticMgr == nil || s.graph == nil { return } if node.Kind != graph.KindFunction && node.Kind != graph.KindMethod { return } - if _, done := s.refsConfirmed.Load(node.ID); done { + claim := s.claimReferenceConfirmation(node.ID, now) + if claim == nil { return } - _, out, provider := s.lspEnrichmentTarget(ctx, EnrichProducerSymbolRefs, node) + path, out, provider := s.lspEnrichmentTarget(ctx, EnrichProducerSymbolRefs, node) if out == nil { + // Admission refusal, particularly a routed view, must leave the primary + // symbol eligible. Only remove our own claim, never a later epoch's entry. + if path == "" { + s.refsConfirmed.CompareAndDelete(node.ID, claim) + } else { + // Output was admitted, but its provider failed to start or resolve. + s.refsConfirmed.CompareAndSwap(node.ID, claim, &referenceConfirmation{retryAfter: now.Add(referenceRetryCooldown)}) + } return } - if _, err := provider.ConfirmSymbolRefs(out.Store, out.Root, node); err != nil { + count, err := provider.ConfirmSymbolRefs(out.Store, out.Root, node) + accepted := false + if err != nil { out.Abandon() } else { - _ = out.Complete() + accepted = out.Complete() == nil } - // Record after the attempt (even on 0/err) so a query doesn't re-spawn the - // server every call; a file re-index clears staleness through the normal - // restub path. Durable-ledger + retry semantics are the fuller version. - s.refsConfirmed.Store(node.ID, struct{}{}) + s.refsConfirmed.CompareAndSwap(node.ID, claim, &referenceConfirmation{ + confirmed: count > 0 && err == nil && accepted, + retryAfter: now.Add(referenceRetryCooldown), + }) } // registerLSPTools wires the LSP-action MCP surface: diff --git a/internal/mcp/tools_safe_delete.go b/internal/mcp/tools_safe_delete.go index 17b194870..08ac47ad4 100644 --- a/internal/mcp/tools_safe_delete.go +++ b/internal/mcp/tools_safe_delete.go @@ -52,10 +52,11 @@ func (s *Server) registerSafeDeleteSymbolTool() { // information for the caller to navigate to it and remove it before // retrying the delete. type safeDeleteReference struct { - FromID string `json:"from_id"` + FromID string `json:"from_id,omitempty"` Kind string `json:"kind"` FromName string `json:"from_name,omitempty"` FilePath string `json:"file_path,omitempty"` + Line int `json:"line,omitempty"` } // cascadeClosureEntry describes a symbol that the cascade pass added @@ -109,6 +110,20 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq // EdgeDefines / EdgeMemberOf are skipped (they don't represent // "someone calls this"). refs := collectReferencingEdges(g, id) + // A call that an edit added and the graph has not read yet blocks the + // delete like a graph reference does: a provisional reference here, a + // manual patch in propagate mode. The same snapshot labels the answer. + gaps := s.laggingMutationReceipts(ctx) + provisional := s.provisionalSites(g, gaps, node) + for _, site := range provisional { + refs = append(refs, safeDeleteReference{Kind: "provisional", FilePath: site.File, Line: site.Line}) + } + respond := func(result map[string]any) (*mcp.CallToolResult, error) { + label := s.labelRelations(ctx, gaps, []string{id}, nil) + label.apply(result) + res, err := s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err + } // SRN-7: propagate-delete — patch the surviving call sites instead of // refusing. Builds a per-caller plan, applies the removable standalone @@ -116,9 +131,18 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq var appliedPatches, failedPatches []callerPatch if req.GetBool("propagate", false) && len(refs) > 0 { plan := s.buildPropagationPlan(ctx, g, node) + for _, site := range provisional { + reason := "provisional: named in a file whose edit the graph has not read yet — patch by hand" + if site.Line == 0 { + reason = "provisional: the graph has not read the latest edit to the symbol's own file — retry once it publishes" + } + plan = append(plan, callerPatch{ + File: site.File, Line: site.Line, CurrentText: site.Text, Action: "manual", Reason: reason, + }) + } manual := countManual(plan) if dryRun { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_plan", "symbol": id, "file": node.FilePath, @@ -131,7 +155,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq }) } if manual > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_blocked_manual", "symbol": id, "caller_patches": plan, @@ -145,7 +169,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq } appliedPatches, failedPatches = applied, failed if len(failed) > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_partial", "symbol": id, "patches_applied": applied, @@ -158,7 +182,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq force = true node = g.GetNode(id) if node == nil { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "deleted_by_propagation", "symbol": id, "patches_applied": appliedPatches, @@ -171,7 +195,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq } if len(refs) > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "rejected_has_references", "symbol": id, "file": node.FilePath, @@ -195,6 +219,40 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if cascadeMode != cascadeModeOff { closure, cascadeTruncated = computeCascadeClosure(g, node, cascadeIntoTests) } + if cascadeMode == cascadeModeApply { + // The closure is orphaned by the graph's edges only. A call that an + // unpublished edit added keeps a member alive, and force does not + // cover the members, so the cascade waits for that edit. + var blocked []safeDeleteReference + var members []string + for _, entry := range closure { + member := g.GetNode(entry.ID) + if member == nil { + continue + } + sites := s.provisionalSites(g, gaps, member) + if len(sites) > 0 { + members = append(members, entry.ID) + } + for _, site := range sites { + blocked = append(blocked, safeDeleteReference{Kind: "provisional", FilePath: site.File, Line: site.Line}) + } + } + if len(blocked) > 0 { + return respond(map[string]any{ + "status": "rejected_has_references", + "symbol": id, + "file": node.FilePath, + "references": blocked, + "reference_count": len(blocked), + "dry_run": dryRun, + "force": force, + "cascade_mode": cascadeMode, + "cascade_closure": closure, + "hint": "cascade member(s) " + strings.Join(members, ", ") + " are named in files whose edits the graph has not read yet; retry once those publish, or delete without cascade", + }) + } + } absPath, err := s.resolveNodePath(ctx, node) if err != nil { @@ -242,7 +300,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if dryRun { result["status"] = "preview" - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // Commit phase. cascadeModeApply expands the work to every entry @@ -282,7 +340,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq result["status"] = "partial_failure" result["cascade_deleted"] = deletedIDs result["error"] = err.Error() - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // Persist session state and re-index every touched file. @@ -310,7 +368,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if len(failedPatches) > 0 { result["patches_failed"] = failedPatches } - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // normaliseCascadeMode coerces the cascade parameter to a known diff --git a/internal/mcp/tools_simulate.go b/internal/mcp/tools_simulate.go index fdfadea7c..4156dfb43 100644 --- a/internal/mcp/tools_simulate.go +++ b/internal/mcp/tools_simulate.go @@ -107,6 +107,9 @@ func (s *Server) handlePreviewEdit(ctx context.Context, req mcp.CallToolRequest) wantDiag := req.GetBool("diagnostics", true) diagTimeout := time.Duration(req.GetInt("diagnostics_timeout_ms", 1500)) * time.Millisecond + // Snapshot the unpublished edits before the simulation reads relations; + // the symbols that scope the label are known only once it has run. + gaps := s.laggingMutationReceipts(ctx) sim, simErr := s.buildSimulation(ctx, []lsp.WorkspaceEdit{edit}, inherit) if simErr != nil { if ctxErr := requestContextError(ctx, simErr); ctxErr != nil { @@ -138,6 +141,20 @@ func (s *Server) handlePreviewEdit(ctx context.Context, req mcp.CallToolRequest) result["diagnostics_summary"] = summariseDiagnostics(diags) } + // broken_callers walks the callers of the removed and renamed symbols. + seeds := append([]string{}, step.symbolsRemoved...) + for _, renamed := range step.symbolsRenamed { + seeds = append(seeds, renamed["from"]) + } + // touched_files keeps each path as the client spelled it; the scope + // takes graph keys. + var touched []string + for _, path := range step.touchedFiles { + if abs, err := s.resolveOverlayAbsPath(path); err == nil && abs != "" { + touched = append(touched, s.repoRelative(abs)) + } + } + s.labelRelations(ctx, gaps, seeds, touched).apply(result) return mcp.NewToolResultText(jsonOK(result)), nil }