indexer: isolate checkout discovery and report live worktree additions

bench/load/README.md

@@ -749,3 +749,59 @@ Inspect a profile with
 `go tool pprof -sample_index=inuse_space -top worktrees.step3.heap-load-peak.pprof`,
 and what one more worktree retains with
 `go tool pprof -sample_index=inuse_space -top -base worktrees.step2.heap-idle.pprof worktrees.step3.heap-idle.pprof`.
+
+### Live checkout discovery containment
+
+`--scenario worktree-discovery` starts with the first two pinned repositories
+in the manifest. It keeps their reading/editing sessions open while adding
+`--worktrees` detached checkouts alternating the two repositories during the offered
+window. Each added checkout gets separate root and subdirectory query sessions.
+Use `--agents 0 --edit-agents 8` for eight combined edit/readback agents, or
+`--agents 8 --edit-agents 0` for a read-only control. Editing requires the same
+validated edit fixtures as the ordinary edit scenarios.
+
+`phase.gate.checkout_discovery` records every added root, first request, first
+observed exact worktree view, and the resulting visibility lag. That lag is an
+upper bound on catalog identity allocation; it does not claim an unobserved
+earlier allocation time. Each cwd has its class, call/error/refusal counts,
+complete-call read latency distribution and separately counted censored reads.
+Attempts retain the actual session cwd and structured RPC `error_data`, including
+the refused root, family, reason and retryability when supplied by the daemon.
+A refusal or completed error outside a currently pending checkout fails the run.
+
+Run an otherwise identical `--worktrees 0` control to compare primary read p95.
+Keep both reports and all refused/censored attempts. Zero refusals without any
+`pending_refusals` is not evidence of fault containment under a stuck discovery.
+A hung-discovery fixture, upgraded-store fixture and the reference-host runs
+remain separate acceptance cells; this scenario does not manufacture them.
+All measurements require an isolated host window and fresh owned clones/stores.
+
+
+Discovery additions alternate the primary and unrelated control families. Each
+new root keeps its own family query and prefix for the exact-view witness;
+foreign-family capacity refusals fail the containment gate even while that
+new root is pending. `--discovery-stall-first` installs a daemon-only Git shim
+for the first new primary-family checkout. Its child ignores TERM and retains
+stdout until explicit release or a 60-second bound. The harness releases it
+before daemon shutdown and retains child/root/operation events. A missing
+stall or pending-refusal observation fails this cell.
+
+For upgraded fixtures, `--discovery-prepare <descriptor.json>` prepares owned
+roots without opening a store or launching a daemon. Add
+`--discovery-existing-worktrees 20 --discovery-existing-nested 5` to create five
+in-repository and fifteen sibling checkouts of the first family. The unrelated
+control is retained, so descriptor counts are two primaries plus twenty linked
+checkouts; selected-family counts are one plus twenty. The descriptor records
+actual paths, Git common directories, commits, source and configuration hashes,
+and a random owner token. Preparation requires a separate exclusive fixture
+producer to populate and upgrade the store at those exact paths.
+
+`--discovery-resume <lineage.json>` consumes that prepared root once. Lineage
+must bind the descriptor, old and upgrade binary/source pins, schema-44 main
+and WAL hashes, and structured normal-stop/catalog-identity receipts. Unknown,
+live, changed or relocated identities are refused. The stopped immutable
+snapshot remains inside the owned root; the destination store must be absent.
+The existing seed copier copies only main/WAL files, then both source and
+copied hashes are checked again. There is no SQL or catalog path rewriting.
+These discovery-only flags preserve the startup scenario's `--seed-store`
+contract. A copied fixture or synthetic parser control is not upgrade acceptance.

bench/load/call.go

@@ -112,18 +112,33 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[
 				}
 			}
 		}
-		if err == nil && callCtx.Err() != nil {
-			err = callCtx.Err()
+		// A socket deadline can fire before the context timer goroutine runs.
+		// Classify against the actual call deadline as well, so a clipped
+		// offered window does not invent a product/checkout failure.
+		expired := callCtx.Err() != nil
+		if deadline, ok := callCtx.Deadline(); ok && !returned.Before(deadline) {
+			expired = true
+		}
+		expiry := callCtx.Err()
+		if expired && expiry == nil {
+			expiry = context.DeadlineExceeded
+		}
+		if err == nil && expired {
+			err = expiry
 		}
 		if err == nil {
 			err = checkToolResult(data)
 		}
 		a := Attempt{Tool: r.name(), Role: role, Request: hash, Agent: agent, Step: step, Slot: slot, Retry: retry, Started: start, DurationMS: float64(returned.Sub(start)) / float64(time.Millisecond), LogicalMS: float64(returned.Sub(logicalStart)) / float64(time.Millisecond), Outcome: "success"}
+		if session, ok := client.(*socketClient); ok {
+			a.CWD = session.cwd
+		}
 		var remote *rpcError
 		if err != nil {
 			a.Outcome, a.Error = "error", err.Error()
 			if errors.As(err, &remote) {
 				a.Code = remote.Code
+				a.ErrorData = copyRPCErrorData(remote.Data)
 			}
 			if remote != nil && remote.Code == -32002 && (!retryTyped || remote.retryEditProbe()) {
 				a.Outcome = "busy"
@@ -135,9 +150,9 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[
 				a.Code = -32000
 				a.ExpectedOutcome = "symbol_absent"
 			}
-			if callCtx.Err() != nil {
+			if expired {
 				a.Outcome = terminalOutcome()
-				err = errors.Join(err, callCtx.Err())
+				err = errors.Join(err, expiry)
 			}
 			if retryTyped && errors.Is(err, errEditProbeEvidenceIncomplete) {
 				a.Outcome = "harness_incomplete"

bench/load/client.go

@@ -64,6 +64,7 @@ type socketClient struct {
 	notifies      []notificationObservation
 	notifyDropped int
 	version       string
+	cwd           string // immutable checkout supplied by the successful handshake
 }
 
 // Per-call values returned to its caller, never a shared last-ID cache. Times
@@ -142,7 +143,7 @@ func openMCP(ctx context.Context, socket, cwd string) (*socketClient, error) {
 	if err != nil {
 		return nil, err
 	}
-	c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, writeGate: make(chan struct{}, 1)}
+	c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: cwd, writeGate: make(chan struct{}, 1)}
 	go c.readLoop()
 	if _, err := c.call(ctx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "gortex-load", "version": "1"}}); err != nil {
 		_ = c.Close()

bench/load/discovery_catalog.go

@@ -0,0 +1,170 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	"github.com/zzet/gortex/internal/indexer"
+)
+
+// These fields come from actual list_checkouts observations, never prepared
+// topology declarations. View riders are intentionally not catalog identity.
+type discoveryCatalogIdentity struct {
+	Root        string `json:"root"`
+	CheckoutID  string `json:"checkout_id"`
+	Incarnation string `json:"incarnation"`
+	FamilyID    string `json:"family_id"`
+	RepoPrefix  string `json:"repo_prefix"`
+	CommonDir   string `json:"common_dir"`
+}
+type discoveryCatalogScope struct {
+	CWD      string                   `json:"cwd"`
+	Overview indexer.FamiliesOverview `json:"overview"`
+}
+type discoveryCatalogReceipt struct {
+	Before []discoveryCatalogScope `json:"before"`
+	After  []discoveryCatalogScope `json:"after"`
+}
+
+func discoveryCatalogIdentities(scopes []discoveryCatalogScope, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) {
+	expected := make(map[string]discoverySeedRepo, len(repos))
+	for _, repo := range repos {
+		expected[repo.Repo.Path] = repo
+	}
+	result := make(map[string]discoveryCatalogIdentity, len(repos))
+	ids := make(map[string]string)
+	owners := make(map[string]string)
+	for _, scope := range scopes {
+		for _, family := range scope.Overview.Families {
+			if family.FamilyID == "" || family.CommonDir == "" || family.PrimaryRepoPrefix == "" {
+				return nil, errors.New("discovery catalog has incomplete family identity")
+			}
+			graphs := make(map[string]indexer.GraphOverview)
+			for _, graph := range family.Graphs {
+				if graph.GraphID == "" || graph.RepoPrefix == "" || graph.OwnerCheckoutID == "" {
+					return nil, errors.New("discovery catalog has incomplete graph identity")
+				}
+				if old, ok := graphs[graph.GraphID]; ok && old != graph {
+					return nil, errors.New("discovery catalog graph has conflicting identities")
+				}
+				graphs[graph.GraphID] = graph
+				owners[family.FamilyID+"\x00"+graph.RepoPrefix] = graph.OwnerCheckoutID
+			}
+			for _, checkout := range family.Checkouts {
+				repo, ok := expected[checkout.RootPath]
+				prefix := family.PrimaryRepoPrefix
+				if checkout.GraphID != "" {
+					prefix = graphs[checkout.GraphID].RepoPrefix
+				}
+				if !ok || checkout.CheckoutID == "" || checkout.Incarnation == "" || prefix != repo.Repo.Prefix || family.CommonDir != repo.CommonDir {
+					return nil, errors.New("discovery catalog checkout does not bind prepared root and family")
+				}
+				row := discoveryCatalogIdentity{Root: checkout.RootPath, CheckoutID: checkout.CheckoutID, Incarnation: checkout.Incarnation, FamilyID: family.FamilyID, RepoPrefix: prefix, CommonDir: family.CommonDir}
+				if old, ok := result[row.Root]; ok && old != row {
+					return nil, errors.New("discovery catalog root has conflicting identity")
+				}
+				if root, ok := ids[row.CheckoutID]; ok && root != row.Root {
+					return nil, errors.New("discovery catalog ID names multiple roots")
+				}
+				result[row.Root] = row
+				ids[row.CheckoutID] = row.Root
+			}
+		}
+	}
+	if len(result) != len(expected) {
+		return nil, errors.New("discovery catalog does not cover every prepared root")
+	}
+	for _, repo := range repos[:2] {
+		row := result[repo.Repo.Path]
+		if owners[row.FamilyID+"\x00"+row.RepoPrefix] != row.CheckoutID {
+			return nil, errors.New("discovery catalog primary graph has wrong checkout owner")
+		}
+	}
+	return result, nil
+}
+
+func discoveryReadCatalogReceipt(root string, proof discoverySeedProof, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) {
+	if err := discoveryOwnedFile(root, proof.CatalogReceipt); err != nil {
+		return nil, err
+	}
+	hash, err := discoveryFileSHA(proof.CatalogReceipt)
+	if err != nil || !discoveryDigestValid(proof.CatalogReceiptSHA256, 32) || hash != proof.CatalogReceiptSHA256 {
+		return nil, errors.New("discovery raw catalog receipt digest mismatch")
+	}
+	var raw discoveryCatalogReceipt
+	if err := discoveryReadJSON(proof.CatalogReceipt, &raw); err != nil {
+		return nil, err
+	}
+	before, err := discoveryCatalogIdentities(raw.Before, repos)
+	if err != nil {
+		return nil, err
+	}
+	after, err := discoveryCatalogIdentities(raw.After, repos)
+	if err != nil {
+		return nil, err
+	}
+	if !reflect.DeepEqual(before, after) {
+		return nil, errors.New("discovery old-to-upgraded catalog identity mapping changed")
+	}
+	return after, nil
+}
+
+func (p *privateRun) verifyDiscoveryCatalog(ctx context.Context) (json.RawMessage, error) {
+	if p.discoveryCatalog == nil {
+		return nil, nil
+	}
+	var scopes []discoveryCatalogScope
+	for _, repo := range p.Repos[:2] {
+		callCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		conn, reader, version, err := dialSocketSurface(callCtx, p.Socket, repo.Path, "mcp", "full", "hide")
+		if err != nil {
+			cancel()
+			return nil, err
+		}
+		client := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: repo.Path, writeGate: make(chan struct{}, 1)}
+		go client.readLoop()
+		_, err = client.call(callCtx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "discovery-lineage", "version": "1"}})
+		if err == nil {
+			err = client.send(callCtx, map[string]any{"jsonrpc": "2.0", "method": "notifications/initialized"}, false)
+		}
+		var raw json.RawMessage
+		if err == nil {
+			raw, err = client.call(callCtx, "tools/call", map[string]any{"name": "list_checkouts", "arguments": map[string]any{"format": "json", "max_bytes": 0}})
+		}
+		_ = client.Close()
+		cancel()
+		if err != nil {
+			return nil, err
+		}
+		payload, err := textPayload(raw, "content")
+		if err != nil {
+			return nil, err
+		}
+		var overview indexer.FamiliesOverview
+		if err = json.Unmarshal(payload, &overview); err != nil {
+			return nil, err
+		}
+		scopes = append(scopes, discoveryCatalogScope{CWD: repo.Path, Overview: overview})
+	}
+	repos := make([]discoverySeedRepo, len(p.Repos))
+	for i, repo := range p.Repos {
+		identity := p.discoveryCatalog[repo.Path]
+		repos[i] = discoverySeedRepo{Repo: repo, CommonDir: identity.CommonDir}
+	}
+	actual, err := discoveryCatalogIdentities(scopes, repos)
+	if err != nil {
+		return nil, err
+	}
+	if !reflect.DeepEqual(actual, p.discoveryCatalog) {
+		return nil, errors.New("resumed discovery changed observed catalog checkout identity")
+	}
+	raw, err := json.Marshal(scopes)
+	if err == nil {
+		err = discoveryWriteJSON(filepath.Join(p.Root, "resumed-catalog-observed.json"), scopes)
+	}
+	return raw, err
+}