From ab05754896b233409e9f758c3bd7a420f4add0a9 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:00:07 +0100 Subject: [PATCH 01/60] fix(sqlite): release the bulk write gate on every bulk-window exit FlushBulk and EndCoordinatedBulkLoad unlocked writeMu by hand on each return path. A panic inside the dense-index seal (or anything else under the gate) skipped the unlock, so every later writer, the owner's retry and Close all blocked on the gate forever. Split both into a write-gate half that releases with a defer and an outer half that runs what the exit still owes after the release, in the same order as before: the residue gate for the coordinated final PASSIVE, then the durability TRUNCATE for a standalone window. A panic now unwinds with the gate free and skips the post-release work. EndCoordinatedBulkLoad clears the coordinated flag before its FTS merges and the seal. A panic there left the pinned writer live behind a window nobody owned: the owner's next End returned nil and the store kept synchronous=OFF until Close. A deferred guard now restores the flag, the same retryable state the seal-error branch already keeps. SetBulkSealHookForTest is a per-store hook the seal calls before each dense-index DDL; an error stands in for that DDL failing and a panic unwinds the seal. TestFlushBulkSealPanicDoesNotDeadlock drives a non-error panic through both exits, probes the gate with a deadline so a regression fails instead of hanging, and checks the retry seals, closes the window and restores synchronous. --- internal/graph/store_sqlite/bulk_load.go | 111 +++++++++++++++--- internal/graph/store_sqlite/bulk_load_test.go | 77 ++++++++++++ internal/graph/store_sqlite/store.go | 3 + 3 files changed, 172 insertions(+), 19 deletions(-) diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 4d69e3b1b..bd7ddc763 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -924,13 +924,53 @@ func (s *Store) scheduleWALDrainAboveLine(result walCheckpointResult, boundary s s.scheduleWALDrain(boundary) } +// bulkWindowExit is what a bulk-window exit still owes once the write gate is +// released. The locked half of FlushBulk / EndCoordinatedBulkLoad releases the +// gate with a defer, so a panic anywhere under it (a seal DDL, an FTS merge, a +// test hook) unwinds with the gate free and every later writer can still get +// in; the owed work below is then skipped along with the rest of the caller. +type bulkWindowExit struct { + // durabilityCheckpoint: a standalone window closed, so the frames it + // committed at synchronous=OFF are synced by one TRUNCATE on a NORMAL + // connection (checkpointBulkWAL). + durabilityCheckpoint bool + // residue is what a coordinated window's final PASSIVE left in the WAL; + // residueBoundary names that boundary for the residue gate, and is empty + // when no such PASSIVE ran. + residue walCheckpointResult + residueBoundary string +} + +// finishBulkWindowExit runs what exit owes after writeMu is released, in the +// order the exits have always run it. The durability checkpoint must run on a +// NORMAL connection, and it must not wait for a second writer slot while the +// bulk connection is still pinned, so neither step may move under the gate. +func (s *Store) finishBulkWindowExit(exit bulkWindowExit) error { + if exit.residueBoundary != "" { + s.scheduleWALDrainAboveLine(exit.residue, exit.residueBoundary) + } + if exit.durabilityCheckpoint { + return s.checkpointBulkWAL() + } + return nil +} + // FlushBulk exits the bulk-load fast path: it rebuilds every dropped index, // restores synchronous + cache_size + wal_autocheckpoint, releases the pinned writer and write gate, // then performs one bounded TRUNCATE checkpoint. The ordering matters: the // durability checkpoint must run on a NORMAL connection, and it must not wait // for a second writer slot while the bulk connection is still pinned. func (s *Store) FlushBulk() error { + exit, err := s.flushBulkLocked() + return errors.Join(err, s.finishBulkWindowExit(exit)) +} + +// flushBulkLocked is FlushBulk's write-gate half. A seal that panics leaves the +// pinned writer and its deferred indexes in place, exactly as a seal error +// does, so the next FlushBulk retries the same window. +func (s *Store) flushBulkLocked() (bulkWindowExit, error) { s.writeMu.Lock() + defer s.writeMu.Unlock() if s.coordinatedBulkLoad { // The outer coordinated window owns index sealing and its final // durability checkpoint. WAL pressure inside that window is already @@ -938,8 +978,7 @@ func (s *Store) FlushBulk() error { // repository boundary can coincide with an unrelated read snapshot and // spend the full passive-checkpoint timeout without advancing the WAL; // repeating that timeout once per repository only stalls the cold path. - s.writeMu.Unlock() - return nil + return bulkWindowExit{}, nil } if s.generationBulkLoad != 0 { // A generation-scoped window belongs to whoever opened it, exactly as @@ -959,31 +998,46 @@ func (s *Store) FlushBulk() error { // index (bulkIndexesDeferred is false for it) and its WAL is already // bounded inside the payload by noteBulkRowsLocked's row intervals, so // leaving it open costs the pass nothing it would otherwise have paid. - s.writeMu.Unlock() - return nil + return bulkWindowExit{}, nil } hadBulk := s.bulkConn != nil sealErr := s.sealBulkIndexesLocked("flush") closeErr := s.closeBulkConnectionLocked() s.jsonbIngestBuffers.release() - s.writeMu.Unlock() - if !hadBulk { - return errors.Join(sealErr, closeErr) - } - return errors.Join(sealErr, closeErr, s.checkpointBulkWAL()) + return bulkWindowExit{durabilityCheckpoint: hadBulk}, errors.Join(sealErr, closeErr) } // EndCoordinatedBulkLoad closes an outer multi-repository cold-load window. // It is idempotent and safe to defer: durability/cache restoration and index // rebuild happen even when one repository failed or panicked. func (s *Store) EndCoordinatedBulkLoad() error { + exit, err := s.endCoordinatedBulkLoadLocked() + return errors.Join(err, s.finishBulkWindowExit(exit)) +} + +// endCoordinatedBulkLoadLocked is EndCoordinatedBulkLoad's write-gate half. +func (s *Store) endCoordinatedBulkLoadLocked() (bulkWindowExit, error) { s.writeMu.Lock() + defer s.writeMu.Unlock() if !s.coordinatedBulkLoad { - s.writeMu.Unlock() - return nil + return bulkWindowExit{}, nil } s.coordinatedBulkLoad = false s.syncBulkWindowLocked() + // The flag is cleared above, before the FTS merges and the seal, so a panic + // in either would otherwise leave the pinned writer live behind a window + // nobody owns: the owner's next End would find no coordinated load and + // return nil, and the store would keep synchronous=OFF until Close. Put the + // window back the way the seal-error branch below keeps it, retryable by + // the same owner. Registered after the early return so it never claims a + // standalone window, and inert once closeBulkConnectionLocked has released + // the connection or the seal-error branch has restored the flag itself. + defer func() { + if s.bulkConn != nil && !s.coordinatedBulkLoad { + s.coordinatedBulkLoad = true + s.syncBulkWindowLocked() + } + }() if s.deferredFTSOptimize { // A full FTS5 optimize is unbounded and previously sat directly on the // cold-start critical path. One bounded merge keeps segment growth in @@ -1008,8 +1062,7 @@ func (s *Store) EndCoordinatedBulkLoad() error { // sealBulkIndexesLocked intentionally requires the pinned bulk writer. s.coordinatedBulkLoad = true s.syncBulkWindowLocked() - s.writeMu.Unlock() - return sealErr + return bulkWindowExit{}, sealErr } // Give the accumulated WAL one longer, still-bounded PASSIVE drain before // planner statistics traverse the large graph indexes. This runs on the @@ -1044,7 +1097,7 @@ func (s *Store) EndCoordinatedBulkLoad() error { // waiting for readers. Committed WAL frames are durable either way; later // passive checkpoints plus SQLite's next WAL reset/journal_size_limit reclaim // the file after the reader leaves. - var residue walCheckpointResult + var exit bulkWindowExit if hadBulk { ctx, cancel := context.WithTimeout(context.Background(), bulkFinalCheckpointTimeout) started := time.Now() @@ -1066,12 +1119,11 @@ func (s *Store) EndCoordinatedBulkLoad() error { // readers allowed. Hand that number to the residue gate rather than // treating "we tried once" as a drained log — the follow-up TRUNCATE // runs on the lane, after this window's readers are gone. - residue = result + exit.residue = result } + exit.residueBoundary = "cold_load_finalize" s.jsonbIngestBuffers.release() - s.writeMu.Unlock() - s.scheduleWALDrainAboveLine(residue, "cold_load_finalize") - return errors.Join(sealErr, statsErr, closeErr) + return exit, errors.Join(sealErr, statsErr, closeErr) } // AbortCoordinatedBulkLoad gives up retryable finalization and restores the @@ -1206,7 +1258,10 @@ func (s *Store) sealBulkIndexesLocked(reason string) error { _ = s.checkpointBulkWALPassiveLocked("index_seal_before") for _, idx := range bulkDroppableIndexes { indexStarted := time.Now() - _, err := conn.ExecContext(ctx, idx.ddl) + err := s.runBulkSealHook(idx.name) + if err == nil { + _, err = conn.ExecContext(ctx, idx.ddl) + } s.emitBulkFinalizeEvent(bulkFinalizeEvent{Stage: "index", Name: idx.name, Elapsed: time.Since(indexStarted), Err: err}) if err != nil { sealErr = errors.Join(sealErr, fmt.Errorf("store_sqlite: rebuild index %s: %w", idx.name, err)) @@ -1232,6 +1287,24 @@ func (s *Store) sealBulkIndexesLocked(reason string) error { return nil } +// SetBulkSealHookForTest installs a hook this store's dense-index seal calls +// before each index's DDL, with the index name. A returned error stands in for +// that DDL failing (the DDL is skipped and the seal stays retryable, as on a +// real failure); a panic unwinds the seal. The hook runs under the write gate +// and must not call back into the store. restore puts the previous hook back. +func (s *Store) SetBulkSealHookForTest(hook func(index string) error) (restore func()) { + previous := s.bulkSealHook.Swap(&hook) + return func() { s.bulkSealHook.Store(previous) } +} + +func (s *Store) runBulkSealHook(index string) error { + hook := s.bulkSealHook.Load() + if hook == nil || *hook == nil { + return nil + } + return (*hook)(index) +} + // closeBulkConnectionLocked restores connection-local pragmas and releases the // pinned writer. Dense-index rebuilding deliberately lives in // sealBulkIndexesLocked so an early seal cannot end the outer window. diff --git a/internal/graph/store_sqlite/bulk_load_test.go b/internal/graph/store_sqlite/bulk_load_test.go index 528c7b659..dd846d7c1 100644 --- a/internal/graph/store_sqlite/bulk_load_test.go +++ b/internal/graph/store_sqlite/bulk_load_test.go @@ -835,6 +835,83 @@ func TestUncoordinatedBulkSealFailureRetriesAtFlushBoundary(t *testing.T) { integrityOK(t, s.db) } +// TestFlushBulkSealPanicDoesNotDeadlock: a seal that panics with a non-error +// value must unwind with the write gate released and the window still +// retryable, so the owner's next exit call seals, restores and closes it. +func TestFlushBulkSealPanicDoesNotDeadlock(t *testing.T) { + for _, tc := range []struct { + name string + begin func(*Store) bool + end func(*Store) error + }{ + {"standalone", func(s *Store) bool { s.BeginBulkLoad(); return s.bulkConn != nil }, (*Store).FlushBulk}, + {"coordinated", (*Store).BeginCoordinatedBulkLoad, (*Store).EndCoordinatedBulkLoad}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _ := openTempStore(t) + syncBefore := pragmaIntDB(t, s.writerDB, "synchronous") + if !tc.begin(s) { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + calls := 0 + defer s.SetBulkSealHookForTest(func(string) error { + calls++ + if calls == 1 { + panic("injected seal panic") + } + return nil + })() + + recovered := func() (r any) { + defer func() { r = recover() }() + _ = tc.end(s) + return nil + }() + if recovered != "injected seal panic" { + t.Fatalf("recovered %v, want the injected seal panic", recovered) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := s.writeMu.LockContext(ctx); err != nil { + // Hand the leaked hold back so cleanup's Close cannot hang too. + s.writeMu.Unlock() + t.Fatalf("write gate still held after the seal panic: %v", err) + } + retryable := s.bulkConn != nil && s.bulkIndexesDeferred && s.bulkWindowOpen.Load() && + s.coordinatedBulkLoad == (tc.name == "coordinated") + s.writeMu.Unlock() + if !retryable { + t.Fatal("seal panic lost the window's retryable state") + } + + if err := tc.end(s); err != nil { + t.Fatalf("exit retry after the seal panic: %v", err) + } + if calls <= 1 { + t.Fatalf("seal hook calls = %d, want the retry to seal again", calls) + } + if s.bulkConn != nil || s.bulkIndexesDeferred || s.coordinatedBulkLoad || s.bulkWindowOpen.Load() { + t.Fatalf("retry left bulk state: conn=%v deferred=%v coordinated=%v open=%v", + s.bulkConn != nil, s.bulkIndexesDeferred, s.coordinatedBulkLoad, s.bulkWindowOpen.Load()) + } + if got := pragmaIntDB(t, s.writerDB, "synchronous"); got != syncBefore { + t.Fatalf("writer synchronous = %d, want restored %d", got, syncBefore) + } + present := indexNames(t, s.db) + for _, idx := range bulkDroppableIndexes { + if !present[idx.name] { + t.Fatalf("index %s missing after the retried seal", idx.name) + } + } + if got := s.NodeCount(); got != 1 { + t.Fatalf("node count = %d, want 1", got) + } + integrityOK(t, s.db) + }) + } +} + func TestCoordinatedBulkLoadRoutesSingleRowSidecarsOnPinnedConnection(t *testing.T) { s, _ := openTempStore(t) s.db.SetMaxOpenConns(1) diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index 110f85e60..816ed8ae4 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -434,6 +434,9 @@ type storeCore struct { // bulkFinalizeObserver is a package-private test/diagnostic hook. It runs // synchronously under writeMu and therefore must not call back into Store. bulkFinalizeObserver func(bulkFinalizeEvent) + // bulkSealHook is SetBulkSealHookForTest's hook. Per store rather than + // package-wide so parallel tests that each own a store do not share it. + bulkSealHook atomic.Pointer[func(index string) error] // Planner-statistics freshness ledger (planner_stats_freshness.go). It // lives on the core rather than the handle so every AtGeneration view From 739ea27d93c2efe5d2647083af9de8b5c3222a01 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:04:30 +0100 Subject: [PATCH 02/60] fix(sqlite): keep the dense indexes a failed seal leaves missing A cold bulk window that closes with its seal still owed (a failed seal at FlushBulk or Close, or AbortCoordinatedBulkLoad) restored durability and released the writer, but closeBulkConnectionLocked then cleared bulkIndexesDeferred and nothing remembered which dense indexes were gone. The planner probe reads an absent index as nothing to judge, so the store reported clean while lookups ran without their indexes. closeBulkConnectionLocked now reads the schema on the pinned connection (after the PRAGMA restores, before it closes) and records the droppable indexes that are absent, falling back to the whole droppable set when that read fails. A successful seal clears the record. It is not persisted: Open's createGraphCoreIndexes rebuilds the indexes on restart. PlannerStatsHealth reports the new reason dense_indexes_missing with the names in PlannerStatsFreshness.DenseIndexesMissing, shaped like bulk_window_active: not stale, nothing seeded, no ANALYZE attempted. TestFailedSealReportsDenseIndexesMissing covers a flush whose seal fails on two named indexes and an abort before any seal. --- internal/graph/planner_stats.go | 5 + internal/graph/store_sqlite/bulk_load.go | 67 +++++++++++++ .../store_sqlite/dense_index_repair_test.go | 97 +++++++++++++++++++ .../store_sqlite/planner_stats_freshness.go | 15 +++ internal/graph/store_sqlite/store.go | 8 ++ 5 files changed, 192 insertions(+) create mode 100644 internal/graph/store_sqlite/dense_index_repair_test.go diff --git a/internal/graph/planner_stats.go b/internal/graph/planner_stats.go index 636c45674..d5dd0f6a5 100644 --- a/internal/graph/planner_stats.go +++ b/internal/graph/planner_stats.go @@ -55,6 +55,11 @@ type PlannerStatsFreshness struct { Refreshed bool Reason string + // DenseIndexesMissing names secondary indexes the backend knows are absent + // and owes a rebuild for (an interrupted bulk load), reported with its own + // Reason instead of a verdict. Empty in the ordinary case. + DenseIndexesMissing []string + LastRefreshAt time.Time LastRefreshReason string diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index bd7ddc763..2a27c3c92 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -1284,6 +1284,8 @@ func (s *Store) sealBulkIndexesLocked(reason string) error { return sealErr } s.bulkIndexesDeferred = false + // Every dense index is back, including any an earlier window left missing. + s.setDenseIndexesMissingLocked(nil) return nil } @@ -1323,6 +1325,14 @@ func (s *Store) closeBulkConnectionLocked() error { _, syncErr := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA synchronous = %d", s.bulkPrevSync)) _, cacheErr := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA cache_size = %d", s.bulkPrevCacheSize)) _, autoCheckpointErr := conn.ExecContext(ctx, fmt.Sprintf("PRAGMA wal_autocheckpoint = %d", s.bulkPrevAutoCheckpoint)) + if s.bulkIndexesDeferred { + // The window is closing with its seal still owed: a failed seal at + // FlushBulk or Close, or an abort that never sealed. The indexes it + // dropped stay absent, so say which ones before the connection that + // can still read the schema goes. Asked after the PRAGMA restores so + // it can never delay durability coming back. + s.noteDenseIndexesMissingLocked(ctx, conn) + } closeErr := conn.Close() s.bulkIndexesDeferred = false s.bulkDeferredNodeRows = 0 @@ -1341,6 +1351,63 @@ func (s *Store) closeBulkConnectionLocked() error { ) } +// noteDenseIndexesMissingLocked records which bulkDroppableIndexes are absent +// from the schema as a cold window closes unsealed. One schema read on the +// pinned connection gives the exact set even after a partial seal; when that +// read fails every droppable index is listed instead, which only costs the +// repair a few CREATE INDEX IF NOT EXISTS that find the index already there. +// Only a cold window defers indexes (a generation window opens with +// bulkIndexesDeferred false), so this never runs for one. +func (s *Store) noteDenseIndexesMissingLocked(ctx context.Context, conn *sql.Conn) { + present, err := schemaIndexNames(ctx, conn) + var missing []string + for _, idx := range bulkDroppableIndexes { + if err != nil || !present[idx.name] { + missing = append(missing, idx.name) + } + } + s.setDenseIndexesMissingLocked(missing) + if len(missing) > 0 { + log.Printf("store_sqlite: dense indexes missing after bulk window close indexes=%s", strings.Join(missing, ",")) + } +} + +// setDenseIndexesMissingLocked replaces the missing-index record. The caller +// holds writeMu; the slice is never modified after it is published. +func (s *Store) setDenseIndexesMissingLocked(missing []string) { + if len(missing) == 0 { + s.denseIndexesMissing.Store(nil) + return + } + s.denseIndexesMissing.Store(&missing) +} + +// denseIndexesMissingNames returns a copy of the missing-index record, or nil. +func (s *storeCore) denseIndexesMissingNames() []string { + missing := s.denseIndexesMissing.Load() + if missing == nil { + return nil + } + return append([]string(nil), (*missing)...) +} + +func schemaIndexNames(ctx context.Context, conn *sql.Conn) (map[string]bool, error) { + rows, err := conn.QueryContext(ctx, `SELECT name FROM sqlite_schema WHERE type = 'index'`) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + present := make(map[string]bool) + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + present[name] = true + } + return present, rows.Err() +} + func wrapBulkRestoreError(pragma string, err error) error { if err == nil { return nil diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go new file mode 100644 index 000000000..04d45001c --- /dev/null +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -0,0 +1,97 @@ +package store_sqlite + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/zzet/gortex/internal/graph" +) + +// TestFailedSealReportsDenseIndexesMissing: a cold window that closes without +// rebuilding its dense indexes must still restore durability and close, and +// must then say which indexes it left absent instead of letting the planner +// probe read their absence as a clean store. +func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { + allDroppable := make([]string, 0, len(bulkDroppableIndexes)) + for _, idx := range bulkDroppableIndexes { + allDroppable = append(allDroppable, idx.name) + } + for _, tc := range []struct { + name string + // fail closes the window the case's way and returns the indexes it + // expects to be left missing, in declaration order. + fail func(t *testing.T, s *Store) []string + }{ + {"flush seal error", func(t *testing.T, s *Store) []string { + s.BeginBulkLoad() + if s.bulkConn == nil { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + injected := errors.New("injected seal failure") + defer s.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" || index == "edges_by_kind" { + return injected + } + return nil + })() + err := s.FlushBulk() + if !errors.Is(err, injected) { + t.Fatalf("FlushBulk error = %v, want the injected seal failure", err) + } + for _, name := range []string{"nodes_by_kind", "edges_by_kind"} { + if !strings.Contains(err.Error(), name) { + t.Fatalf("FlushBulk error %q does not name %s", err, name) + } + } + return []string{"nodes_by_kind", "edges_by_kind"} + }}, + {"abort before seal", func(t *testing.T, s *Store) []string { + if !s.BeginCoordinatedBulkLoad() { + t.Fatal("coordinated fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + if err := s.AbortCoordinatedBulkLoad(); err != nil { + t.Fatalf("AbortCoordinatedBulkLoad: %v", err) + } + return allDroppable + }}, + } { + t.Run(tc.name, func(t *testing.T) { + s, _ := openTempStore(t) + syncBefore := pragmaIntDB(t, s.writerDB, "synchronous") + want := tc.fail(t, s) + + if s.bulkConn != nil || s.coordinatedBulkLoad || s.bulkWindowOpen.Load() { + t.Fatal("window left open after closing unsealed") + } + if got := pragmaIntDB(t, s.writerDB, "synchronous"); got != syncBefore { + t.Fatalf("writer synchronous = %d, want restored %d", got, syncBefore) + } + health, err := s.PlannerStatsHealth(context.Background()) + if err != nil { + t.Fatalf("PlannerStatsHealth: %v", err) + } + if health.Reason != plannerStatsDenseIndexesMissingReason || health.Stale { + t.Fatalf("health reason=%q stale=%v, want %q and not stale", health.Reason, health.Stale, plannerStatsDenseIndexesMissingReason) + } + if !reflect.DeepEqual(health.DenseIndexesMissing, want) { + t.Fatalf("DenseIndexesMissing = %v, want %v", health.DenseIndexesMissing, want) + } + present := indexNames(t, s.db) + for _, name := range allDroppable { + missing := false + for _, w := range want { + missing = missing || w == name + } + if present[name] == missing { + t.Fatalf("index %s present=%v, want present=%v", name, present[name], !missing) + } + } + integrityOK(t, s.db) + }) + } +} diff --git a/internal/graph/store_sqlite/planner_stats_freshness.go b/internal/graph/store_sqlite/planner_stats_freshness.go index 86dbae87b..193aa2aef 100644 --- a/internal/graph/store_sqlite/planner_stats_freshness.go +++ b/internal/graph/store_sqlite/planner_stats_freshness.go @@ -150,6 +150,14 @@ const ( // finalize anyway. const plannerStatsBulkWindowReason = "bulk_window_active" +// plannerStatsDenseIndexesMissingReason is reported instead of a verdict while +// a cold bulk window that closed unsealed still owes dense indexes (listed in +// PlannerStatsFreshness.DenseIndexesMissing). Some of the critical indexes may +// be among them, and every rule would read an absent index as a clean store, +// so the probe names the real problem instead. No ANALYZE can fix it: the +// maintenance lane's repair rebuilds the indexes and the reason clears with it. +const plannerStatsDenseIndexesMissingReason = "dense_indexes_missing" + // Reasons a stale verdict can come back UNREFRESHED. None of them is a // failure, and none of them stamps the ledger or arms the anti-loop guard: the // verdict stands, the resume cursor keeps the indexes already finished, and the @@ -620,6 +628,13 @@ func (s *Store) plannerStatsHealth(ctx context.Context, probeReceivers bool) (gr health.Reason = plannerStatsBulkWindowReason return health, bases, nil } + // Same shape as the window above: not a judgeable state, Stale stays + // false and Known stays false, so nothing is seeded from it. + if missing := s.denseIndexesMissingNames(); len(missing) > 0 { + health.Reason = plannerStatsDenseIndexesMissingReason + health.DenseIndexesMissing = missing + return health, bases, nil + } present, err := s.plannerStatsPresentIndexes(ctx) if err != nil { diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index 816ed8ae4..77b0d14ad 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -505,6 +505,14 @@ type storeCore struct { // health probe. Written only under writeMu, by syncBulkWindowLocked. bulkWindowOpen atomic.Bool + // denseIndexesMissing names the dense indexes a cold bulk window closed + // without rebuilding (its seal failed, or it was aborted before sealing), + // in bulkDroppableIndexes order; nil when none are owed. Written only + // under writeMu and replaced, never mutated in place, so the health probe + // reads it without a lock for the same reason it reads bulkWindowOpen. + // Not persisted: Open's createGraphCoreIndexes rebuilds them on restart. + denseIndexesMissing atomic.Pointer[[]string] + // Prepared statements (compiled once in Open, closed in Close). stmtInsertNode *sql.Stmt stmtGetNode *sql.Stmt From 70c23001fb67e618e4105f0a012ee08b944a1dba Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:09:25 +0100 Subject: [PATCH 03/60] feat(sqlite): repair missing dense indexes on the maintenance lane A cold window that closed unsealed now records which dense indexes it left missing, but nothing rebuilt them before the next restart, so a long-running daemon kept serving lookups without them. The flush and abort exits now post a dense_index_repair job after they release the write gate. The lane worker runs it after any owed WAL drain and before the statistics pass, as a quiescing whole-database job. Each attempt first checks free space on the store's volume and skips while it is below the store's used bytes (64 MiB floor): a rebuild needs about twice the index size at its peak and the dense set is a fraction of the used pages, so a full volume is never asked to rebuild while a volume with the store's own size free always can. A skipped or failed attempt re-posts itself after 30 s, doubling up to 15 min. The rebuild runs the same DDL as createGraphCoreIndexes (both come from bulkDroppableIndexes), but takes the write gate once per index instead of once for the set, so a writer waits behind at most one CREATE INDEX. Like VACUUM it defers while a bulk window pins the writer. A completed repair clears the record and asks the lane for a statistics pass for the rebuilt indexes. The job stays out of the lane's settle point so a long rebuild cannot hold AwaitMaintenanceIdle callers. TestFailedSealReportsDenseIndexesMissing now also checks that no rebuild is attempted while the volume reads as full, and that one attempt rebuilds every listed index and clears the reason once space is back. --- internal/graph/store_sqlite/bulk_load.go | 29 ++- .../graph/store_sqlite/dense_index_repair.go | 229 ++++++++++++++++++ .../store_sqlite/dense_index_repair_test.go | 69 +++++- internal/graph/store_sqlite/store.go | 16 ++ internal/graph/store_sqlite/store_compact.go | 15 +- 5 files changed, 347 insertions(+), 11 deletions(-) create mode 100644 internal/graph/store_sqlite/dense_index_repair.go diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 2a27c3c92..2b3414b2e 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -939,6 +939,9 @@ type bulkWindowExit struct { // when no such PASSIVE ran. residue walCheckpointResult residueBoundary string + // repairDenseIndexes: the window closed with its seal still owed, so it + // recorded missing dense indexes and the maintenance lane owes a repair. + repairDenseIndexes bool } // finishBulkWindowExit runs what exit owes after writeMu is released, in the @@ -949,10 +952,14 @@ func (s *Store) finishBulkWindowExit(exit bulkWindowExit) error { if exit.residueBoundary != "" { s.scheduleWALDrainAboveLine(exit.residue, exit.residueBoundary) } + var err error if exit.durabilityCheckpoint { - return s.checkpointBulkWAL() + err = s.checkpointBulkWAL() } - return nil + if exit.repairDenseIndexes { + s.scheduleDenseIndexRepair() + } + return err } // FlushBulk exits the bulk-load fast path: it rebuilds every dropped index, @@ -1002,9 +1009,10 @@ func (s *Store) flushBulkLocked() (bulkWindowExit, error) { } hadBulk := s.bulkConn != nil sealErr := s.sealBulkIndexesLocked("flush") + repair := s.bulkIndexesDeferred closeErr := s.closeBulkConnectionLocked() s.jsonbIngestBuffers.release() - return bulkWindowExit{durabilityCheckpoint: hadBulk}, errors.Join(sealErr, closeErr) + return bulkWindowExit{durabilityCheckpoint: hadBulk, repairDenseIndexes: repair}, errors.Join(sealErr, closeErr) } // EndCoordinatedBulkLoad closes an outer multi-repository cold-load window. @@ -1128,16 +1136,23 @@ func (s *Store) endCoordinatedBulkLoadLocked() (bulkWindowExit, error) { // AbortCoordinatedBulkLoad gives up retryable finalization and restores the // writer connection unconditionally. It is for terminal owners only: dense -// indexes whose rebuild failed may remain absent until normal schema repair or -// restart, but the store no longer holds synchronous=OFF, disabled automatic -// checkpoints, or a pinned writer indefinitely. +// indexes whose rebuild failed stay absent, recorded as missing, until the +// maintenance lane's repair or a restart rebuilds them, but the store no +// longer holds synchronous=OFF, disabled automatic checkpoints, or a pinned +// writer indefinitely. func (s *Store) AbortCoordinatedBulkLoad() error { + exit, err := s.abortCoordinatedBulkLoadLocked() + return errors.Join(err, s.finishBulkWindowExit(exit)) +} + +func (s *Store) abortCoordinatedBulkLoadLocked() (bulkWindowExit, error) { s.writeMu.Lock() defer s.writeMu.Unlock() s.coordinatedBulkLoad = false + repair := s.bulkIndexesDeferred err := s.closeBulkConnectionLocked() s.jsonbIngestBuffers.release() - return err + return bulkWindowExit{repairDenseIndexes: repair}, err } // noteBulkRowsLocked advances independent index-seal and WAL-checkpoint budgets diff --git a/internal/graph/store_sqlite/dense_index_repair.go b/internal/graph/store_sqlite/dense_index_repair.go new file mode 100644 index 000000000..5fe5b9d53 --- /dev/null +++ b/internal/graph/store_sqlite/dense_index_repair.go @@ -0,0 +1,229 @@ +package store_sqlite + +import ( + "context" + "fmt" + "log" + "math" + "path/filepath" + "strings" + "time" + + "github.com/zzet/gortex/internal/platform" +) + +// Dense-index repair: the maintenance-lane job that rebuilds the dense indexes +// a cold bulk window closed without (denseIndexesMissing). The usual cause is +// the volume filling up during the seal, so the job is built around not +// making that worse: +// +// - it never runs inline. The exit that recorded the missing indexes posts +// one request after it has released the write gate, and the lane worker +// runs it as a quiescing whole-database job, like VACUUM. +// - it does not try while the disk is still full. An attempt is skipped +// while the free space on the store's volume is below the store's used +// bytes (with a floor of denseIndexRepairMinFreeBytes). The dense indexes +// are a fraction of the used pages, and a CREATE INDEX needs about twice +// the index size at its peak (the sort plus the WAL frames), so a volume +// with the store's own size free can always take the rebuild while a full +// one is never asked to. +// - it backs off. A skipped or failed attempt re-posts itself after +// denseIndexRepairBaseDelay, doubling up to denseIndexRepairMaxDelay, +// so a volume that stays full costs one statfs per backoff step and one +// log line, not a retry loop. +// - it holds the write gate one index at a time. The DDL is exactly what +// createGraphCoreIndexes runs (bulkDroppableIndexes is the single source +// for both), but each CREATE INDEX takes and releases writeMu on its own, +// so a writer waits behind at most one index build rather than the set. +// +// It is deliberately not part of the lane's settle point (waitMaintenanceIdle): +// a long rebuild must not hold AwaitMaintenanceIdle callers, which sit on +// publication paths. A completed repair asks the lane for a statistics pass, +// because the rebuilt indexes have no sqlite_stat1 rows yet. + +// Vars rather than constants only so the in-package cases can shorten them. +var ( + denseIndexRepairBaseDelay = 30 * time.Second + denseIndexRepairMaxDelay = 15 * time.Minute + denseIndexRepairMinFreeBytes = int64(64 << 20) +) + +// scheduleDenseIndexRepair posts one immediate repair request, resetting any +// backoff an earlier record had built up. The caller has just recorded the +// missing set and must not hold writeMu; like scheduleWALDrain it takes only +// maintenanceSched and never blocks. +func (s *Store) scheduleDenseIndexRepair() { + if s.coreless() { + return + } + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal == nil { + return + } + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + if s.denseRepairOwed { + return + } + s.denseRepairOwed = true + select { + case s.maintenanceSignal <- struct{}{}: + default: + } +} + +// takeDenseIndexRepair claims an owed repair for the lane worker. +func (s *Store) takeDenseIndexRepair() bool { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if !s.denseRepairOwed || s.maintenanceClosed { + return false + } + s.denseRepairOwed = false + return true +} + +// retryDenseIndexRepairLater re-posts the repair once the current backoff +// elapses and doubles the backoff for the attempt after it. It returns the +// delay it armed, or 0 when the lane is closed. +func (s *Store) retryDenseIndexRepairLater() time.Duration { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal == nil { + return 0 + } + delay := s.denseRepairDelay + if delay <= 0 { + delay = denseIndexRepairBaseDelay + } + s.denseRepairDelay = min(2*delay, denseIndexRepairMaxDelay) + s.stopDenseIndexRepairTimerLocked() + signal := s.maintenanceSignal + s.denseRepairTimer = time.AfterFunc(delay, func() { + s.maintenanceSched.Lock() + defer s.maintenanceSched.Unlock() + if s.maintenanceClosed || s.maintenanceSignal != signal || s.denseRepairOwed { + return + } + s.denseRepairOwed = true + select { + case signal <- struct{}{}: + default: + } + }) + return delay +} + +// stopDenseIndexRepairTimerLocked cancels a pending retry. The caller holds +// maintenanceSched. +func (s *Store) stopDenseIndexRepairTimerLocked() { + if s.denseRepairTimer != nil { + s.denseRepairTimer.Stop() + s.denseRepairTimer = nil + } +} + +// runDenseIndexRepair is one attempt, run by the lane worker. Nothing it hits +// is an error to anyone: the record stays, the attempt is logged, and the +// backoff decides when the next one comes. +func (s *Store) runDenseIndexRepair(ctx context.Context) { + missing := s.denseIndexesMissingNames() + if len(missing) == 0 { + // A later seal already rebuilt them. + s.maintenanceSched.Lock() + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + s.maintenanceSched.Unlock() + return + } + avail, need, ok := s.denseIndexRepairSpace() + if !ok { + s.denseIndexRepairSkips.Add(1) + delay := s.retryDenseIndexRepairLater() + log.Printf("store_sqlite: dense index repair skipped: low free space avail_bytes=%d need_bytes=%d indexes=%s retry_in=%s", avail, need, strings.Join(missing, ","), delay) + return + } + started := time.Now() + err := s.runMaintenance(ctx, maintenanceDenseIndexRepair, true, func(ctx context.Context) error { + s.denseIndexRepairAttempts.Add(1) + return s.repairDenseIndexes(ctx) + }) + if err != nil { + if ctx.Err() != nil { + return + } + delay := s.retryDenseIndexRepairLater() + log.Printf("store_sqlite: dense index repair deferred indexes=%s retry_in=%s error=%q", strings.Join(s.denseIndexesMissingNames(), ","), delay, err) + return + } + s.maintenanceSched.Lock() + s.stopDenseIndexRepairTimerLocked() + s.denseRepairDelay = 0 + s.maintenanceSched.Unlock() + s.denseIndexRepairs.Add(1) + log.Printf("store_sqlite: dense index repair complete indexes=%s elapsed=%s", strings.Join(missing, ","), time.Since(started).Round(time.Millisecond)) + // The rebuilt indexes carry no statistics yet; the lane's next pass + // notices that and analyzes them. + s.schedulePublishMaintenance() +} + +// denseIndexRepairSpace reports the free space on the store's volume, the +// amount an attempt needs, and whether the attempt may run. An unreadable +// free-space figure is a skip: the gate exists for the full volume, which is +// exactly where a probe may fail too. +func (s *Store) denseIndexRepairSpace() (avail, need int64, ok bool) { + free, total := s.CompactStats() + need = max(total-free, denseIndexRepairMinFreeBytes) + probe := platform.DiskAvailBytes + if override := s.denseRepairDiskAvail.Load(); override != nil && *override != nil { + probe = *override + } + bytes, err := probe(filepath.Dir(s.dbPath)) + if err != nil { + return -1, need, false + } + avail = int64(min(bytes, uint64(math.MaxInt64))) + return avail, need, avail >= need +} + +// repairDenseIndexes rebuilds every index in the missing record, one write-gate +// hold per index. It refuses, like VACUUM, while a bulk window pins the writer: +// that window drops and seals the same indexes itself. +func (s *Store) repairDenseIndexes(ctx context.Context) error { + for _, name := range s.denseIndexesMissingNames() { + if err := s.repairDenseIndex(ctx, name); err != nil { + return err + } + } + return nil +} + +func (s *Store) repairDenseIndex(ctx context.Context, name string) error { + if err := s.writeMu.LockContext(ctx); err != nil { + return fmt.Errorf("%w: %s: write gate: %w", ErrMaintenanceBusy, maintenanceDenseIndexRepair, err) + } + defer s.writeMu.Unlock() + if s.bulkConn != nil { + return fmt.Errorf("%w: %s: bulk writer pinned", ErrMaintenanceBusy, maintenanceDenseIndexRepair) + } + for _, idx := range bulkDroppableIndexes { + if idx.name != name { + continue + } + if _, err := s.writerDB.ExecContext(ctx, idx.ddl); err != nil { + return fmt.Errorf("store_sqlite: repair index %s: %w", name, err) + } + break + } + // Under the hold that built it, so whichever window recorded the name, the + // index it names is present now. + var remaining []string + for _, missing := range s.denseIndexesMissingNames() { + if missing != name { + remaining = append(remaining, missing) + } + } + s.setDenseIndexesMissingLocked(remaining) + return nil +} diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go index 04d45001c..94ed711ef 100644 --- a/internal/graph/store_sqlite/dense_index_repair_test.go +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -5,16 +5,29 @@ import ( "errors" "reflect" "strings" + "sync/atomic" "testing" + "time" "github.com/zzet/gortex/internal/graph" ) +// shortenDenseIndexRepairBackoff makes a skipped repair retry within +// milliseconds for the rest of the test. +func shortenDenseIndexRepairBackoff(t *testing.T) { + t.Helper() + base, maxDelay := denseIndexRepairBaseDelay, denseIndexRepairMaxDelay + denseIndexRepairBaseDelay, denseIndexRepairMaxDelay = 5*time.Millisecond, 20*time.Millisecond + t.Cleanup(func() { denseIndexRepairBaseDelay, denseIndexRepairMaxDelay = base, maxDelay }) +} + // TestFailedSealReportsDenseIndexesMissing: a cold window that closes without -// rebuilding its dense indexes must still restore durability and close, and -// must then say which indexes it left absent instead of letting the planner -// probe read their absence as a clean store. +// rebuilding its dense indexes must still restore durability and close, must +// say which indexes it left absent instead of letting the planner probe read +// their absence as a clean store, must not retry the rebuild while the volume +// is full, and must rebuild them in one maintenance attempt once it is not. func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { + shortenDenseIndexRepairBackoff(t) allDroppable := make([]string, 0, len(bulkDroppableIndexes)) for _, idx := range bulkDroppableIndexes { allDroppable = append(allDroppable, idx.name) @@ -62,6 +75,15 @@ func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) syncBefore := pragmaIntDB(t, s.writerDB, "synchronous") want := tc.fail(t, s) @@ -91,7 +113,48 @@ func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { t.Fatalf("index %s present=%v, want present=%v", name, present[name], !missing) } } + + // While the volume reads as full the lane keeps skipping on its + // backoff and never reaches a CREATE INDEX. + waitUntil(t, "a repair attempt skipped for free space", func() bool { return s.denseIndexRepairSkips.Load() >= 2 }) + if got := s.denseIndexRepairAttempts.Load(); got != 0 { + t.Fatalf("repair attempts while full = %d, want 0", got) + } + if health, err := s.PlannerStatsHealth(context.Background()); err != nil || health.Reason != plannerStatsDenseIndexesMissingReason { + t.Fatalf("health while full = %q, %v; want %q", health.Reason, err, plannerStatsDenseIndexesMissingReason) + } + + // Once space is back, the next retry rebuilds every listed index. + full.Store(false) + waitUntil(t, "the dense index repair", func() bool { return s.denseIndexRepairs.Load() == 1 }) + if got := s.denseIndexRepairAttempts.Load(); got != 1 { + t.Fatalf("repair attempts = %d, want one", got) + } + health, err = s.PlannerStatsHealth(context.Background()) + if err != nil { + t.Fatalf("PlannerStatsHealth after repair: %v", err) + } + if health.Reason == plannerStatsDenseIndexesMissingReason || len(health.DenseIndexesMissing) != 0 { + t.Fatalf("after repair reason=%q missing=%v, want both cleared", health.Reason, health.DenseIndexesMissing) + } + present = indexNames(t, s.db) + for _, name := range allDroppable { + if !present[name] { + t.Fatalf("index %s missing after the repair", name) + } + } integrityOK(t, s.db) }) } } + +func waitUntil(t *testing.T, what string, done func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for !done() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(time.Millisecond) + } +} diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index 77b0d14ad..c171c74cd 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -339,6 +339,22 @@ type storeCore struct { maintenanceDrainRunning bool maintenanceDrainReason string + // Dense-index repair (dense_index_repair.go). denseRepairOwed is the + // one-slot request the lane worker consumes, posted by a bulk-window exit + // that left indexes missing or by the backoff timer; denseRepairDelay is + // the backoff the next retry waits and denseRepairTimer that pending + // retry. All three are guarded by maintenanceSched. The counters record + // attempts that reached their DDL, attempts skipped for free space, and + // completed repairs; denseRepairDiskAvail is a test override of the + // free-space probe. + denseRepairOwed bool + denseRepairDelay time.Duration + denseRepairTimer *time.Timer + denseIndexRepairAttempts atomic.Int64 + denseIndexRepairSkips atomic.Int64 + denseIndexRepairs atomic.Int64 + denseRepairDiskAvail atomic.Pointer[func(dir string) (uint64, error)] + // Lane counters. Requests counts scheduling calls, passes counts lane // passes actually started (so a burst of requests collapsing into one // pass is observable), jobs counts actions that reached their SQL, diff --git a/internal/graph/store_sqlite/store_compact.go b/internal/graph/store_sqlite/store_compact.go index a114768ab..56e482bd6 100644 --- a/internal/graph/store_sqlite/store_compact.go +++ b/internal/graph/store_sqlite/store_compact.go @@ -92,6 +92,9 @@ const ( maintenancePlannerStats maintenanceJob = "planner_stats" maintenanceVacuum maintenanceJob = "vacuum" maintenanceCheckpoint maintenanceJob = "wal_checkpoint_truncate" + // maintenanceDenseIndexRepair rebuilds the dense indexes an unsealed cold + // window left missing (dense_index_repair.go). + maintenanceDenseIndexRepair maintenanceJob = "dense_index_repair" ) // preemptible reports whether a job in flight may be asked to yield the lane. @@ -641,8 +644,16 @@ func (s *Store) runMaintenanceLane(ctx context.Context, signal <-chan struct{}, return } } + // Ahead of the pass for the same reason as the drain, and because a + // completed repair owes the pass that analyzes what it rebuilt. + if s.takeDenseIndexRepair() { + s.runDenseIndexRepair(ctx) + if ctx.Err() != nil { + return + } + } if !s.statsPassOwed() { - // The wakeup carried a drain and nothing else. + // The wakeup carried a drain or a repair and nothing else. continue } if !s.claimMaintenancePass() { @@ -732,6 +743,8 @@ func (s *Store) stopMaintenanceLane() { s.maintenanceClosed = true s.maintenanceOwed = false s.maintenanceDrainOwed = false + s.denseRepairOwed = false + s.stopDenseIndexRepairTimerLocked() cancel := s.maintenanceCancel done := s.maintenanceDone s.maintenanceSched.Unlock() From 5efb337d699566ccc4eb3272bf3d7b253825b982 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:11:44 +0100 Subject: [PATCH 04/60] fix(mcp): list missing dense indexes in index_health index_health already reports the planner verdict's reason, so a store that lost dense indexes to an interrupted bulk load would show dense_indexes_missing with no way to tell which lookups now scan. The planner_stats block now carries dense_indexes_missing with the index names, and the recommendation says what they cost and how they come back (the background repair once the volume has room, or a restart). The subtest fixes the verdict through a wrapper over the real store, because the real state repairs itself on the maintenance lane as soon as the test volume has room and could not be observed without a race. --- .../mcp/index_health_planner_stats_test.go | 34 +++++++++++++++++++ internal/mcp/tools_enhancements.go | 13 +++++++ 2 files changed, 47 insertions(+) diff --git a/internal/mcp/index_health_planner_stats_test.go b/internal/mcp/index_health_planner_stats_test.go index b89263823..0245957bc 100644 --- a/internal/mcp/index_health_planner_stats_test.go +++ b/internal/mcp/index_health_planner_stats_test.go @@ -199,6 +199,30 @@ func TestIndexHealth_SurfacesPlannerStats(t *testing.T) { "published a receiver cardinality for an index that is not in the schema") }) + // Dense indexes an interrupted bulk load left out are named, with the + // sentence that says what they cost and how they come back. The real + // state repairs itself on the store's maintenance lane as soon as the + // volume has room, so the verdict is fixed here rather than raced. + t.Run("missing dense indexes are named", func(t *testing.T) { + _, store := newEmptySQLiteBackedServer(t) + missing := []string{"nodes_by_kind", "edges_by_kind"} + wrapped := plannerHealthOverride{Store: store, health: graph.PlannerStatsFreshness{ + Reason: "dense_indexes_missing", DenseIndexesMissing: missing, + }} + idx := indexer.New(wrapped, testRegistry(), config.Default().Index, zap.NewNop()) + missingSrv := NewServer(query.NewEngine(wrapped), wrapped, idx, nil, zap.NewNop(), nil) + + payload, err := missingSrv.buildIndexHealthPayloadCtx(context.Background()) + require.NoError(t, err) + stats, ok := payload["planner_stats"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "dense_indexes_missing", stats["reason"]) + assert.Equal(t, missing, stats["dense_indexes_missing"]) + recommendation, _ := payload["recommendation"].(string) + assert.Contains(t, recommendation, "nodes_by_kind, edges_by_kind") + assert.NotContains(t, recommendation, "planner statistics are stale") + }) + // The window between a deferred cooperative pass and the boundary that // finishes it, seen from the side an agent reads. // @@ -304,6 +328,16 @@ func TestIndexHealth_SurfacesPlannerStats(t *testing.T) { }) } +// plannerHealthOverride reports a fixed planner verdict over a real store. +type plannerHealthOverride struct { + *store_sqlite.Store + health graph.PlannerStatsFreshness +} + +func (s plannerHealthOverride) PlannerStatsHealth(context.Context) (graph.PlannerStatsFreshness, error) { + return s.health, nil +} + // A backend without the capability must omit the key rather than publish a // struct of zeros, which would read as "the planner believes nothing". func TestIndexHealth_PlannerStatsAbsentOnBackendWithoutCapability(t *testing.T) { diff --git a/internal/mcp/tools_enhancements.go b/internal/mcp/tools_enhancements.go index 6dc75a5ef..6d473e205 100644 --- a/internal/mcp/tools_enhancements.go +++ b/internal/mcp/tools_enhancements.go @@ -3694,6 +3694,19 @@ func (s *Server) buildIndexHealthBasePayloadCtx(ctx context.Context) (map[string plannerStats["last_refresh_at"] = planner.LastRefreshAt.UTC().Format(time.RFC3339) plannerStats["last_refresh_reason"] = planner.LastRefreshReason } + // Named, not counted: they are what an operator checks the schema for, + // and the reason above alone does not say which lookups are scanning. + if len(planner.DenseIndexesMissing) > 0 { + plannerStats["dense_indexes_missing"] = planner.DenseIndexesMissing + msg := "A bulk index load ended without rebuilding these SQLite indexes: " + strings.Join(planner.DenseIndexesMissing, ", ") + + ". Queries that use them scan instead until they are back. The store rebuilds them in the background once " + + "its volume has free space at least the size of the store, and a daemon restart rebuilds them too." + if recommendation == "" { + recommendation = msg + } else { + recommendation = msg + " " + recommendation + } + } result["planner_stats"] = plannerStats if planner.Stale { msg := "SQLite planner statistics are stale (" + planner.Reason + "): the graph has grown past what the query " + From 37462e09b54de7226a77d34be690a04337847c6b Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:16:54 +0100 Subject: [PATCH 05/60] fix(indexer): write drain batches through the checked store path drainAddBatch wrote every shadow chunk through the void AddBatch, so a storage failure during a first-index drain (SQLITE_FULL, an I/O error, a failed deferred-index seal at a row limit) panicked out of the drain instead of failing the build, and only the backstop flush stood between that panic and a bulk window left open. The drain now writes through AddBatchCheckedContext when the target has it and falls back to AddBatch otherwise. The write runs under context.WithoutCancel: cancellation is still observed only between sub-batches, so an ordinary drain never stops inside a chunk. The node loop now stops at the first refused write (the edge loop already did), and the result names a refused write ("persist bulk graph batch") apart from a cooperative stop. The builtin re-assert after the edge drain also goes through the checked write, and only while the drain is still succeeding: a failed drain publishes nothing and is evicted before its retry. One visible change: errors the void AddBatch used to swallow (a store closed under the drain) now fail the build instead of dropping rows. SetBatchWriteHookForTest is a per-store hook every AddBatch-family write calls inside its transaction, so a test can make the store refuse a batch with a real driver error. TestDrainAddBatchReturnsStorageFailure hands it a genuine SQLITE_FULL and checks both drain shapes return a *StorageError with nothing committed. The first-index probe intercepts the checked write as well as AddBatch. --- internal/graph/store_sqlite/add_batch_set.go | 23 +++++++ internal/graph/store_sqlite/store.go | 8 ++- internal/indexer/drain_lane_yield.go | 45 ++++++++++++-- internal/indexer/first_index_bulk_test.go | 63 +++++++++++++++++++- internal/indexer/indexer.go | 21 +++++-- 5 files changed, 147 insertions(+), 13 deletions(-) diff --git a/internal/graph/store_sqlite/add_batch_set.go b/internal/graph/store_sqlite/add_batch_set.go index c9dfed5d3..89f7f9640 100644 --- a/internal/graph/store_sqlite/add_batch_set.go +++ b/internal/graph/store_sqlite/add_batch_set.go @@ -551,6 +551,26 @@ func insertEdgeChunksTxLimitedContext(ctx context.Context, tx *sql.Tx, viewGen i return rowsInserted, statements, insertedKeys, nil } +// SetBatchWriteHookForTest installs a hook every AddBatch-family write on this +// store calls inside its transaction, under the write gate and before any row +// is written, with the batch's node and edge counts. A returned error fails the +// batch exactly as a failed insert would (the transaction rolls back and the +// error reaches the caller the way that caller reports storage errors); a panic +// unwinds the batch the same way. The hook must not call back into the store. +// restore puts the previous hook back. +func (s *Store) SetBatchWriteHookForTest(hook func(nodeRows, edgeRows int) error) (restore func()) { + previous := s.batchWriteHook.Swap(&hook) + return func() { s.batchWriteHook.Store(previous) } +} + +func (s *Store) runBatchWriteHook(nodeRows, edgeRows int) error { + hook := s.batchWriteHook.Load() + if hook == nil || *hook == nil { + return nil + } + return (*hook)(nodeRows, edgeRows) +} + func (s *Store) addBatchSetOriented(nodes []*graph.Node, edges []*graph.Edge) (sqliteAddBatchStats, error) { return s.addBatchSetOrientedContext(context.Background(), nodes, edges) } @@ -708,6 +728,9 @@ func (s *Store) addBatchSetOrientedContext(ctx context.Context, nodes []*graph.N } defer owner.finish(&err) tx := owner.tx + if err := s.runBatchWriteHook(len(nodes), len(edges)); err != nil { + return stats, err + } var ( receiptDelta *sqliteMutationReceiptAccumulator diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index c171c74cd..af331b37d 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -450,9 +450,11 @@ type storeCore struct { // bulkFinalizeObserver is a package-private test/diagnostic hook. It runs // synchronously under writeMu and therefore must not call back into Store. bulkFinalizeObserver func(bulkFinalizeEvent) - // bulkSealHook is SetBulkSealHookForTest's hook. Per store rather than - // package-wide so parallel tests that each own a store do not share it. - bulkSealHook atomic.Pointer[func(index string) error] + // bulkSealHook and batchWriteHook are SetBulkSealHookForTest's and + // SetBatchWriteHookForTest's hooks. Per store rather than package-wide so + // parallel tests that each own a store do not share them. + bulkSealHook atomic.Pointer[func(index string) error] + batchWriteHook atomic.Pointer[func(nodeRows, edgeRows int) error] // Planner-statistics freshness ledger (planner_stats_freshness.go). It // lives on the core rather than the handle so every AtGeneration view diff --git a/internal/indexer/drain_lane_yield.go b/internal/indexer/drain_lane_yield.go index df25c84b8..8d7699458 100644 --- a/internal/indexer/drain_lane_yield.go +++ b/internal/indexer/drain_lane_yield.go @@ -2,6 +2,8 @@ package indexer import ( "context" + "errors" + "fmt" "time" "github.com/zzet/gortex/internal/graph" @@ -87,11 +89,13 @@ func drainYieldable(ctx context.Context) bool { // claimed drain uses sub-batches and stops on context cancellation. The initial // claimed drain also stands down, boundedly, for interactive demand before each // slice; ordinary and foreground drains keep their single-batch behavior. +// +// A cooperative stop returns the context's own error; anything else is the +// store refusing a write (see drainWrite). drainBatchError tells them apart. func drainAddBatch(ctx context.Context, target graph.Store, nodes []*graph.Node, edges []*graph.Edge) error { wanted := initialDrainDemand(ctx) if !drainYieldable(ctx) && wanted == nil { - target.AddBatch(nodes, edges) - return nil + return drainWrite(ctx, target, nodes, edges) } step := drainYieldRows if step <= 0 { @@ -110,13 +114,46 @@ func drainAddBatch(ctx context.Context, target graph.Store, nodes []*graph.Node, } if len(nodes) > 0 { n := min(len(nodes), step) - target.AddBatch(nodes[:n], nil) + if err := drainWrite(ctx, target, nodes[:n], nil); err != nil { + return err + } nodes = nodes[n:] continue } n := min(len(edges), step) - target.AddBatch(nil, edges[:n]) + if err := drainWrite(ctx, target, nil, edges[:n]); err != nil { + return err + } edges = edges[n:] } return nil } + +// drainCheckedWriter is the error-returning batch write a durable backend +// offers beside the void graph.Store AddBatch. +type drainCheckedWriter interface { + AddBatchCheckedContext(ctx context.Context, nodes []*graph.Node, edges []*graph.Edge) error +} + +// drainWrite writes one batch, through the checked path when target has one, +// so a storage failure (a full volume, an I/O error, a failed deferred-index +// seal) fails the drain as an error instead of panicking out of AddBatch. The +// write itself never observes ctx's cancellation: a drain stops between +// batches, never inside one. +func drainWrite(ctx context.Context, target graph.Store, nodes []*graph.Node, edges []*graph.Edge) error { + if checked, ok := target.(drainCheckedWriter); ok { + return checked.AddBatchCheckedContext(context.WithoutCancel(ctx), nodes, edges) + } + target.AddBatch(nodes, edges) + return nil +} + +// drainBatchError wraps a drainAddBatch error for the build's result: a +// cooperative stop keeps its established wording, and a refused write says the +// graph could not be persisted. +func drainBatchError(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil && errors.Is(err, ctxErr) { + return fmt.Errorf("indexer: cooperative drain stopped: %w", err) + } + return fmt.Errorf("indexer: persist bulk graph batch: %w", err) +} diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index d6e2e0ba5..e087435b8 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -44,7 +44,7 @@ func (s *firstIndexBulkProbe) BeginBulkLoad() { s.Store.BeginBulkLoad() } -func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) { +func (s *firstIndexBulkProbe) beforeBatch() { s.batches++ if s.duringDrain != nil { s.duringDrain() @@ -52,9 +52,19 @@ func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) if s.panicDrain { panic("injected drain panic") } +} + +func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) { + s.beforeBatch() s.Store.AddBatch(nodes, edges) } +// The drain writes through the checked path when the target has one. +func (s *firstIndexBulkProbe) AddBatchCheckedContext(ctx context.Context, nodes []*graph.Node, edges []*graph.Edge) error { + s.beforeBatch() + return s.Store.AddBatchCheckedContext(ctx, nodes, edges) +} + func (s *firstIndexBulkProbe) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) error { if s.ftsErr != nil { return s.ftsErr @@ -145,6 +155,57 @@ func assertFirstIndexRestored(t *testing.T, store *store_sqlite.Store, db *sql.D // store_sqlite checks the actual writer connection instead. } +// sqliteFullError provokes a genuine SQLITE_FULL from the driver on a +// throwaway in-memory database capped at a few pages, so a test can hand a +// store the exact error a full volume produces without filling a disk. +func sqliteFullError(t *testing.T) error { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + require.NoError(t, err) + defer func() { require.NoError(t, db.Close()) }() + db.SetMaxOpenConns(1) + _, err = db.Exec(`PRAGMA max_page_count = 2`) + require.NoError(t, err) + _, err = db.Exec(`CREATE TABLE filler(v BLOB)`) + require.NoError(t, err) + _, err = db.Exec(`INSERT INTO filler VALUES (randomblob(65536))`) + require.Error(t, err) + return err +} + +// A drain chunk the store refuses comes back as a typed storage error, for an +// ordinary drain and for the initial claimed one, instead of panicking out of +// the void AddBatch, and none of the refused chunk is committed. +func TestDrainAddBatchReturnsStorageFailure(t *testing.T) { + full := sqliteFullError(t) + for _, tc := range []struct { + name string + ctx context.Context + }{ + {"ordinary", context.Background()}, + {"initial claimed", withInitialDrainCooperation(context.Background(), func() bool { return false })}, + } { + t.Run(tc.name, func(t *testing.T) { + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + defer store.SetBatchWriteHookForTest(func(int, int) error { return full })() + nodes := []*graph.Node{{ + ID: "app.py::fetch", Name: "fetch", Kind: graph.KindFunction, + RepoPrefix: "repo", FilePath: "app.py", Language: "python", + }} + + var drainErr error + require.NotPanics(t, func() { drainErr = drainAddBatch(tc.ctx, store, nodes, nil) }) + var storageErr *store_sqlite.StorageError + require.ErrorAs(t, drainErr, &storageErr) + assert.Contains(t, store_sqlite.SafeStorageFailureReason(drainErr), "SQLite code 13") + assert.Contains(t, drainBatchError(tc.ctx, drainErr).Error(), "indexer: persist bulk graph batch:") + assert.Zero(t, store.NodeCount(), "the refused chunk must roll back") + }) + } +} + func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { probe, db := newFirstIndexBulkFixture(t) indexes := firstIndexSchemaIndexes(t, db) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 64c006ea5..600d9883e 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3513,8 +3513,13 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index restampedBuiltins = append(restampedBuiltins, &copied) } } - if err := drainAddBatch(ctx, diskTarget, nodes, nil); err != nil && retErr == nil { - retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", err) + if err := drainAddBatch(ctx, diskTarget, nodes, nil); err != nil { + // A refused write fails every later batch the same way, and + // a cooperative stop is final; either ends the node drain. + if retErr == nil { + retErr = drainBatchError(ctx, err) + } + break } if retErr != nil && drainChecksCancellation(ctx) && ctx.Err() != nil { break @@ -3576,7 +3581,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index } if err := drainAddBatch(ctx, diskTarget, nil, edges); err != nil { if retErr == nil { - retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", err) + retErr = drainBatchError(ctx, err) } break } @@ -3605,8 +3610,14 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // row or no composed view can equal it. The set is one row per // distinct builtin the repository references, so the copy is // bounded by the language's builtin vocabulary, not by corpus size. - if len(restampedBuiltins) > 0 { - diskTarget.AddBatch(restampedBuiltins, nil) + // + // Only after a drain that is still succeeding: a failed one publishes + // nothing and is evicted before its retry, and after a refused write + // this one would only be refused again. + if len(restampedBuiltins) > 0 && retErr == nil { + if err := drainWrite(ctx, diskTarget, restampedBuiltins, nil); err != nil { + retErr = drainBatchError(ctx, err) + } } flushStart := time.Now() From 15323a235bb6061e622772148a58a560724e03e4 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:21:18 +0100 Subject: [PATCH 06/60] fix(indexer): retry the bulk flush once after it panics The first-index backstop skipped FlushBulk after the explicit flush had panicked, because that panic could leave the store's write gate held and re-entering would deadlock. The store now releases the gate on every exit and keeps an unfinished seal retryable, so skipping the retry only left the standalone window open: a pinned writer at synchronous=OFF with the dense indexes dropped, until the store closed. After a panicking explicit flush the backstop now calls FlushBulk once more while the panic unwinds. The retry recovers its own panic and only logs what it hit, so the original panic still reaches the caller. Under an outer coordinated or generation window FlushBulk is a store no-op, so the retry is too. TestFirstIndexFlushPanicPropagates now runs on the real store with a seal that panics once: the panic propagates, the retry seals and closes the window, and the writer's synchronous, the schema's index list, the receipts, the disk target and the memory lease are all back where they started. It inverts the old assertion that the backstop must not retry. WriterPragmaForTest reads a PRAGMA off the writer connection under a deadline, which is how indexer tests check durability restoration and a free write gate. --- internal/graph/store_sqlite/bulk_load.go | 24 ++++++++ internal/indexer/drain_lane_yield.go | 19 +++++++ internal/indexer/first_index_bulk_test.go | 69 +++++++++++++++-------- internal/indexer/indexer.go | 14 ++++- 4 files changed, 100 insertions(+), 26 deletions(-) diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 2b3414b2e..460f5d768 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -1546,6 +1546,30 @@ SELECT NOT EXISTS(SELECT 1 FROM nodes WHERE view_gen = ?) return err == nil && empty == 1 } +// WriterPragmaForTest reads an integer PRAGMA off the store's writer +// connection: the pinned bulk connection while a window holds one, the single +// writer-pool connection otherwise. PRAGMAs such as synchronous are +// connection-local, so a read-pool connection cannot show what the writer runs +// with. It takes the write gate under ctx, so a deadline also proves the gate +// is free. name must be a bare PRAGMA name. +func (s *Store) WriterPragmaForTest(ctx context.Context, name string) (int64, error) { + for _, r := range name { + if (r < 'a' || r > 'z') && r != '_' { + return 0, fmt.Errorf("store_sqlite: not a pragma name: %q", name) + } + } + if err := s.writeMu.LockContext(ctx); err != nil { + return 0, err + } + defer s.writeMu.Unlock() + conn, release, err := s.activeWriteConnLocked(ctx) + if err != nil { + return 0, err + } + defer release() + return pragmaInt(ctx, conn, name) +} + // pragmaInt reads a single-integer PRAGMA (synchronous, cache_size, wal_autocheckpoint) off the // given connection. func pragmaInt(ctx context.Context, conn *sql.Conn, pragma string) (int64, error) { diff --git a/internal/indexer/drain_lane_yield.go b/internal/indexer/drain_lane_yield.go index 8d7699458..bd6e549a0 100644 --- a/internal/indexer/drain_lane_yield.go +++ b/internal/indexer/drain_lane_yield.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "go.uber.org/zap" + "github.com/zzet/gortex/internal/graph" ) @@ -157,3 +159,20 @@ func drainBatchError(ctx context.Context, err error) error { } return fmt.Errorf("indexer: persist bulk graph batch: %w", err) } + +// retryBulkFlushAfterPanic retries a FlushBulk that panicked, once, while that +// panic is unwinding. The retry recovers its own panic and only logs what it +// hit, so the original panic is the one that reaches the caller. Under an +// outer coordinated or generation window FlushBulk is a no-op, so this is too. +func retryBulkFlushAfterPanic(bl graph.BulkLoader, logger *zap.Logger, repo string) { + defer func() { + if r := recover(); r != nil { + logger.Error("indexer: bulk flush retry after panic panicked", + zap.String("repo", repo), zap.Any("panic", r)) + } + }() + if err := bl.FlushBulk(); err != nil { + logger.Error("indexer: bulk flush retry after panic failed", + zap.String("repo", repo), zap.Error(err)) + } +} diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index e087435b8..87044eb06 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -6,8 +6,8 @@ import ( "errors" "os" "path/filepath" - "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,8 +29,6 @@ type firstIndexBulkProbe struct { duringDrain func() ftsErr error panicDrain bool - panicFlush bool - flushMu sync.Mutex begins int flushes int batches int @@ -74,15 +72,18 @@ func (s *firstIndexBulkProbe) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) func (s *firstIndexBulkProbe) FlushBulk() error { s.flushes++ - if s.panicFlush { - // Model a FlushBulk panic that leaves its writer lock held. - s.flushMu.Lock() - panic("injected flush panic") - } return s.Store.FlushBulk() } func newFirstIndexBulkFixture(t *testing.T) (*firstIndexBulkProbe, *sql.DB) { + t.Helper() + store, db := newFirstIndexStore(t) + return &firstIndexBulkProbe{Store: store}, db +} + +// newFirstIndexStore opens a fresh on-disk store for the shadow route with a +// read-only pool beside it for schema probes. +func newFirstIndexStore(t *testing.T) (*store_sqlite.Store, *sql.DB) { t.Helper() t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") t.Setenv("GORTEX_SHADOW_MAX_BYTES", "1073741824") @@ -94,7 +95,19 @@ func newFirstIndexBulkFixture(t *testing.T) (*firstIndexBulkProbe, *sql.DB) { db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?mode=ro&_pragma=synchronous(NORMAL)") require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) - return &firstIndexBulkProbe{Store: store}, db + return store, db +} + +// firstIndexWriterSynchronous reads PRAGMA synchronous off the store's writer +// connection. It takes the write gate under a deadline, so it also fails, +// rather than hangs, when a failure path leaked the gate. +func firstIndexWriterSynchronous(t *testing.T, store *store_sqlite.Store) int64 { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + sync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err, "the store's write gate is still held") + return sync } func firstIndexFixtureRepo(t *testing.T) string { @@ -340,27 +353,37 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { } func TestFirstIndexFlushPanicPropagates(t *testing.T) { - // Given a FlushBulk implementation that panics while holding its writer lock. - probe, _ := newFirstIndexBulkFixture(t) - probe.panicFlush = true - idx := newFirstIndexBulkIndexer(t, probe, "repo") + // Given a real store whose dense-index seal panics once, inside the + // drain's explicit FlushBulk. + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + seals := 0 + defer store.SetBulkSealHookForTest(func(string) error { + seals++ + if seals == 1 { + panic("injected seal panic") + } + return nil + })() + idx := newFirstIndexBulkIndexer(t, store, "repo") idx.indexMemoryAdmission = newIndexMemoryAdmissionBudget(defaultIndexMemoryAdmissionBytes) - // The probe's lock is independent of Store; close the real bulk window at cleanup. - t.Cleanup(func() { require.NoError(t, probe.Store.FlushBulk()) }) - // When the completed shadow drain flushes, it must not retry that locked call. - require.PanicsWithValue(t, "injected flush panic", func() { + // When the completed shadow drain flushes, the panic reaches the caller. + require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(context.Background(), firstIndexFixtureRepo(t)) }) - // Then the panic propagates and the backstop does not re-enter FlushBulk. - // This fixture has no failure-ledger rows, so it does not cover a real - // store whose FlushBulk panics with writeMu held while the ledger is dirty. - assert.Equal(t, 1, probe.flushes) - assert.Same(t, probe, idx.graph) + // Then the backstop retried the flush once while the panic unwound, and + // the retry sealed and closed the window: the write gate is free, the + // writer is back at its own durability, and every index is rebuilt. + assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + assert.Same(t, store, idx.graph) assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") - assertFirstIndexNoReceipts(t, probe.Store, "repo") + assertFirstIndexNoReceipts(t, store, "repo") } type firstIndexDrainPanicReporter struct{} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 600d9883e..0258f75df 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3466,13 +3466,21 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index bulkFlushPending := true bulkFlushInProgress := false defer func() { - // A FlushBulk panic may leave the store's writer lock held. - // Do not re-enter it; let the panic unwind the earlier defers. - if bulkFlushPending && !bulkFlushInProgress { + if !bulkFlushPending { + return + } + if !bulkFlushInProgress { if err := bl.FlushBulk(); err != nil && retErr == nil { retErr = fmt.Errorf("indexer: persist bulk graph: %w", err) } + return } + // The explicit FlushBulk below panicked. The store releases its + // write gate on every exit and keeps a window whose seal did not + // finish retryable, so one more flush closes it (restoring + // durability, or recording the indexes it could not rebuild) + // instead of leaving a pinned synchronous=OFF writer behind. + retryBulkFlushAfterPanic(bl, idx.logger, idx.RepoPrefix()) }() // Transfer the shadow in explicit row/byte-capped batches. The // process-wide shadow admission caps all live shadows at 1 GiB; these From 2398b83026baef634241ec5b099950f2cfde90f9 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:26:20 +0100 Subject: [PATCH 07/60] test(indexer): run first-index failure cases on the real SQLite store TestFirstIndexFailureRestoresStore ran its four cases through a wrapper around the store. The wrapper is not a *store_sqlite.Store, so the drain took the generic eviction rather than the shadow-replacement retirement a real daemon takes, and its injected faults sat outside the store's write gate, so they could not show what a failure inside the store leaves behind. The test also could not check the writer's durability, because PRAGMA synchronous is connection-local. The test now runs on the real store with seven cases: an FTS error in the drain, a cancel before the drain, a panic in the body, a panic in a drain write, a genuine SQLITE_FULL from a drain write, a seal panic, and a seal DDL error. Store faults come from the store's own seal and batch hooks. Every case checks that the write gate is free and the writer's synchronous is back, that the window is closed and the schema's index list is restored, that no receipt was written and that the indexer handed its disk target back. The seal-error case also waits for the maintenance lane to rebuild the missing index. Cases that reach the drain assert that it took the retirement branch, through a test hook beside drainSubBatchHook, and that the window was engaged when they wrote. The wrapper's FTS and drain-panic injectors are gone; the replacement test shares the FTS trigger helper. --- internal/graph/store_sqlite/add_batch_set.go | 4 +- internal/indexer/drain_lane_yield.go | 6 + internal/indexer/first_index_bulk_test.go | 134 +++++++++++++------ internal/indexer/indexer.go | 9 +- 4 files changed, 108 insertions(+), 45 deletions(-) diff --git a/internal/graph/store_sqlite/add_batch_set.go b/internal/graph/store_sqlite/add_batch_set.go index 89f7f9640..aa7299787 100644 --- a/internal/graph/store_sqlite/add_batch_set.go +++ b/internal/graph/store_sqlite/add_batch_set.go @@ -556,8 +556,8 @@ func insertEdgeChunksTxLimitedContext(ctx context.Context, tx *sql.Tx, viewGen i // is written, with the batch's node and edge counts. A returned error fails the // batch exactly as a failed insert would (the transaction rolls back and the // error reaches the caller the way that caller reports storage errors); a panic -// unwinds the batch the same way. The hook must not call back into the store. -// restore puts the previous hook back. +// unwinds the batch the same way. The hook runs under the write gate, so it +// must not call the store's write methods. restore puts the previous hook back. func (s *Store) SetBatchWriteHookForTest(hook func(nodeRows, edgeRows int) error) (restore func()) { previous := s.batchWriteHook.Swap(&hook) return func() { s.batchWriteHook.Store(previous) } diff --git a/internal/indexer/drain_lane_yield.go b/internal/indexer/drain_lane_yield.go index bd6e549a0..4e09a33d6 100644 --- a/internal/indexer/drain_lane_yield.go +++ b/internal/indexer/drain_lane_yield.go @@ -37,6 +37,12 @@ var drainYieldRows = 1024 // cooperative drain. var drainSubBatchHook func() +// shadowEvictionBranchHook, when set (tests), observes which eviction a shadow +// drain chose before its first disk write: true for the SQLite +// shadow-replacement retirement, false for the generic current-generation +// eviction. +var shadowEvictionBranchHook func(retirement bool) + type initialDrainCooperationKey struct{} // Only the initial claimed base pauses in place for announced checkout work. diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 87044eb06..344910188 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -27,8 +27,6 @@ type firstIndexBulkProbe struct { *store_sqlite.Store beforeBegin func() duringDrain func() - ftsErr error - panicDrain bool begins int flushes int batches int @@ -47,9 +45,6 @@ func (s *firstIndexBulkProbe) beforeBatch() { if s.duringDrain != nil { s.duringDrain() } - if s.panicDrain { - panic("injected drain panic") - } } func (s *firstIndexBulkProbe) AddBatch(nodes []*graph.Node, edges []*graph.Edge) { @@ -63,13 +58,6 @@ func (s *firstIndexBulkProbe) AddBatchCheckedContext(ctx context.Context, nodes return s.Store.AddBatchCheckedContext(ctx, nodes, edges) } -func (s *firstIndexBulkProbe) BatchUpsertSymbolFTS(items []graph.SymbolFTSItem) error { - if s.ftsErr != nil { - return s.ftsErr - } - return s.Store.BatchUpsertSymbolFTS(items) -} - func (s *firstIndexBulkProbe) FlushBulk() error { s.flushes++ return s.Store.FlushBulk() @@ -164,8 +152,7 @@ func assertFirstIndexRestored(t *testing.T, store *store_sqlite.Store, db *sql.D require.False(t, firstIndexBulkActive(t, store), "standalone window must be closed before restoration probes") assert.Equal(t, indexes, firstIndexSchemaIndexes(t, db)) // PRAGMA synchronous is connection-local, so this read-only pool cannot - // check writer restoration. TestBulkLoadDropsAndRebuildsIndexes in - // store_sqlite checks the actual writer connection instead. + // check writer restoration; firstIndexWriterSynchronous reads the writer. } // sqliteFullError provokes a genuine SQLITE_FULL from the driver on a @@ -301,53 +288,123 @@ func (firstIndexPanicReporter) Report(stage string, _, _ int) { } } +// firstIndexDrainFTSTrigger makes every symbol FTS append on store fail, the +// way a storage error inside the drain's FTS writes would. +func firstIndexDrainFTSTrigger(t *testing.T, store *store_sqlite.Store) { + t.Helper() + writer, err := sql.Open("sqlite", "file:"+filepath.ToSlash(store.Path())) + require.NoError(t, err) + _, err = writer.Exec(`CREATE TRIGGER fail_first_index_fts BEFORE INSERT ON symbol_fts_rowid +BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) + require.NoError(t, err) + require.NoError(t, writer.Close()) +} + +// Every way a first index can fail, on the real store: the bulk window must +// close (no leaked write gate, writer durability back, the dense indexes back), +// no success receipt may be written, and the indexer must hand its disk target +// back. Faults inside the store come from its own test hooks, so the eviction +// the drain picks is the production one for a SQLite target. func TestFirstIndexFailureRestoresStore(t *testing.T) { - for _, mode := range []string{"drain error", "cancel before drain", "panic in body", "panic in drain"} { + for _, mode := range []string{ + "drain error", "cancel before drain", "panic in body", "panic in drain", + "storage full", "seal panic", "seal error", + } { t.Run(mode, func(t *testing.T) { - probe, db := newFirstIndexBulkFixture(t) + store, db := newFirstIndexStore(t) indexes := firstIndexSchemaIndexes(t, db) - injected := errors.New("injected drain FTS error") + writerSync := firstIndexWriterSynchronous(t, store) + var evictions []bool + shadowEvictionBranchHook = func(retirement bool) { evictions = append(evictions, retirement) } + t.Cleanup(func() { shadowEvictionBranchHook = nil }) + var fault func() error + batches, engaged := 0, false + defer store.SetBatchWriteHookForTest(func(int, int) error { + batches++ + engaged = engaged || firstIndexBulkActive(t, store) + if fault != nil { + return fault() + } + return nil + })() + seals := 0 + defer store.SetBulkSealHookForTest(func(index string) error { + seals++ + switch { + case mode == "seal panic" && seals == 1: + panic("injected seal panic") + case mode == "seal error" && index == "nodes_by_kind": + return errors.New("injected seal failure") + } + return nil + })() ctx, cancel := context.WithCancel(context.Background()) defer cancel() switch mode { case "drain error": - probe.ftsErr = injected + firstIndexDrainFTSTrigger(t, store) case "panic in drain": - probe.panicDrain = true + fault = func() error { panic("injected drain panic") } + case "storage full": + full := sqliteFullError(t) + fault = func() error { return full } case "cancel before drain": ctx = progress.WithReporter(ctx, firstIndexCancelReporter{cancel: cancel}) case "panic in body": ctx = progress.WithReporter(ctx, firstIndexPanicReporter{}) } - probe.duringDrain = func() { - assert.True(t, firstIndexBulkActive(t, probe.Store), "failure fixture must exercise an engaged bulk window") - } - idx := newFirstIndexBulkIndexer(t, probe, "repo") + idx := newFirstIndexBulkIndexer(t, store, "repo") dir := firstIndexFixtureRepo(t) + switch mode { case "panic in body": assert.PanicsWithValue(t, "injected body panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) - assert.Zero(t, probe.NodeCount(), "body panic must not drain shadow nodes") - assert.Zero(t, probe.EdgeCount(), "body panic must not drain shadow edges") + assert.Zero(t, store.NodeCount(), "body panic must not drain shadow nodes") + assert.Zero(t, store.EdgeCount(), "body panic must not drain shadow edges") case "panic in drain": require.PanicsWithValue(t, "injected drain panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + case "seal panic": + require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") + case "cancel before drain": + _, err := idx.IndexCtx(ctx, dir) + require.ErrorIs(t, err, context.Canceled) case "drain error": _, err := idx.IndexCtx(ctx, dir) - require.ErrorIs(t, err, injected) - default: + require.ErrorContains(t, err, "injected drain FTS error") + case "storage full": + _, err := idx.IndexCtx(ctx, dir) + var storageErr *store_sqlite.StorageError + require.ErrorAs(t, err, &storageErr) + assert.Contains(t, store_sqlite.SafeStorageFailureReason(err), "SQLite code 13") + case "seal error": _, err := idx.IndexCtx(ctx, dir) - require.ErrorIs(t, err, context.Canceled) + require.ErrorContains(t, err, "nodes_by_kind") + // Either still owed or already rebuilt by the maintenance lane, + // which runs as soon as the window closes on a volume with room. + health, healthErr := store.PlannerStatsHealth(context.Background()) + require.NoError(t, healthErr) + if health.Reason == "dense_indexes_missing" { + assert.Equal(t, []string{"nodes_by_kind"}, health.DenseIndexesMissing) + } + deadline := time.Now().Add(10 * time.Second) + for !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } } + if mode == "cancel before drain" || mode == "panic in body" { - assert.Zero(t, probe.begins) - assert.Zero(t, probe.batches) + assert.Empty(t, evictions, "a failure before the drain must not evict") + assert.Zero(t, batches, "a failure before the drain must not write") } else { - require.Positive(t, probe.batches) - assert.Equal(t, 1, probe.begins) + assert.Equal(t, []bool{true}, evictions, "a SQLite target must take the shadow-replacement retirement") + require.Positive(t, batches, "test must observe the shadow drain") + assert.True(t, engaged, "failure fixture must exercise an engaged bulk window") } - assertFirstIndexRestored(t, probe.Store, db, indexes) - assertFirstIndexNoReceipts(t, probe.Store, "repo") - assert.Same(t, probe, idx.graph, "panic and error paths must restore disk target") + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store), "writer durability must be restored") + assertFirstIndexRestored(t, store, db, indexes) + assertFirstIndexNoReceipts(t, store, "repo") + assert.Same(t, store, idx.graph, "panic and error paths must restore disk target") }) } } @@ -411,12 +468,7 @@ func TestFirstIndexFailureRestoresSQLiteReplacement(t *testing.T) { writeFile(t, baseline, "old replacement baseline") ctx := context.Background() if mode == "drain error" { - writer, err := sql.Open("sqlite", "file:"+filepath.ToSlash(probe.Path())) - require.NoError(t, err) - _, err = writer.Exec(`CREATE TRIGGER fail_first_index_fts BEFORE INSERT ON symbol_fts_rowid -BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) - require.NoError(t, err) - require.NoError(t, writer.Close()) + firstIndexDrainFTSTrigger(t, probe.Store) } else { ctx = progress.WithReporter(ctx, firstIndexDrainPanicReporter{}) } diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 0258f75df..ae6e744be 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3438,11 +3438,16 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index restubGoModuleRepoIncoming(diskTarget, idx.RepoPrefix()) } var evictedNodes, evictedEdges int - if sqlite, ok := diskTarget.(*store_sqlite.Store); ok && sqlite.SupportsShadowReplacementRetirement() { + sqliteTarget, isSQLite := diskTarget.(*store_sqlite.Store) + retirement := isSQLite && sqliteTarget.SupportsShadowReplacementRetirement() + if shadowEvictionBranchHook != nil { + shadowEvictionBranchHook(retirement) + } + if retirement { var err error shadowReplacementStarted = true restubTrackedNpmRepoIncoming(diskTarget, idx.RepoPrefix()) - evictedNodes, evictedEdges, err = sqlite.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) + evictedNodes, evictedEdges, err = sqliteTarget.EvictRepoForShadowReplacement(ctx, idx.RepoPrefix()) if err != nil { retErr = fmt.Errorf("indexer: retire shadow replacement rows: %w", err) return From adc0ca8fdaad7fff12570b393baf2dd91594ca0a Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 14:36:10 +0100 Subject: [PATCH 08/60] test(indexer): first-index drain engages the window via TrackRepoCtx and on retry TestFirstIndexDrainUsesBulkWindow covered only IndexCtx, through a wrapper that counted BeginBulkLoad and FlushBulk calls. That shows the calls were made, not that the store's window was open while the drain wrote, and it said nothing about the multi-repository route or a retry. It now runs on the real store and watches the window from the store's batch hook: the planner probe reports bulk_window_active and the dense indexes are absent from the schema while drain batches land. Three routes: - IndexCtx on a fresh store. - TrackRepoCtx on an empty multi-repository store. - A retry: the first IndexCtx has its first edge batch refused, leaving nodes behind and no receipt. The second IndexCtx evicts them, finds the store cold again, engages the window and succeeds. Each route also checks that no receipt precedes the drain (at the eviction, just before the window opens), that the drain took the retirement branch, and that the writer's synchronous, the schema's index list and the receipts end up right. --- internal/indexer/first_index_bulk_test.go | 141 ++++++++++++++++++---- 1 file changed, 119 insertions(+), 22 deletions(-) diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 344910188..2ca45f673 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -19,13 +19,13 @@ import ( "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" "github.com/zzet/gortex/internal/progress" + "github.com/zzet/gortex/internal/search" ) // Embed the real store so the shadow and its optional persistence capabilities // stay intact. Only the drain boundaries are intercepted. type firstIndexBulkProbe struct { *store_sqlite.Store - beforeBegin func() duringDrain func() begins int flushes int @@ -34,9 +34,6 @@ type firstIndexBulkProbe struct { func (s *firstIndexBulkProbe) BeginBulkLoad() { s.begins++ - if s.beforeBegin != nil { - s.beforeBegin() - } s.Store.BeginBulkLoad() } @@ -206,29 +203,129 @@ func TestDrainAddBatchReturnsStorageFailure(t *testing.T) { } } -func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { - probe, db := newFirstIndexBulkFixture(t) - indexes := firstIndexSchemaIndexes(t, db) - probe.beforeBegin = func() { assertFirstIndexNoReceipts(t, probe.Store, "repo") } - probe.duringDrain = func() { - assert.True(t, firstIndexBulkActive(t, probe.Store), "cold shadow drain must engage standalone bulk mode") - assertFirstIndexNoReceipts(t, probe.Store, "repo") - assert.NotEqual(t, indexes, firstIndexSchemaIndexes(t, db), "dense indexes must be deferred during drain") +// firstIndexWindowWitness counts a store's batch writes, and how many of them +// landed inside an engaged bulk window with the dense indexes deferred. fail, +// when set, decides whether the store refuses a batch. +type firstIndexWindowWitness struct { + batches, engaged, deferred int + fail func(edgeRows int) error +} + +func watchFirstIndexWindow(t *testing.T, store *store_sqlite.Store, db *sql.DB, indexes []string) *firstIndexWindowWitness { + t.Helper() + w := &firstIndexWindowWitness{} + t.Cleanup(store.SetBatchWriteHookForTest(func(_, edgeRows int) error { + w.batches++ + if firstIndexBulkActive(t, store) { + w.engaged++ + } + if !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) { + w.deferred++ + } + if w.fail != nil { + return w.fail(edgeRows) + } + return nil + })) + // The eviction runs just before the window opens, so this is the last + // point at which a receipt could wrongly precede the drain. + shadowEvictionBranchHook = func(retirement bool) { + assert.True(t, retirement, "a SQLite target must take the shadow-replacement retirement") + assertFirstIndexNoReceipts(t, store, "repo") } - idx := newFirstIndexBulkIndexer(t, probe, "repo") - result, err := idx.IndexCtx(context.Background(), firstIndexFixtureRepo(t)) - require.NoError(t, err) - require.Positive(t, result.NodeCount) - require.Positive(t, probe.batches, "test must observe the shadow drain") - assert.Equal(t, 1, probe.begins) - assert.Equal(t, 1, probe.flushes) - state, found, err := probe.GetRepoIndexState("repo") + t.Cleanup(func() { shadowEvictionBranchHook = nil }) + return w +} + +func (w *firstIndexWindowWitness) assertEngaged(t *testing.T) { + t.Helper() + require.Positive(t, w.batches, "test must observe the shadow drain") + assert.Positive(t, w.engaged, "cold shadow drain must engage standalone bulk mode") + assert.Positive(t, w.deferred, "dense indexes must be deferred during drain") +} + +func assertFirstIndexReceipts(t *testing.T, store *store_sqlite.Store, result *IndexResult) { + t.Helper() + state, found, err := store.GetRepoIndexState("repo") require.NoError(t, err) require.True(t, found) assert.Equal(t, result.NodeCount, state.NodeCount) assert.Equal(t, result.EdgeCount, state.EdgeCount) - assert.Contains(t, probe.LoadFileMtimes("repo"), "app.py") - assertFirstIndexRestored(t, probe.Store, db, indexes) + assert.NotEmpty(t, store.LoadFileMtimes("repo")) +} + +// A cold first index drains through a standalone bulk window on every route +// into it, and again on a retry after a failed drain left rows behind. +func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { + t.Run("IndexCtx on a fresh store", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + + result, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), firstIndexFixtureRepo(t)) + require.NoError(t, err) + require.Positive(t, result.NodeCount) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Contains(t, store.LoadFileMtimes("repo"), "app.py") + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) + + t.Run("TrackRepoCtx on an empty multi-repo store", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "main.go"), "package app\n\nfunc Fetch(url string) string { return url }\n") + mi := NewMultiIndexer(store, newTestRegistry(), search.NewNull(), newTestConfigManager(t), zap.NewNop()) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + + result, err := mi.TrackRepoCtx(t.Context(), config.RepoEntry{Name: "repo", Path: dir}) + require.NoError(t, err) + require.Positive(t, result.NodeCount) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) + + t.Run("a retry after a failed drain engages the window again", func(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + w := watchFirstIndexWindow(t, store, db, indexes) + injected := errors.New("injected edge batch failure") + refused := false + w.fail = func(edgeRows int) error { + if edgeRows > 0 && !refused { + refused = true + return injected + } + return nil + } + dir := firstIndexFixtureRepo(t) + + // The node batches land; the first edge batch is refused. + _, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), dir) + require.ErrorIs(t, err, injected) + require.True(t, refused, "the drain never reached an edge batch") + w.assertEngaged(t) + require.Positive(t, store.NodeCount(), "the failed drain must leave rows for the retry to evict") + assertFirstIndexNoReceipts(t, store, "repo") + assertFirstIndexRestored(t, store, db, indexes) + + // The retry evicts them, which leaves the store cold again. + w.batches, w.engaged, w.deferred = 0, 0, 0 + result, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), dir) + require.NoError(t, err) + w.assertEngaged(t) + assertFirstIndexReceipts(t, store, result) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) + assertFirstIndexRestored(t, store, db, indexes) + }) } func TestSecondRepoWarmStoreNoBulkWindow(t *testing.T) { From c10c5ff6caaf599f2509d2496c32978081806fbf Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 15:48:09 +0100 Subject: [PATCH 09/60] fix(sqlite): build each dense-index repair from the DDL its window recorded The missing-index record held only names, and the maintenance lane looked each name up in bulkDroppableIndexes when it got to the repair. That list is fixed in production, but in-package tests replace it to force a seal failure and put it back when they finish. A failed seal now schedules a repair, so the lane can read the list while such a test restores it. TestBulkRebuildFailureStillRestoresCheckpointsAndLeavesWriterUsable restores the list before its deferred Close joins the lane, and nothing orders that write against the lane's read, which is a data race under -race whenever the lane takes the write gate after the test's last write. A lookup that finds no entry also dropped the name from the record as if it had been built. The window now records each missing index together with its DDL, copied from the list under the write gate as it closes, and the repair runs that DDL. The lane no longer reads the list. The existing test keeps its injection and its defer order unchanged. TestDenseIndexRepairBuildsFromTheRecordedDDL records an index whose DDL cannot succeed, restores the list before the lane may run, and checks that every attempt still runs the recorded DDL and fails, instead of reporting a repair it never made. --- internal/graph/store_sqlite/bulk_load.go | 26 +++++++--- .../graph/store_sqlite/dense_index_repair.go | 28 +++++------ .../store_sqlite/dense_index_repair_test.go | 50 +++++++++++++++++++ internal/graph/store_sqlite/store.go | 12 +++-- 4 files changed, 89 insertions(+), 27 deletions(-) diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 460f5d768..3fbbb6fcb 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -1375,21 +1375,21 @@ func (s *Store) closeBulkConnectionLocked() error { // bulkIndexesDeferred false), so this never runs for one. func (s *Store) noteDenseIndexesMissingLocked(ctx context.Context, conn *sql.Conn) { present, err := schemaIndexNames(ctx, conn) - var missing []string + var missing []bulkDroppableIndex for _, idx := range bulkDroppableIndexes { if err != nil || !present[idx.name] { - missing = append(missing, idx.name) + missing = append(missing, idx) } } s.setDenseIndexesMissingLocked(missing) if len(missing) > 0 { - log.Printf("store_sqlite: dense indexes missing after bulk window close indexes=%s", strings.Join(missing, ",")) + log.Printf("store_sqlite: dense indexes missing after bulk window close indexes=%s", strings.Join(s.denseIndexesMissingNames(), ",")) } } // setDenseIndexesMissingLocked replaces the missing-index record. The caller // holds writeMu; the slice is never modified after it is published. -func (s *Store) setDenseIndexesMissingLocked(missing []string) { +func (s *Store) setDenseIndexesMissingLocked(missing []bulkDroppableIndex) { if len(missing) == 0 { s.denseIndexesMissing.Store(nil) return @@ -1397,13 +1397,27 @@ func (s *Store) setDenseIndexesMissingLocked(missing []string) { s.denseIndexesMissing.Store(&missing) } -// denseIndexesMissingNames returns a copy of the missing-index record, or nil. +// denseIndexesMissingRecord returns a copy of the missing-index record, or nil. +func (s *storeCore) denseIndexesMissingRecord() []bulkDroppableIndex { + missing := s.denseIndexesMissing.Load() + if missing == nil { + return nil + } + return append([]bulkDroppableIndex(nil), (*missing)...) +} + +// denseIndexesMissingNames returns the names in the missing-index record, or +// nil. func (s *storeCore) denseIndexesMissingNames() []string { missing := s.denseIndexesMissing.Load() if missing == nil { return nil } - return append([]string(nil), (*missing)...) + names := make([]string, 0, len(*missing)) + for _, idx := range *missing { + names = append(names, idx.name) + } + return names } func schemaIndexNames(ctx context.Context, conn *sql.Conn) (map[string]bool, error) { diff --git a/internal/graph/store_sqlite/dense_index_repair.go b/internal/graph/store_sqlite/dense_index_repair.go index 5fe5b9d53..3752c6c83 100644 --- a/internal/graph/store_sqlite/dense_index_repair.go +++ b/internal/graph/store_sqlite/dense_index_repair.go @@ -33,8 +33,10 @@ import ( // log line, not a retry loop. // - it holds the write gate one index at a time. The DDL is exactly what // createGraphCoreIndexes runs (bulkDroppableIndexes is the single source -// for both), but each CREATE INDEX takes and releases writeMu on its own, -// so a writer waits behind at most one index build rather than the set. +// for both, copied into the record by the window that closed without it, +// so the lane never reads that list), but each CREATE INDEX takes and +// releases writeMu on its own, so a writer waits behind at most one index +// build rather than the set. // // It is deliberately not part of the lane's settle point (waitMaintenanceIdle): // a long rebuild must not hold AwaitMaintenanceIdle callers, which sit on @@ -191,15 +193,15 @@ func (s *Store) denseIndexRepairSpace() (avail, need int64, ok bool) { // hold per index. It refuses, like VACUUM, while a bulk window pins the writer: // that window drops and seals the same indexes itself. func (s *Store) repairDenseIndexes(ctx context.Context) error { - for _, name := range s.denseIndexesMissingNames() { - if err := s.repairDenseIndex(ctx, name); err != nil { + for _, idx := range s.denseIndexesMissingRecord() { + if err := s.repairDenseIndex(ctx, idx); err != nil { return err } } return nil } -func (s *Store) repairDenseIndex(ctx context.Context, name string) error { +func (s *Store) repairDenseIndex(ctx context.Context, idx bulkDroppableIndex) error { if err := s.writeMu.LockContext(ctx); err != nil { return fmt.Errorf("%w: %s: write gate: %w", ErrMaintenanceBusy, maintenanceDenseIndexRepair, err) } @@ -207,20 +209,14 @@ func (s *Store) repairDenseIndex(ctx context.Context, name string) error { if s.bulkConn != nil { return fmt.Errorf("%w: %s: bulk writer pinned", ErrMaintenanceBusy, maintenanceDenseIndexRepair) } - for _, idx := range bulkDroppableIndexes { - if idx.name != name { - continue - } - if _, err := s.writerDB.ExecContext(ctx, idx.ddl); err != nil { - return fmt.Errorf("store_sqlite: repair index %s: %w", name, err) - } - break + if _, err := s.writerDB.ExecContext(ctx, idx.ddl); err != nil { + return fmt.Errorf("store_sqlite: repair index %s: %w", idx.name, err) } // Under the hold that built it, so whichever window recorded the name, the // index it names is present now. - var remaining []string - for _, missing := range s.denseIndexesMissingNames() { - if missing != name { + var remaining []bulkDroppableIndex + for _, missing := range s.denseIndexesMissingRecord() { + if missing.name != idx.name { remaining = append(remaining, missing) } } diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go index 94ed711ef..53961f96a 100644 --- a/internal/graph/store_sqlite/dense_index_repair_test.go +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -148,6 +148,56 @@ func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { } } +// TestDenseIndexRepairBuildsFromTheRecordedDDL: the repair runs the DDL the +// closing window recorded with each missing name, never a lookup in +// bulkDroppableIndexes made when the lane gets to it. In-package cases replace +// that list to force a seal failure and put it back while the lane may still be +// on its way to the repair; a lookup there reads the list unsynchronised with +// that restore, and finds no DDL for a name the record still holds. +func TestDenseIndexRepairBuildsFromTheRecordedDDL(t *testing.T) { + shortenDenseIndexRepairBackoff(t) + s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) + + originalIndexes := bulkDroppableIndexes + bulkDroppableIndexes = append(append([]bulkDroppableIndex(nil), originalIndexes...), bulkDroppableIndex{ + name: "forced_bad_index", + ddl: `CREATE INDEX forced_bad_index ON definitely_missing_table(id)`, + }) + s.BeginBulkLoad() + if s.bulkConn == nil { + bulkDroppableIndexes = originalIndexes + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + err := s.FlushBulk() + bulkDroppableIndexes = originalIndexes + if err == nil || !strings.Contains(err.Error(), "forced_bad_index") { + t.Fatalf("FlushBulk error = %v, want the forced index's DDL failure", err) + } + + // The list no longer names the index; the record does, with the DDL that + // cannot succeed. Every attempt must run it and fail, not report a repair. + full.Store(false) + waitUntil(t, "a finished repair attempt", func() bool { + return s.denseIndexRepairs.Load() > 0 || s.denseIndexRepairAttempts.Load() >= 2 + }) + if got := s.denseIndexRepairs.Load(); got != 0 { + t.Fatalf("repairs = %d for an index whose DDL cannot run, want 0", got) + } + if got := s.denseIndexesMissingNames(); !reflect.DeepEqual(got, []string{"forced_bad_index"}) { + t.Fatalf("missing record = %v, want [forced_bad_index]", got) + } +} + func waitUntil(t *testing.T, what string, done func() bool) { t.Helper() deadline := time.Now().Add(10 * time.Second) diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index af331b37d..6b131f3f0 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -523,13 +523,15 @@ type storeCore struct { // health probe. Written only under writeMu, by syncBulkWindowLocked. bulkWindowOpen atomic.Bool - // denseIndexesMissing names the dense indexes a cold bulk window closed + // denseIndexesMissing holds the dense indexes a cold bulk window closed // without rebuilding (its seal failed, or it was aborted before sealing), - // in bulkDroppableIndexes order; nil when none are owed. Written only - // under writeMu and replaced, never mutated in place, so the health probe - // reads it without a lock for the same reason it reads bulkWindowOpen. + // in bulkDroppableIndexes order and each with the DDL that builds it; nil + // when none are owed. Written only under writeMu and replaced, never + // mutated in place, so the health probe reads it without a lock for the + // same reason it reads bulkWindowOpen, and the maintenance lane's repair + // builds from it rather than reading bulkDroppableIndexes itself. // Not persisted: Open's createGraphCoreIndexes rebuilds them on restart. - denseIndexesMissing atomic.Pointer[[]string] + denseIndexesMissing atomic.Pointer[[]bulkDroppableIndex] // Prepared statements (compiled once in Open, closed in Close). stmtInsertNode *sql.Stmt From 7a8ce7f857a82ee883c98cb310c75dc6df27d0e3 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 15:48:54 +0100 Subject: [PATCH 10/60] fix(sqlite): leave a generation window to its owner on a coordinated abort AbortCoordinatedBulkLoad closed whatever bulk connection was pinned. Its only caller so far opened a coordinated window and aborted that same window, so this never mattered. The next change makes the coordinated owners abort after a final EndCoordinatedBulkLoad fails. If that end failed after it had already closed the window (planner statistics, a PRAGMA restore), a payload generation can have opened its own window by the time the abort runs. A generation window and a coordinated window refuse each other, so a pinned generation writer is never the window being aborted. Closing it would strand the window's checkpoint lease, which only EndGenerationBulkLoad releases, and turn that end into a no-op. The abort now returns without touching a generation window. TestAbortCoordinatedBulkLoadLeavesAGenerationWindowToItsOwner checks that the window survives the abort, that its owner's end succeeds, and that the next generation can then open. --- internal/graph/store_sqlite/bulk_load.go | 18 ++++++++---- .../store_sqlite/bulk_load_generation_test.go | 29 +++++++++++++++++++ 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 3fbbb6fcb..1d27b7d74 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -1135,11 +1135,11 @@ func (s *Store) endCoordinatedBulkLoadLocked() (bulkWindowExit, error) { } // AbortCoordinatedBulkLoad gives up retryable finalization and restores the -// writer connection unconditionally. It is for terminal owners only: dense -// indexes whose rebuild failed stay absent, recorded as missing, until the -// maintenance lane's repair or a restart rebuilds them, but the store no -// longer holds synchronous=OFF, disabled automatic checkpoints, or a pinned -// writer indefinitely. +// writer connection unconditionally, unless a generation window holds it. It +// is for terminal owners only: dense indexes whose rebuild failed stay absent, +// recorded as missing, until the maintenance lane's repair or a restart +// rebuilds them, but the store no longer holds synchronous=OFF, disabled +// automatic checkpoints, or a pinned writer indefinitely. func (s *Store) AbortCoordinatedBulkLoad() error { exit, err := s.abortCoordinatedBulkLoadLocked() return errors.Join(err, s.finishBulkWindowExit(exit)) @@ -1148,6 +1148,14 @@ func (s *Store) AbortCoordinatedBulkLoad() error { func (s *Store) abortCoordinatedBulkLoadLocked() (bulkWindowExit, error) { s.writeMu.Lock() defer s.writeMu.Unlock() + if s.generationBulkLoad != 0 { + // Never a coordinated window (the two refuse each other), and not this + // caller's to close. An owner that aborts after an end which failed + // once the window was already closed can find the writer pinned by a + // payload generation opened since; closing that connection would leave + // its checkpoint lease bound and its EndGenerationBulkLoad inert. + return bulkWindowExit{}, nil + } s.coordinatedBulkLoad = false repair := s.bulkIndexesDeferred err := s.closeBulkConnectionLocked() diff --git a/internal/graph/store_sqlite/bulk_load_generation_test.go b/internal/graph/store_sqlite/bulk_load_generation_test.go index 67ae10223..0f3597b08 100644 --- a/internal/graph/store_sqlite/bulk_load_generation_test.go +++ b/internal/graph/store_sqlite/bulk_load_generation_test.go @@ -210,6 +210,35 @@ func TestGenerationBulkLoadRefusesWhatItCannotProveEmpty(t *testing.T) { }) } +// TestAbortCoordinatedBulkLoadLeavesAGenerationWindowToItsOwner: an owner that +// aborts after its coordinated end failed may find the writer pinned by a +// payload generation opened since. The abort must leave that window, and the +// checkpoint lease only its own end releases, to the generation's owner. +func TestAbortCoordinatedBulkLoadLeavesAGenerationWindowToItsOwner(t *testing.T) { + store, _ := openTempStore(t) + engaged, err := store.BeginGenerationBulkLoad(3) + if err != nil || !engaged { + t.Fatalf("BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + if err := store.AbortCoordinatedBulkLoad(); err != nil { + t.Fatalf("AbortCoordinatedBulkLoad: %v", err) + } + if id, open := store.InGenerationBulkLoad(); id != 3 || !open { + t.Fatalf("generation window after abort = (%d, %v), want (3, true)", id, open) + } + if err := store.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } + // The owner's end released the lease: the next generation can open. + engaged, err = store.BeginGenerationBulkLoad(4) + if err != nil || !engaged { + t.Fatalf("next BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + if err := store.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } +} + func TestGenerationBulkLoadLeavesBaseReadersAndIndexesAlone(t *testing.T) { store, _ := openTempStore(t) baseNodes, baseEdges := bulkFixture(256, 512) From 7a0d38527062877f57b6b975584d70aa1d0334a5 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 15:50:54 +0100 Subject: [PATCH 11/60] fix(indexer): give up the multi-repo coordinated window when its end fails twice indexMultiRepo ends its coordinated window explicitly and again in a defer when the explicit end fails. A final seal that fails keeps the window open by design, so its owner can retry. When the deferred retry failed as well, the pass logged it and returned, and the window stayed open until the store closed: one pinned writer at synchronous=OFF with automatic checkpoints off, carrying every later write from every repository. The dense indexes the seal could not build were never recorded, so neither the planner probe nor the maintenance repair saw them. CoordinatedBulkLoader now includes AbortCoordinatedBulkLoad. The only implementation, the SQLite store, already had it, so no backend changes. EndOrAbortCoordinatedBulkLoad ends the window and aborts it if the end fails. The defer uses it, so the explicit end still gets one retry and a second failure closes the window. The abort restores the writer and records the missing indexes, which the maintenance lane repairs once the volume has room. TestIndexMultiRepoFailedFinalSealGivesUpTheWindow runs the cold pass on a real store with a final seal that fails every time. It checks that the error reaches the caller, the writer's synchronous level is back, the planner probe no longer reports a live window, and the schema's index list is whole again after the repair. --- internal/graph/store.go | 19 ++++++++++++ internal/indexer/multi.go | 6 +++- .../indexer/multi_cold_orchestration_test.go | 30 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/internal/graph/store.go b/internal/graph/store.go index 5f4ffacc9..ddbdfb58b 100644 --- a/internal/graph/store.go +++ b/internal/graph/store.go @@ -2,6 +2,7 @@ package graph import ( "context" + "errors" "iter" "sync" ) @@ -525,9 +526,27 @@ type BulkLoader interface { // multi-repository cold load. Implementations engage only on an empty store; // nested per-repository BulkLoader calls must leave the outer window open so // secondary indexes are rebuilt once after every repository drains. +// +// A failed EndCoordinatedBulkLoad may leave the window open so its owner can +// retry it. AbortCoordinatedBulkLoad is how an owner that will not retry gives +// it up: the window closes unfinished, ordinary durability comes back, and +// whatever the end could not rebuild is left to the backend to repair. type CoordinatedBulkLoader interface { BeginCoordinatedBulkLoad() bool EndCoordinatedBulkLoad() error + AbortCoordinatedBulkLoad() error +} + +// EndOrAbortCoordinatedBulkLoad ends l's coordinated window and, when the end +// fails, aborts it. It is for an owner's last call on the window: an end that +// fails with the window still open would otherwise leave it open until the +// store closes, with every later write at the window's relaxed durability. +func EndOrAbortCoordinatedBulkLoad(l CoordinatedBulkLoader) error { + err := l.EndCoordinatedBulkLoad() + if err == nil { + return nil + } + return errors.Join(err, l.AbortCoordinatedBulkLoad()) } // SymbolHit is a single full-text-search result: the matched node ID diff --git a/internal/indexer/multi.go b/internal/indexer/multi.go index f2507de14..9b57508c7 100644 --- a/internal/indexer/multi.go +++ b/internal/indexer/multi.go @@ -2463,8 +2463,12 @@ func (mi *MultiIndexer) indexMultiRepo(repos []config.RepoEntry) (map[string]*In coordinatedBulk, _ := mi.graph.(graph.CoordinatedBulkLoader) coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() defer func() { + // The last call on the window: a retry of the finalize below + // when that failed, or the only end after an early return or + // a panic. Abort if it fails too, so the window cannot hold + // its relaxed-durability writer until the store closes. if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil { mi.logger.Error("multi-repo bulk-load cleanup failed", zap.Error(err)) } } diff --git a/internal/indexer/multi_cold_orchestration_test.go b/internal/indexer/multi_cold_orchestration_test.go index 28bc6ba3a..fb6f66966 100644 --- a/internal/indexer/multi_cold_orchestration_test.go +++ b/internal/indexer/multi_cold_orchestration_test.go @@ -473,6 +473,36 @@ func TestIndexMultiRepoRefFactFailureStopsGlobalConsumers(t *testing.T) { "a failed reference-fact boundary must not publish global completion") } +// TestIndexMultiRepoFailedFinalSealGivesUpTheWindow: when the coordinated +// window's final seal keeps failing, the cold pass reports the failure and then +// gives the window up instead of leaving it open behind the error. The writer +// is back at its own durability, the planner probe no longer sees a live +// window, and the store's repair rebuilds the index the seal could not. +func TestIndexMultiRepoFailedFinalSealGivesUpTheWindow(t *testing.T) { + store, db := newFirstIndexStore(t) + indexes := firstIndexSchemaIndexes(t, db) + writerSync := firstIndexWriterSynchronous(t, store) + injected := errors.New("injected final seal failure") + restore := store.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + mi := newColdOrchestrationMulti(store, zap.NewNop()) + + _, err := mi.indexMultiRepo(coldRefFactRepos(t, 2)) + restore() + require.ErrorIs(t, err, injected) + assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store), "writer durability must be restored") + assert.False(t, firstIndexBulkActive(t, store), "the failed window must not stay open") + deadline := time.Now().Add(10 * time.Second) + for !assert.ObjectsAreEqual(indexes, firstIndexSchemaIndexes(t, db)) && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + assert.Equal(t, indexes, firstIndexSchemaIndexes(t, db), "the repair must rebuild the index the seal left out") +} + // TestBeginDeferredPassesOverlapSplitsPoolFromTail pins the overlap contract: // BeginDeferredPasses drains the enrichment pool on its own goroutine while // the caller is free to run the resolve phase; contracts and the catch-up From df7d38a80eabf9769f84392eab600aa40293b2e9 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 15:50:54 +0100 Subject: [PATCH 12/60] fix(daemon): give up the warmup's coordinated window when its final seal fails Daemon warmup ends its coordinated cold-load window once, after the go.mod drain. On failure it logged the error and cleared its active flag, which also skipped the deferred end, and never retried. A final seal that fails keeps the window open so its owner can retry, so a failed seal at warmup, typically a full disk, left the daemon running on one pinned writer at synchronous=OFF until shutdown. The indexes the seal could not build were never recorded, so index_health reported bulk_window_active instead of the missing indexes, and the maintenance repair could not run while the writer stayed pinned. The finalize and the deferred cleanup now go through EndOrAbortCoordinatedBulkLoad. A failed end is followed by an abort, which restores the writer and records the missing indexes for the maintenance lane to rebuild once there is room. TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow runs a real warmup on a SQLite store with a failing final seal. It checks that the failure is logged, that the writer's synchronous level is back, that the planner probe no longer reports a live window, and that the repair then clears dense_indexes_missing. --- cmd/gortex/daemon_state.go | 9 +++- cmd/gortex/daemon_warmup_bulk_test.go | 75 +++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 8f097b733..b25f5b3b6 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -743,7 +743,7 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad() defer func() { if coordinatedBulkActive { - if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil { + if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil { logger.Error("daemon: coordinated cold bulk-load cleanup failed", zap.Error(err)) } } @@ -916,9 +916,14 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z // final seal includes them in bulk index creation, checkpointing, and // planner statistics instead of reopening ordinary write transactions // immediately after a large cold load. + // The finalize is the window's last call: the deferred end above is + // skipped once it has run, and nothing retries a failed seal. So a + // failed end is followed by an abort, which closes the window and + // leaves the indexes it could not rebuild to the store's repair, rather + // than keeping the warmup's relaxed-durability writer until shutdown. var finalizeBulk func() error if coordinatedBulkActive { - finalizeBulk = coordinatedBulk.EndCoordinatedBulkLoad + finalizeBulk = func() error { return graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk) } } drain := state.multiIndexer.RunDeferredGoModAll if batchCtx.Err() != nil { diff --git a/cmd/gortex/daemon_warmup_bulk_test.go b/cmd/gortex/daemon_warmup_bulk_test.go index 393bf6000..412d9bf9b 100644 --- a/cmd/gortex/daemon_warmup_bulk_test.go +++ b/cmd/gortex/daemon_warmup_bulk_test.go @@ -1,8 +1,23 @@ package main import ( + "context" "errors" + "os" + "path/filepath" "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" ) func TestDrainGoModAndFinalizeWarmupBulkOrdersDrainBeforeFinalize(t *testing.T) { @@ -38,3 +53,63 @@ func TestDrainGoModAndFinalizeWarmupBulkWithoutBulkFinalizer(t *testing.T) { t.Fatalf("finalize elapsed = %s, want 0", finalizeElapsed) } } + +// TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow: a cold warmup whose +// final seal fails logs the failure and closes the coordinated window, rather +// than keeping its relaxed-durability writer until the daemon shuts down. The +// writer is back at its own synchronous level, the planner probe no longer +// sees a live window, and the store's repair rebuilds the index the seal +// could not. +func TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow(t *testing.T) { + t.Setenv("GORTEX_PARSER_ISOLATION", "0") + root := t.TempDir() + t.Setenv("HOME", filepath.Join(root, "home")) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, "config")) + t.Setenv("XDG_DATA_HOME", filepath.Join(root, "data")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(root, "cache")) + repo := filepath.Join(root, "repo") + require.NoError(t, os.Mkdir(repo, 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(repo, "main.go"), []byte("package main\nfunc Main() {}\n"), 0o600)) + store, err := store_sqlite.Open(filepath.Join(root, "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + ctx := context.Background() + writerSync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err) + cm, err := config.NewConfigManager(filepath.Join(root, "config.yaml")) + require.NoError(t, err) + require.NoError(t, cm.Global().AddRepo(config.RepoEntry{Path: repo})) + reg := parser.NewRegistry() + reg.Register(languages.NewGoExtractor()) + logs, observed := observer.New(zap.InfoLevel) + logger := zap.New(logs) + idx := indexer.New(store, reg, config.Default().Index, logger) + mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, logger) + t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) }) + state := &daemonState{graph: store, indexer: idx, multiIndexer: mi, configManager: cm} + injected := errors.New("injected final seal failure") + restore := store.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + + _, _ = warmupDaemonState(state, logger, func() {}) + restore() + require.NotEmpty(t, observed.FilterMessage("daemon: coordinated cold bulk-load finalize failed").All(), + "the fixture must fail the coordinated window's final seal") + gotSync, err := store.WriterPragmaForTest(ctx, "synchronous") + require.NoError(t, err) + assert.Equal(t, writerSync, gotSync, "writer durability must be restored") + health, err := store.PlannerStatsHealth(ctx) + require.NoError(t, err) + assert.NotEqual(t, "bulk_window_active", health.Reason, "the failed window must not stay open") + deadline := time.Now().Add(10 * time.Second) + for health.Reason == "dense_indexes_missing" && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + health, err = store.PlannerStatsHealth(ctx) + require.NoError(t, err) + } + assert.NotEqual(t, "dense_indexes_missing", health.Reason, "the repair must rebuild the index the seal left out") +} From 167a35f516c268320437f542aae7729ebcac2625 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 16:04:11 +0100 Subject: [PATCH 13/60] test(sqlite): a dense-index repair waits for a pinned writer repairDenseIndex refuses with ErrMaintenanceBusy while a bulk window pins the writer, as VACUUM does, so the CREATE INDEX never runs under a window that owns the connection. No test covered that branch: a cold window cannot reopen on the populated store that a failed seal leaves behind. A generation window can, so this test uses one. A flush whose seal fails on nodes_by_kind records the index as missing. While a generation window is open, every repair attempt is refused: nothing is reported as repaired, the record keeps the name, and the index stays absent. Once the window's owner ends it, the next retry builds the index and clears the record. --- .../store_sqlite/dense_index_repair_test.go | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go index 53961f96a..55be847f4 100644 --- a/internal/graph/store_sqlite/dense_index_repair_test.go +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -198,6 +198,69 @@ func TestDenseIndexRepairBuildsFromTheRecordedDDL(t *testing.T) { } } +// TestDenseIndexRepairWaitsForAPinnedWriter: a repair that comes due while a +// window pins the writer defers, as VACUUM does, instead of building under +// that window, and builds on a later retry once the window has closed. A +// generation window is the one that can open on the populated store a failed +// seal leaves behind. +func TestDenseIndexRepairWaitsForAPinnedWriter(t *testing.T) { + shortenDenseIndexRepairBackoff(t) + s, _ := openTempStore(t) + var full atomic.Bool + full.Store(true) + diskAvail := func(string) (uint64, error) { + if full.Load() { + return 0, nil + } + return 1 << 40, nil + } + s.denseRepairDiskAvail.Store(&diskAvail) + s.BeginBulkLoad() + if s.bulkConn == nil { + t.Fatal("bulk fast path did not engage") + } + s.AddNode(&graph.Node{ID: "repo/a.go::A", Kind: graph.KindFunction, Name: "A"}) + injected := errors.New("injected seal failure") + restore := s.SetBulkSealHookForTest(func(index string) error { + if index == "nodes_by_kind" { + return injected + } + return nil + }) + err := s.FlushBulk() + restore() + if !errors.Is(err, injected) { + t.Fatalf("FlushBulk error = %v, want the injected seal failure", err) + } + engaged, err := s.BeginGenerationBulkLoad(3) + if err != nil || !engaged { + t.Fatalf("BeginGenerationBulkLoad = (%v, %v), want engaged", engaged, err) + } + + full.Store(false) + waitUntil(t, "two repair attempts under the pinned writer", func() bool { return s.denseIndexRepairAttempts.Load() >= 2 }) + if got := s.denseIndexRepairs.Load(); got != 0 { + t.Fatalf("repairs under a pinned writer = %d, want 0", got) + } + if got := s.denseIndexesMissingNames(); !reflect.DeepEqual(got, []string{"nodes_by_kind"}) { + t.Fatalf("missing record under a pinned writer = %v, want [nodes_by_kind]", got) + } + if indexNames(t, s.db)["nodes_by_kind"] { + t.Fatal("nodes_by_kind was built under the pinned writer") + } + + if err := s.EndGenerationBulkLoad(); err != nil { + t.Fatalf("EndGenerationBulkLoad: %v", err) + } + waitUntil(t, "the repair after the window closed", func() bool { return s.denseIndexRepairs.Load() == 1 }) + if !indexNames(t, s.db)["nodes_by_kind"] { + t.Fatal("nodes_by_kind missing after the repair") + } + if got := s.denseIndexesMissingNames(); len(got) != 0 { + t.Fatalf("missing record after the repair = %v, want empty", got) + } +} + func waitUntil(t *testing.T, what string, done func() bool) { t.Helper() deadline := time.Now().Add(10 * time.Second) From 041570de6eaaa95fbd93f42c4143f2c237b89b6a Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 17:22:13 +0100 Subject: [PATCH 14/60] gitstate: read a checkout's family common dir from its .git marker Checkout discovery needs to know whether a request path belongs to a known Git family before it spends a discovery slot on it. Asking Git costs a process spawn per request, which is exactly the work a path in an unknown family should never cause. MarkerCommonDir walks up to the nearest .git marker and follows it with the reader the dirty sampler already uses (a .git directory or a gitdir file, then commondir), then resolves symlinks the way `git rev-parse --path-format=absolute` reports the common dir, so both key the same family. The family-dirs tests now assert the marker read matches Git for a main checkout, a linked worktree, a subdirectory and a relative path, and that it finds nothing outside Git. --- internal/gitstate/family_dirs.go | 34 +++++++++++++++++++++++++++ internal/gitstate/family_dirs_test.go | 9 +++++++ 2 files changed, 43 insertions(+) diff --git a/internal/gitstate/family_dirs.go b/internal/gitstate/family_dirs.go index 7accb524b..4412f87f4 100644 --- a/internal/gitstate/family_dirs.go +++ b/internal/gitstate/family_dirs.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "os" + "path/filepath" ) // ResolveFamilyDirs resolves dir's own Git directory and shared common @@ -27,3 +29,35 @@ func ResolveFamilyDirs(ctx context.Context, dir string) (gitDir, commonDir strin } return gitDir, commonDir, nil } + +// MarkerCommonDir reads, without running Git, the common directory of the +// checkout that holds dir: the nearest .git marker at or above dir (a +// directory or a "gitdir:" file), then the commondir file of the Git +// directory it names. root is the directory holding the marker, and commonDir +// is spelled the way ResolveFamilyDirs reports it. ok is false when no marker +// is found or this read cannot follow it; Git remains the authority on +// anything that must be proven. +func MarkerCommonDir(dir string) (root, commonDir string, ok bool) { + root, err := absDir(dir) + if err != nil { + return "", "", false + } + for { + if _, err := os.Lstat(filepath.Join(root, ".git")); err == nil { + break + } + parent := filepath.Dir(root) + if parent == root { + return "", "", false + } + root = parent + } + if _, commonDir = resolveGitDirsFromDotGit(root); commonDir == "" { + return "", "", false + } + // Git reports the common directory with its symlinks resolved. + if commonDir, err = filepath.EvalSymlinks(commonDir); err != nil { + return "", "", false + } + return root, commonDir, true +} diff --git a/internal/gitstate/family_dirs_test.go b/internal/gitstate/family_dirs_test.go index 1d8789898..1c0cb2b06 100644 --- a/internal/gitstate/family_dirs_test.go +++ b/internal/gitstate/family_dirs_test.go @@ -77,6 +77,12 @@ func TestResolveFamilyDirsMatchesMainLinkedAndSubdirectory(t *testing.T) { } familyDirsSameDirectory(t, gitDir, inv.GitDir) familyDirsSameDirectory(t, commonDir, filepath.Join(primary, ".git")) + // The marker read must name the same family key without Git. + markerRoot, markerCommonDir, ok := MarkerCommonDir(path) + if !ok || markerCommonDir != commonDir { + t.Fatalf("marker of %q names common dir %q (ok=%v), Git names %q", path, markerCommonDir, ok, commonDir) + } + familyDirsSameDirectory(t, markerRoot, root) } } } @@ -88,6 +94,9 @@ func TestResolveFamilyDirsUnavailableHasNoBinding(t *testing.T) { if !errors.Is(err, ErrInventoryUnavailable) || gitDir != "" || commonDir != "" { t.Fatalf("unavailable path %q returned %q, %q, %v", path, gitDir, commonDir, err) } + if root, commonDir, ok := MarkerCommonDir(path); ok { + t.Fatalf("path %q outside Git has a marker at %q naming %q", path, root, commonDir) + } } } From 8202e0ec9ef897a71b253c1729baf69b0558ad6d Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 18:00:15 +0100 Subject: [PATCH 15/60] test(sqlite): a completed dense-index repair leads to a statistics pass A rebuilt index has no sqlite_stat1 row, so until ANALYZE runs on it the planner costs it from defaults. runDenseIndexRepair asks the maintenance lane for a statistics pass when it completes, and nothing else does at that point: the window that left the index missing scheduled its own pass long before the repair ran. No test covered that request. TestFailedSealReportsDenseIndexesMissing now waits, in both of its cases, for nodes_by_kind to gain a sqlite_stat1 row after the repair. Without the request both cases time out. --- internal/graph/store_sqlite/dense_index_repair_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go index 55be847f4..ceca4b105 100644 --- a/internal/graph/store_sqlite/dense_index_repair_test.go +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -25,7 +25,8 @@ func shortenDenseIndexRepairBackoff(t *testing.T) { // rebuilding its dense indexes must still restore durability and close, must // say which indexes it left absent instead of letting the planner probe read // their absence as a clean store, must not retry the rebuild while the volume -// is full, and must rebuild them in one maintenance attempt once it is not. +// is full, and must rebuild them in one maintenance attempt once it is not, +// followed by the statistics pass the rebuilt indexes need. func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { shortenDenseIndexRepairBackoff(t) allDroppable := make([]string, 0, len(bulkDroppableIndexes)) @@ -143,6 +144,12 @@ func TestFailedSealReportsDenseIndexesMissing(t *testing.T) { t.Fatalf("index %s missing after the repair", name) } } + // A rebuilt index has no sqlite_stat1 row, and once the repair is + // done nothing else asks for a statistics pass, so the repair must + // ask for one itself. nodes_by_kind holds the case's node. + waitUntil(t, "statistics for the rebuilt nodes_by_kind", func() bool { + return s.plannerStatsIndexesWithStats(context.Background())["nodes_by_kind"] + }) integrityOK(t, s.db) }) } From 16cb03a33437b18ca104b8d8a2abe21a1b2a6aea Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 17:26:23 +0100 Subject: [PATCH 16/60] mcp: label change answers instead of refusing on pending edits explain_change_impact and detect_changes waited up to 3 s on every pending reindex receipt in the repository and then failed with "refused a stale graph". The edit that triggered the wait was usually in a file unrelated to the answer, and an id that did not resolve widened the wait to every receipt, so concurrent agents got refusals for most impact calls and each one held a dispatch slot for the full wait. Both handlers now wait at most 250 ms, and only for the edits that can change the answer: the queried symbols' own files, files with edges into them, files whose current bytes name them as a whole identifier (a call the graph has not read yet), and for detect_changes the changed files of the diff, which is now mapped files-first so the scope is known before the wait. An unresolved id contributes the file and name its spelling carries instead of widening the scope. Whatever in scope is still pending or failed labels the answer (complete:false, graph_freshness, freshness_scope, pending_files, pending_file_count, and recovery_hint when failed); the walk always runs. An earlier wait_deadline ends the wait sooner, including one that passed between admission and the handler reading it. TOON and plain text answers lead with a "relations: (N files)" line, and GCX keeps GCX1 as line 1 with the label as header meta and the line as a comment on line 2. Every answer reports barrier_wait_ms. Only require_fresh refuses, once its wait_deadline passes with the scope still lagging. The label lives in relation_freshness.go so the relation-reading mutators can share it. --- internal/analysis/diffmap.go | 7 + internal/mcp/gcx.go | 5 + internal/mcp/mutation_freshness_test.go | 103 +++++++--- internal/mcp/relation_freshness.go | 258 ++++++++++++++++++++++++ internal/mcp/relation_freshness_test.go | 194 ++++++++++++++++++ internal/mcp/tools_analysis.go | 72 +++++-- 6 files changed, 590 insertions(+), 49 deletions(-) create mode 100644 internal/mcp/relation_freshness.go create mode 100644 internal/mcp/relation_freshness_test.go diff --git a/internal/analysis/diffmap.go b/internal/analysis/diffmap.go index ca9b76fe7..b58f8643a 100644 --- a/internal/analysis/diffmap.go +++ b/internal/analysis/diffmap.go @@ -113,6 +113,13 @@ func MapGitDiffContext(ctx context.Context, g graph.Reader, repoRoot, repoPrefix return joinHunksToSymbolsContext(ctx, g, repoPrefix, hunks, files) } +// JoinDiffSymbolsContext maps a file-only diff (MapGitDiffContext with a nil +// reader) to the symbols g indexes, without running git again. A caller that +// needs the changed files before it can pick its reader diffs once this way. +func JoinDiffSymbolsContext(ctx context.Context, g graph.Reader, repoPrefix string, diff *DiffResult) (*DiffResult, error) { + return joinHunksToSymbolsContext(ctx, g, repoPrefix, diff.Hunks, diff.FileChanges) +} + // PathDomain names the vocabulary a file path is spelled in. The two overlap, // so the domain has to travel with the path rather than be recovered from it: // in a repo whose tree carries a top-level directory named like the repo diff --git a/internal/mcp/gcx.go b/internal/mcp/gcx.go index ef2b16ae0..2a4caddb3 100644 --- a/internal/mcp/gcx.go +++ b/internal/mcp/gcx.go @@ -1937,8 +1937,13 @@ func encodeChangeImpact(result map[string]any) ([]byte, error) { communityNote, _ := result["community_note"].(string) contractRiskUpgrade, _ := result["contract_risk_upgrade"].(string) + var meta []string + if waited, ok := result["barrier_wait_ms"].(int64); ok { + meta = append(meta, "barrier_wait_ms", strconv.FormatInt(waited, 10)) + } sumEnc := newGCX(&buf, "explain_change_impact.summary", []string{"risk", "summary", "total_affected", "cross_repo", "processes", "communities", "test_files", "warning", "note", "risk_upgrade"}, + meta..., ) if err := sumEnc.WriteRow( string(risk), diff --git a/internal/mcp/mutation_freshness_test.go b/internal/mcp/mutation_freshness_test.go index d9dd4b21c..230398750 100644 --- a/internal/mcp/mutation_freshness_test.go +++ b/internal/mcp/mutation_freshness_test.go @@ -434,41 +434,44 @@ func TestMutationReposForSymbolIDsUnresolvedWidensBarrier(t *testing.T) { } } -func TestChangeImpactFreshnessGuardAndRetry(t *testing.T) { +func TestChangeImpactLabelsLaggingReceiptsInScope(t *testing.T) { g := graph.New() g.AddNode(&graph.Node{ ID: "repo-a::target", Kind: graph.KindFunction, Name: "target", - FilePath: "target.go", + FilePath: "/repo-a/target.go", StartLine: 1, EndLine: 1, Language: "go", RepoPrefix: "repo-a", }) s := &Server{ - graph: g, - engine: query.NewEngine(g), - session: newSessionState(), - mutationSafetyWait: time.Millisecond, + graph: g, + engine: query.NewEngine(g), + session: newSessionState(), } receipt := pendingFreshnessReceipt(s, "receipt-impact", "repo-a", "/repo-a/target.go", 11) req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{ Name: "explain_change_impact", Arguments: map[string]any{"ids": "repo-a::target", "format": "json"}, }} - - result, err := s.handleEnhancedChangeImpact(context.Background(), req) - if err != nil { - t.Fatalf("pending impact handler error: %v", err) + impact := func() map[string]any { + t.Helper() + result, err := s.handleEnhancedChangeImpact(context.Background(), req) + if err != nil { + t.Fatalf("impact handler error: %v", err) + } + return lifecycleResultPayload(t, result) } - if result == nil || !result.IsError { - t.Fatalf("pending impact result = %#v, want MCP error", result) + + // A pending edit to the symbol's own file is labelled, never refused. + payload := impact() + if payload["graph_freshness"] != "pending" || payload["complete"] != false { + t.Fatalf("pending impact = %v, want a pending partial", payload) } - for _, want := range []string{"change impact refused a stale graph", "receipt-impact", "generation=11"} { - if !strings.Contains(freshnessToolResultText(result), want) { - t.Fatalf("pending impact response %q does not contain %q", freshnessToolResultText(result), want) - } + if files, _ := payload["pending_files"].([]any); len(files) != 1 || files[0] != "/repo-a/target.go" { + t.Fatalf("pending impact files = %v", payload["pending_files"]) } completeFreshnessReceipt(receipt, indexer.MutationResult{ @@ -476,22 +479,31 @@ func TestChangeImpactFreshnessGuardAndRetry(t *testing.T) { AppliedGeneration: 11, Reindexed: true, }) - result, err = s.handleEnhancedChangeImpact(context.Background(), req) - if err != nil { - t.Fatalf("completed impact handler error: %v", err) + if payload = impact(); payload["graph_freshness"] != nil { + t.Fatalf("published edit still labelled: %v", payload) + } + + // A terminally failed edit in scope is a labelled partial with a + // recovery hint, not a tool error. + failed := pendingFreshnessReceipt(s, "receipt-failed", "repo-a", "/repo-a/target.go", 12) + completeFreshnessReceipt(failed, indexer.MutationResult{ + RequestedGeneration: 12, + AppliedGeneration: 12, + Err: errors.New("syntax patch failed"), + }) + payload = impact() + if payload["graph_freshness"] != "failed" || payload["complete"] != false { + t.Fatalf("failed impact = %v, want a failed partial", payload) } - if result == nil || result.IsError { - t.Fatalf("completed impact remained blocked: %q", freshnessToolResultText(result)) + if hint, _ := payload["recovery_hint"].(string); !strings.Contains(hint, "reindex") { + t.Fatalf("failed impact recovery hint = %q", hint) } } -func TestDetectChangesFreshnessGuard(t *testing.T) { - s := &Server{ - graph: graph.New(), - session: newSessionState(), - mutationSafetyWait: time.Millisecond, - } - pendingFreshnessReceipt(s, "receipt-detect", "repo-a", "/repo-a/changed.go", 12) +func TestDetectChangesLabelsPendingChangedFile(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-detect", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 12) req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{ Name: "detect_changes", Arguments: map[string]any{"format": "json"}, @@ -501,12 +513,37 @@ func TestDetectChangesFreshnessGuard(t *testing.T) { if err != nil { t.Fatalf("detect handler error: %v", err) } - if result == nil || !result.IsError { - t.Fatalf("pending detect result = %#v, want MCP error", result) + payload := lifecycleResultPayload(t, result) + for key, want := range map[string]any{ + "complete": false, + "graph_freshness": "pending", + "pending_file_count": float64(1), + } { + if payload[key] != want { + t.Fatalf("detect %s = %v, want %v: %v", key, payload[key], want, payload) + } + } + if files, _ := payload["changed_files"].([]any); len(files) != 1 || files[0] != "caller.go" { + t.Fatalf("detect changed_files = %v", payload["changed_files"]) } - for _, want := range []string{"change detection refused a stale graph", "receipt-detect", "generation=12"} { - if !strings.Contains(freshnessToolResultText(result), want) { - t.Fatalf("pending detect response %q does not contain %q", freshnessToolResultText(result), want) + // The walk still ran: the changed symbol is mapped and risk is scored. + if symbols, _ := payload["changed_symbols"].([]any); len(symbols) == 0 || payload["risk"] == "UNKNOWN" { + t.Fatalf("labelled detect skipped the walk: %v", payload) + } + + // The Stop hook bounds the wait with an earlier wait_deadline, which can + // pass between admission and the handler reading it. Either way the + // answer is labelled without sitting out the default budget. + for _, offset := range []time.Duration{50 * time.Millisecond, -time.Second} { + req.Params.Arguments = map[string]any{"format": "json", + "wait_deadline": time.Now().Add(offset).Format(time.RFC3339Nano)} + result, err = s.handleDetectChanges(context.Background(), req) + if err != nil { + t.Fatalf("detect handler error: %v", err) + } + payload = lifecycleResultPayload(t, result) + if waited, _ := payload["barrier_wait_ms"].(float64); payload["graph_freshness"] != "pending" || waited >= 200 { + t.Fatalf("detect with wait_deadline %v: graph_freshness %v after %v ms", offset, payload["graph_freshness"], waited) } } } diff --git a/internal/mcp/relation_freshness.go b/internal/mcp/relation_freshness.go new file mode 100644 index 000000000..024c687f0 --- /dev/null +++ b/internal/mcp/relation_freshness.go @@ -0,0 +1,258 @@ +package mcp + +import ( + "bytes" + "context" + "fmt" + "os" + "sort" + "strings" + "time" + + wire "github.com/gortexhq/gcx-go" + "github.com/mark3labs/mcp-go/mcp" + + "github.com/zzet/gortex/internal/graph" +) + +// relationFreshnessWait bounds how long a change tool waits for the edits in +// its relation scope before it answers with a label instead. +const relationFreshnessWait = 250 * time.Millisecond + +// maxLabelledPendingFiles caps the pending_files list of a label; +// pending_file_count always carries the full count. +const maxLabelledPendingFiles = 16 + +// relationFreshness labels an answer read from graph relations while edits in +// its scope are still being published. The zero value is a fresh scope. +type relationFreshness struct { + state string // "", "pending" or "failed" + files []string // graph-keyed paths of the lagging in-scope edits, sorted + waited time.Duration +} + +// relationWaitDeadline is when a change tool stops waiting for its relation +// scope: relationFreshnessWait from now, or an earlier wait_deadline. With +// require_fresh it is the request's own freshness deadline instead. The +// middleware has already refused a malformed or past wait_deadline, so one +// the handler cannot take passed after admission and ends the wait now. +func relationWaitDeadline(ctx context.Context, freshness requestFreshness) time.Time { + now := time.Now() + if freshness.err != nil { + return now + } + if freshness.requireFresh { + return freshness.effectiveDeadline(now, ctx) + } + deadline := now.Add(relationFreshnessWait) + if freshness.hasDeadline && freshness.deadline.Before(deadline) { + return freshness.deadline + } + return deadline +} + +// awaitRelationFreshness waits until deadline for the edits in the relation +// scope of ids and files, then labels whatever in that scope is still pending +// or failed. A deadline already passed labels without waiting. Edits outside +// the scope are neither waited on nor reported. +func (s *Server) awaitRelationFreshness(ctx context.Context, ids, files []string, deadline time.Time) relationFreshness { + started := time.Now() + lagging := s.laggingMutationReceipts(ctx) + if len(lagging) == 0 { + return relationFreshness{waited: time.Since(started)} + } + inScope := s.relationScope(ctx, ids, files) + if remaining := time.Until(deadline); remaining > 0 { + timer := time.NewTimer(remaining) + defer timer.Stop() + wait: + for _, receipt := range lagging { + if !inScope(receipt.path) { + continue + } + select { + case <-receipt.done: + case <-timer.C: + break wait + case <-ctx.Done(): + break wait + } + } + lagging = s.laggingMutationReceipts(ctx) + } + + states := make(map[string]string) + for _, receipt := range lagging { + if !inScope(receipt.path) { + continue + } + key := s.repoRelative(receipt.path) + if receipt.outcome(true).Pending { + states[key] = "pending" + } else if states[key] == "" { + states[key] = "failed" + } + } + label := relationFreshness{waited: time.Since(started)} + for path, state := range states { + label.files = append(label.files, path) + if label.state != "failed" { + label.state = state + } + } + sort.Strings(label.files) + return label +} + +// laggingMutationReceipts returns the request's mutation receipts whose edit +// the graph has not published: still pending, or terminally failed. +func (s *Server) laggingMutationReceipts(ctx context.Context) []*mutationReceipt { + var lagging []*mutationReceipt + for _, receipt := range s.mutationReceiptsForRepos(ctx) { + if outcome := receipt.outcome(true); outcome.Pending || outcome.Err != nil || !outcome.Reindexed { + lagging = append(lagging, receipt) + } + } + return lagging +} + +// relationScope reports whether an edit to a file can change the relations +// of ids: the file defines one of them, holds an edge into one of them, is one +// of files (graph-keyed), or its current bytes name one of them as a whole +// identifier, which is how a call added by an edit the graph has not read yet +// shows up. An id that does not resolve contributes the file and name its +// spelling carries, so it narrows the scope instead of widening it. +func (s *Server) relationScope(ctx context.Context, ids, files []string) func(absPath string) bool { + scoped := make(map[string]bool, len(files)+len(ids)) + for _, file := range files { + scoped[file] = true + } + var names []string + reader := s.readerFor(ctx) + for _, id := range ids { + id = strings.TrimSpace(id) + if id == "" { + continue + } + node := reader.GetNode(id) + if node == nil { + if file := graph.IDFile(id); file != "" { + scoped[file] = true + } + names = append(names, identifierName(id[strings.LastIndex(id, ":")+1:])) + continue + } + scoped[node.FilePath] = true + names = append(names, identifierName(node.Name)) + for _, edge := range reader.GetInEdges(id) { + file := edge.FilePath + if file == "" { + file = graph.IDFile(edge.From) + } + if file != "" { + scoped[file] = true + } + } + } + named := make(map[string]bool) + return func(absPath string) bool { + if scoped[s.repoRelative(absPath)] { + return true + } + hit, seen := named[absPath] + if !seen && len(names) > 0 { + data, err := os.ReadFile(absPath) + hit = err == nil && namesAnyIdentifier(data, names) + named[absPath] = hit + } + return hit + } +} + +// identifierName is the last identifier of a possibly qualified name, the +// spelling a call site uses: Recv.Method is called as Method. +func identifierName(name string) string { + return name[strings.LastIndex(name, ".")+1:] +} + +// namesAnyIdentifier reports whether data holds one of names as a whole +// identifier rather than as part of a longer one. +func namesAnyIdentifier(data []byte, names []string) bool { + for _, name := range names { + if name == "" { + continue + } + needle := []byte(name) + for offset := 0; offset < len(data); { + i := bytes.Index(data[offset:], needle) + if i < 0 { + break + } + start, end := offset+i, offset+i+len(needle) + if (start == 0 || !isIdentByte(data[start-1])) && (end == len(data) || !isIdentByte(data[end])) { + return true + } + offset = start + 1 + } + } + return false +} + +// apply writes the label into an answer's fields. A fresh scope writes +// nothing, so an answer with nothing lagging keeps its shape. +func (f relationFreshness) apply(result map[string]any) { + if f.state == "" { + return + } + result["complete"] = false + result["graph_freshness"] = f.state + result["freshness_scope"] = "relations" + result["pending_file_count"] = len(f.files) + result["pending_files"] = f.files[:min(maxLabelledPendingFiles, len(f.files))] + if f.state == "failed" { + result["recovery_hint"] = mutationFreshnessRecoveryHint() + } +} + +// line is the relations line a text answer leads with. +func (f relationFreshness) line() string { + return fmt.Sprintf("relations: %s (%d files)", f.state, len(f.files)) +} + +// requireFreshError is the answer to a require_fresh call whose wait ended +// with relations in its scope still lagging. +func (f relationFreshness) requireFreshError() *mcp.CallToolResult { + message := fmt.Sprintf("require_fresh: %s after waiting %d ms: %s", + f.line(), f.waited.Milliseconds(), strings.Join(f.files[:min(maxLabelledPendingFiles, len(f.files))], ", ")) + if f.state == "failed" { + message += "; " + mutationFreshnessRecoveryHint() + } + return mcp.NewToolResultError(message) +} + +// decorate makes a rendered text answer lead with its label: TOON and plain +// text start with the relations line; GCX keeps GCX1 as line 1, carries the +// label as header meta and the relations line as a comment on line 2. JSON +// already carries the label in its fields and passes through unchanged. +func (f relationFreshness) decorate(res *mcp.CallToolResult) *mcp.CallToolResult { + if f.state == "" || res == nil || res.IsError || len(res.Content) == 0 { + return res + } + text, ok := res.Content[0].(mcp.TextContent) + if !ok || strings.HasPrefix(strings.TrimSpace(text.Text), "{") { + return res + } + body := text.Text + if strings.HasPrefix(body, wire.Tag) { + body, _ = injectGCXHeaderMeta(body, map[string]any{ + "graph_freshness": f.state, + "freshness_scope": "relations", + "pending_file_count": len(f.files), + }) + head, rows, _ := strings.Cut(body, wire.RowSep) + body = head + wire.RowSep + wire.CommentPrefix + " " + f.line() + wire.RowSep + rows + } else { + body = f.line() + "\n" + body + } + return rebuildTextResult(res, body) +} diff --git a/internal/mcp/relation_freshness_test.go b/internal/mcp/relation_freshness_test.go new file mode 100644 index 000000000..4eef99956 --- /dev/null +++ b/internal/mcp/relation_freshness_test.go @@ -0,0 +1,194 @@ +package mcp + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + mcpgo "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/query" +) + +// newRelationFixture indexes a committed git repository in which Target +// (target.go) is called from caller.go, beside the unrelated other.go and +// namer.go. It returns the server and the indexed root receipts are keyed by. +func newRelationFixture(t *testing.T) (*Server, string) { + t.Helper() + dir := t.TempDir() + for name, src := range map[string]string{ + "target.go": "package p\n\nfunc Target() {}\n", + "caller.go": "package p\n\nfunc Caller() {\n\tTarget()\n}\n", + "other.go": "package p\n\nfunc Other() {}\n", + "namer.go": "package p\n\nfunc Namer() {}\n", + } { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(src), 0o644)) + } + checkoutMutationGit(t, dir, "init", "-q") + checkoutMutationGit(t, dir, "add", ".") + checkoutMutationGit(t, dir, "commit", "-q", "-m", "base") + + g := graph.New() + idx := indexer.New(g, testRegistry(), config.Default().Index, zap.NewNop()) + _, err := idx.Index(dir) + require.NoError(t, err) + return NewServer(query.NewEngine(g), g, idx, nil, zap.NewNop(), nil), idx.RootPath() +} + +// writeUnindexed changes a file on disk without telling the graph, the state +// a pending edit leaves behind. +func writeUnindexed(t *testing.T, root, name, src string) string { + t.Helper() + path := filepath.Join(root, name) + require.NoError(t, os.WriteFile(path, []byte(src), 0o644)) + return path +} + +func targetID(t *testing.T, s *Server) string { + t.Helper() + for _, n := range s.graph.AllNodes() { + if n.Name == "Target" && n.Kind == graph.KindFunction { + return n.ID + } + } + t.Fatal("Target is not indexed") + return "" +} + +func callChangeTool(t *testing.T, handler func(context.Context, mcpgo.CallToolRequest) (*mcpgo.CallToolResult, error), name string, args map[string]any) *mcpgo.CallToolResult { + t.Helper() + req := mcpgo.CallToolRequest{Params: mcpgo.CallToolParams{Name: name, Arguments: args}} + result, err := handler(context.Background(), req) + require.NoError(t, err) + require.NotNil(t, result) + return result +} + +func TestChangeToolsIgnorePendingReceiptOutsideScope(t *testing.T) { + s, root := newRelationFixture(t) + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n") + pendingFreshnessReceipt(s, "receipt-other", "", filepath.Join(root, "other.go"), 1) + + impact := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json"})) + detect := lifecycleResultPayload(t, callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"format": "json"})) + for tool, payload := range map[string]map[string]any{"impact": impact, "detect": detect} { + require.NotContains(t, payload, "graph_freshness", "%s labelled an edit outside its scope: %v", tool, payload) + require.Less(t, payload["barrier_wait_ms"], float64(100), "%s waited on an edit outside its scope", tool) + } + require.Equal(t, []any{"caller.go"}, detect["changed_files"]) + require.NotEmpty(t, detect["changed_symbols"]) +} + +func TestChangeToolsScopeIncludesFilesNamingTheSymbol(t *testing.T) { + s, root := newRelationFixture(t) + // namer.go now calls Target, but the graph has no edge for it yet; + // other.go names only a longer identifier. + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + pendingFreshnessReceipt(s, "receipt-other", "", + writeUnindexed(t, root, "other.go", "package p\n\nfunc Other() {\n\tTargetX()\n}\n"), 2) + + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json"})) + require.Equal(t, "pending", payload["graph_freshness"]) + require.Equal(t, "relations", payload["freshness_scope"]) + require.Equal(t, false, payload["complete"]) + require.Equal(t, []any{"namer.go"}, payload["pending_files"]) + require.Equal(t, float64(1), payload["pending_file_count"]) + require.Less(t, payload["barrier_wait_ms"], float64(500)) +} + +func TestChangeToolsUnresolvedIDDoesNotWidenBarrier(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-other", "", filepath.Join(root, "other.go"), 1) + pendingFreshnessReceipt(s, "receipt-fresh", "", + writeUnindexed(t, root, "fresh.go", "package p\n\nfunc Fresh() {}\n"), 2) + + // Defined by a pending edit the graph has not read: labelled, and bounded + // by the budget rather than by every receipt in the repository. + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": "fresh.go::Fresh", "format": "json"})) + require.Equal(t, "pending", payload["graph_freshness"]) + require.Equal(t, []any{"fresh.go"}, payload["pending_files"]) + require.Less(t, payload["barrier_wait_ms"], float64(500)) + + // No pending edit relates to it: answered at once, unlabelled. + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": "gone.go::Gone", "format": "json"})) + require.NotContains(t, payload, "graph_freshness") + require.Less(t, payload["barrier_wait_ms"], float64(100)) +} + +func TestChangeToolsRequireFreshWaitsUntilWaitDeadline(t *testing.T) { + s, root := newRelationFixture(t) + target := pendingFreshnessReceipt(s, "receipt-target", "", filepath.Join(root, "target.go"), 1) + caller := pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 2) + deadline := func(d time.Duration) string { return time.Now().Add(d).Format(time.RFC3339Nano) } + + // The receipt never lands: the wait runs past the default budget to the + // caller's deadline, then refuses. + started := time.Now() + result := callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "require_fresh": true, "wait_deadline": deadline(700 * time.Millisecond)}) + require.True(t, result.IsError) + require.GreaterOrEqual(t, time.Since(started), 600*time.Millisecond) + require.Contains(t, freshnessToolResultText(result), "relations: pending (2 files)") + require.Contains(t, freshnessToolResultText(result), "target.go") + + result = callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"require_fresh": true, "wait_deadline": deadline(500 * time.Millisecond)}) + require.True(t, result.IsError) + require.Contains(t, freshnessToolResultText(result), "caller.go") + + // The receipts land after the default budget but before the deadline: + // the answer waits for them and carries no label. + go func() { + time.Sleep(400 * time.Millisecond) + completeFreshnessReceipt(target, indexer.MutationResult{RequestedGeneration: 1, AppliedGeneration: 1, Reindexed: true}) + completeFreshnessReceipt(caller, indexer.MutationResult{RequestedGeneration: 2, AppliedGeneration: 2, Reindexed: true}) + }() + started = time.Now() + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": targetID(t, s), "format": "json", "require_fresh": true, "wait_deadline": deadline(5 * time.Second)})) + require.GreaterOrEqual(t, time.Since(started), 350*time.Millisecond) + require.NotContains(t, payload, "graph_freshness") +} + +func TestChangeAnswersStartWithRelationsLine(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n\tTarget()\n}\n"), 1) + const line = "relations: pending (1 files)" + id := targetID(t, s) + + for _, format := range []string{"toon", "gcx"} { + impact := freshnessToolResultText(callChangeTool(t, s.handleEnhancedChangeImpact, "explain_change_impact", + map[string]any{"ids": id, "format": format})) + lines := strings.SplitN(impact, "\n", 3) + require.GreaterOrEqual(t, len(lines), 2, impact) + if format == "gcx" { + require.True(t, strings.HasPrefix(lines[0], "GCX1 "), impact) + require.Contains(t, lines[0], " graph_freshness=pending") + require.Contains(t, lines[0], " pending_file_count=1") + require.Contains(t, lines[0], " barrier_wait_ms=") + require.Equal(t, "# "+line, lines[1]) + } else { + require.Equal(t, line, lines[0], impact) + } + } + + detect := freshnessToolResultText(callChangeTool(t, s.handleDetectChanges, "detect_changes", + map[string]any{"format": "toon"})) + require.Equal(t, line, strings.SplitN(detect, "\n", 2)[0], detect) +} diff --git a/internal/mcp/tools_analysis.go b/internal/mcp/tools_analysis.go index feae760a1..942467169 100644 --- a/internal/mcp/tools_analysis.go +++ b/internal/mcp/tools_analysis.go @@ -352,12 +352,39 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques if scopeErr := s.repoPrefixInSessionScope(ctx, repoPrefix, repoPrefix); scopeErr != nil { return mcp.NewToolResultError(scopeErr.Error()), nil } + checkoutScoped := control != nil && control.CheckoutScoped var pending bool var freshnessErr error - if control != nil && control.CheckoutScoped { + var label relationFreshness + var diff *analysis.DiffResult + if checkoutScoped { pending, freshnessErr = s.mutationFreshnessSummaryForRepos(ctx, repoPrefix) - } else if err := s.awaitMutationFreshnessForRepos(ctx, repoPrefix); err != nil { - return mcp.NewToolResultError("change detection refused a stale graph: " + err.Error()), nil + } else { + // Diff the files first: they are the relation scope the wait below + // is bounded to, so an edit elsewhere in the repository neither + // delays nor labels this answer. + files, err := analysis.MapGitDiffContext(ctx, nil, repoRoot, repoPrefix, scope, baseRef) + if err != nil { + return mcp.NewToolResultError(err.Error()), nil + } + keys := make([]string, len(files.ChangedFiles)) + for i, file := range files.ChangedFiles { + keys[i] = analysis.GraphKey(repoPrefix, file, analysis.RepoRelativePath) + } + freshness := takeRequestFreshness(&req) + label = s.awaitRelationFreshness(ctx, nil, keys, relationWaitDeadline(ctx, freshness)) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil + } + diff = files + } + respond := func(result map[string]any) (*mcp.CallToolResult, error) { + if !checkoutScoped { + result["barrier_wait_ms"] = label.waited.Milliseconds() + label.apply(result) + } + res, err := s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err } reader := s.readerFor(ctx) graphStatus, graphDetail := "ready", "" @@ -381,7 +408,12 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques reader = nil } - diff, err := analysis.MapGitDiffContext(ctx, reader, repoRoot, repoPrefix, scope, baseRef) + var err error + if diff == nil { + diff, err = analysis.MapGitDiffContext(ctx, reader, repoRoot, repoPrefix, scope, baseRef) + } else { + diff, err = analysis.JoinDiffSymbolsContext(ctx, reader, repoPrefix, diff) + } if err != nil { return mcp.NewToolResultError(err.Error()), nil } @@ -398,7 +430,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques fileChanges = []analysis.FileChange{} } if graphStatus != "ready" { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "changed_symbols": []any{}, "changed_files": changedFiles, "file_changes": fileChanges, "risk": "UNKNOWN", "complete": false, "graph_status": graphStatus, "summary": "Git file changes are available; symbol mapping and impact await an exact published graph", @@ -407,7 +439,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques } if len(diff.ChangedSymbols) == 0 { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "changed_symbols": []any{}, "changed_files": changedFiles, "file_changes": fileChanges, @@ -459,7 +491,7 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques req.GetBool("summary_only", false), req.GetInt("offset", 0), req.GetInt("limit", 100)) - return s.respondJSONOrTOON(ctx, req, detectResult) + return respond(detectResult) } // tryImpactAnalysisSnapshots reads optional cached enrichments without waiting @@ -487,8 +519,12 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo ids[i] = strings.TrimSpace(ids[i]) } - if freshnessErr := s.awaitMutationFreshnessForRepos(ctx, s.mutationReposForSymbolIDs(ctx, ids)...); freshnessErr != nil { - return mcp.NewToolResultError("change impact refused a stale graph: " + freshnessErr.Error()), nil + // Wait briefly for the edits these symbols' relations depend on, then + // answer; whatever is still lagging is labelled, not refused. + freshness := takeRequestFreshness(&req) + label := s.awaitRelationFreshness(ctx, ids, nil, relationWaitDeadline(ctx, freshness)) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil } // Keep the mandatory pre-edit safety gate well below host transport @@ -511,7 +547,9 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo "test_files": impact.TestFiles, "total_affected": impact.TotalAffected, "cross_repo_impact": impact.CrossRepoImpact, + "barrier_wait_ms": label.waited.Milliseconds(), } + label.apply(result) // GNX-3: by_depth_counts is the headline; the heavy by_depth rows are // paged (offset / limit) or dropped (summary_only) so the agent gets the @@ -605,18 +643,20 @@ func (s *Server) handleEnhancedChangeImpact(ctx context.Context, req mcp.CallToo } } - if s.isGCX(ctx, req) { + var res *mcp.CallToolResult + switch { + case s.isGCX(ctx, req): // encodeChangeImpact reads the same map shape we'd return as // JSON; routing through it keeps a single source of truth for // field names and avoids divergence on the next analyzer // addition. - return s.gcxResponseWithBudget(req)(encodeChangeImpact(result)) + res, err = s.gcxResponseWithBudget(req)(encodeChangeImpact(result)) + case s.isTOON(ctx, req): + res, err = returnTOON(result) + default: + res, err = s.respondJSONOrTOON(ctx, req, result) } - if s.isTOON(ctx, req) { - return returnTOON(result) - } - - return s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err } func impactComplete(impact *analysis.ImpactResult) bool { From c2f78c0c9b55c2269af6bfcf73af627ac81f023f Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 17:22:13 +0100 Subject: [PATCH 17/60] indexer: bound checkout discovery per family, not daemon-wide Every new-path discovery shared one daemon-wide limit of 32 jobs, and a job was allocated before its family was known. One hung mount, a burst of worktree adds in one family, or sessions opened in untracked Git directories could fill it, and from then on every new worktree in every family was refused with "capacity reached". Resolve the family from the .git marker before allocating: a path outside Git or in an unknown family returns not-found without a job or a Git spawn. A discovery already running for a path still answers it, so a binding that changes under its pending proof is still reported stale. A family runs at most 8 discoveries at once, and only its own new paths are refused while it is full; finished jobs that only retain a result do not count. The daemon-wide bound of 32 now counts only workers still running past their work deadline and refuses only families that own one, so work that ignores cancellation stays capped. The proof inside the job and its stale checks are unchanged. Refusals are a CheckoutDiscoveryPendingError naming the checkout root, the family and a reason (in_progress, draining, family_capacity, stuck_capacity). It still unwraps to ErrCheckoutMutationBusy, so every caller keeps treating it as retryable. --- internal/indexer/checkout_observe.go | 124 +++++++++- internal/indexer/checkout_observe_test.go | 216 +++++++++++++++--- .../checkout_observe_unknown_family_test.go | 4 +- 3 files changed, 299 insertions(+), 45 deletions(-) diff --git a/internal/indexer/checkout_observe.go b/internal/indexer/checkout_observe.go index f0af1dd59..8bdd9ac25 100644 --- a/internal/indexer/checkout_observe.go +++ b/internal/indexer/checkout_observe.go @@ -19,9 +19,60 @@ import ( const ( checkoutObservationTimeout = 250 * time.Millisecond checkoutObservationWorkTimeout = 5 * time.Second - checkoutObservationCapacity = 32 + // A family runs at most this many discoveries at once. Reaching it refuses + // only that family's new paths. + checkoutObservationFamilyCapacity = 8 + // Workers still running past their work deadline (Git or a mount that + // ignores cancellation) are bounded daemon-wide. Reaching it refuses only + // families that own such a worker. + checkoutObservationStuckCapacity = 32 ) +// Reasons a CheckoutDiscoveryPendingError carries. Every one is retryable. +const ( + checkoutDiscoveryInProgress = "in_progress" + checkoutDiscoveryDraining = "draining" + checkoutDiscoveryFamilyCapacity = "family_capacity" + checkoutDiscoveryStuckCapacity = "stuck_capacity" +) + +// CheckoutDiscoveryPendingError refuses a request whose checkout discovery +// cannot answer yet. It applies to Root, the checkout root holding the +// requested path, in Family; no request outside Root is refused because of it. +type CheckoutDiscoveryPendingError struct { + Root string + Family string + Reason string + cause error +} + +func (e *CheckoutDiscoveryPendingError) Error() string { + msg := fmt.Sprintf("%v: checkout discovery is pending for %s (family %s, reason %s)", ErrCheckoutMutationBusy, e.Root, e.Family, e.Reason) + if e.cause != nil { + msg += ": " + e.cause.Error() + } + return msg +} + +// Unwrap keeps the refusal a retryable busy-lane error and its cause, such as +// the request deadline, inspectable. +func (e *CheckoutDiscoveryPendingError) Unwrap() []error { + if e.cause == nil { + return []error{ErrCheckoutMutationBusy} + } + return []error{ErrCheckoutMutationBusy, e.cause} +} + +// checkoutObservationTarget is what a request learns from its .git marker, +// without running Git, before its discovery may take capacity. +type checkoutObservationTarget struct { + root, familyID string +} + +func (t checkoutObservationTarget) refusal(reason string) error { + return &CheckoutDiscoveryPendingError{Root: t.root, Family: t.familyID, Reason: reason} +} + type checkoutObservationProof struct { inventory *gitstate.FamilyInventory familyID string @@ -41,6 +92,7 @@ type checkoutObservationMarker struct { } type checkoutObservationJob struct { + familyID string ctx context.Context cancel context.CancelFunc proofReady chan struct{} @@ -61,8 +113,9 @@ type checkoutObservationJob struct { // up to five seconds across retries. Slow Git startup therefore cannot restart // the same unfinished work indefinitely. Graph builds use existing activation. // -// found=false, err=nil means the path has no known family to serve it from. -// A busy/error outcome carries no permission to fall through to another graph. +// found=false, err=nil means the path has no known family to serve it from; +// that is decided from its .git marker before any discovery work starts. A +// busy/error outcome carries no permission to fall through to another graph. // Explicit selectors must provide an authorizer for the known primary prefix; // it runs outside locks and before any catalog observation or build activation. func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string, authorize ...func(string) error) (checkout store_sqlite.Checkout, found bool, err error) { @@ -77,6 +130,7 @@ func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string } ctx, cancel := context.WithTimeout(ctx, checkoutObservationTimeout) defer cancel() + var target checkoutObservationTarget defer func() { if err != nil && ctx.Err() != nil { err = errors.Join(ctx.Err(), err) @@ -84,14 +138,22 @@ func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string return } } - if err != nil && (errors.Is(err, context.DeadlineExceeded) || retryableCheckoutRefreshError(err)) { - err = fmt.Errorf("%w: selected checkout discovery is pending: %w", ErrCheckoutMutationBusy, err) + var pending *CheckoutDiscoveryPendingError + if err != nil && !errors.As(err, &pending) && (errors.Is(err, context.DeadlineExceeded) || retryableCheckoutRefreshError(err)) { + err = &CheckoutDiscoveryPendingError{Root: target.root, Family: target.familyID, Reason: checkoutDiscoveryInProgress, cause: err} } }() - job, err := l.checkoutObservation(pathkey.CanonicalExistingRoot(path)) + path = pathkey.CanonicalExistingRoot(path) + target, known, err := l.observationTarget(ctx, path) if err != nil { return checkout, false, err } + // A discovery already running for this path still answers it, so a binding + // that changed under its pending proof is reported stale. + job, err := l.checkoutObservation(path, target, known) + if err != nil || job == nil { + return checkout, false, err + } select { case <-ctx.Done(): return checkout, false, ctx.Err() @@ -147,7 +209,24 @@ func (job *checkoutObservationJob) invalidateStaleProof(err error) error { return err } -func (l *CheckoutLifecycle) checkoutObservation(path string) (*checkoutObservationJob, error) { +// observationTarget resolves path's family from its .git marker alone. A path +// outside Git, or in a family the catalog does not know, has nothing to +// observe and never takes a discovery slot; ordinary scope resolution handles +// it. The job's own proof still resolves the family through Git. +func (l *CheckoutLifecycle) observationTarget(ctx context.Context, path string) (checkoutObservationTarget, bool, error) { + root, commonDir, ok := gitstate.MarkerCommonDir(path) + if !ok { + return checkoutObservationTarget{}, false, nil + } + target := checkoutObservationTarget{root: root, familyID: FamilyIDFor(commonDir)} + _, known, err := l.catalog.GetRepositoryFamily(ctx, target.familyID) + return target, known, err +} + +// checkoutObservation joins path's running discovery. Without one it starts +// a discovery when start is set and target's family has capacity, and +// otherwise returns no job. +func (l *CheckoutLifecycle) checkoutObservation(path string, target checkoutObservationTarget, start bool) (*checkoutObservationJob, error) { l.observationMu.Lock() defer l.observationMu.Unlock() if l.observationClosed { @@ -157,16 +236,39 @@ func (l *CheckoutLifecycle) checkoutObservation(path string) (*checkoutObservati if job.ctx.Err() == nil { return job, nil } - return nil, fmt.Errorf("%w: expired checkout discovery is draining", ErrCheckoutMutationBusy) + return nil, target.refusal(checkoutDiscoveryDraining) + } + if !start { + return nil, nil + } + // One family's burst or stall never refuses another family's paths. + inFlight, stuck, familyStuck := 0, 0, false + for _, job := range l.observationJobs { + select { + case <-job.done: + continue // Finished; at most a retained result. + default: + } + expired := job.ctx.Err() != nil + if expired { + stuck++ + } + if job.familyID == target.familyID { + inFlight++ + familyStuck = familyStuck || expired + } + } + if inFlight >= checkoutObservationFamilyCapacity { + return nil, target.refusal(checkoutDiscoveryFamilyCapacity) } - if len(l.observationJobs) >= checkoutObservationCapacity { - return nil, fmt.Errorf("%w: checkout discovery capacity reached", ErrCheckoutMutationBusy) + if familyStuck && stuck >= checkoutObservationStuckCapacity { + return nil, target.refusal(checkoutDiscoveryStuckCapacity) } if l.observationJobs == nil { l.observationJobs = make(map[string]*checkoutObservationJob) } ctx, cancel := context.WithTimeout(context.Background(), checkoutObservationWorkTimeout) - job := &checkoutObservationJob{ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} + job := &checkoutObservationJob{familyID: target.familyID, ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} l.observationJobs[path] = job l.observationWG.Add(1) go l.runCheckoutObservation(path, job) diff --git a/internal/indexer/checkout_observe_test.go b/internal/indexer/checkout_observe_test.go index 2dcb634be..7ebe34e15 100644 --- a/internal/indexer/checkout_observe_test.go +++ b/internal/indexer/checkout_observe_test.go @@ -201,7 +201,7 @@ func TestObserveCheckoutPathRetainsStaleObservationAcrossRetries(t *testing.T) { if job == nil || job.ctx.Err() != nil { t.Fatal("stale refusal did not retain a live discovery job") } - // The retained job owns capacity and a worker until expiry or shutdown. + // The retained job holds a worker until expiry or shutdown. lc.closeCheckoutObservations() if !errors.Is(job.ctx.Err(), context.Canceled) { t.Fatalf("Close did not cancel retained stale discovery: %v", job.ctx.Err()) @@ -264,6 +264,128 @@ func TestObserveCheckoutPathUnknownFamilyAndDeniedScopeHaveNoSideEffects(t *test } } +// One family's stuck discovery bounds only that family. Its own new paths are +// refused, another known family's new worktree still gets its identity, and a +// path outside every known family never takes a discovery slot. +func TestObserveCheckoutPathStuckFamilyLeavesOtherRequestsAlone(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + base := filepath.Dir(primary) + other := filepath.Join(base, "other") + if err := os.Mkdir(other, 0o755); err != nil { + t.Fatal(err) + } + builderGit(t, other, "init", "--initial-branch=main") + builderWriteTree(t, other, builderTreeA()) + builderGit(t, other, "add", "-A") + builderGit(t, other, "commit", "-m", "other") + tracked, err := lc.Register(t.Context(), config.RepoEntry{Path: other, Name: "other"}, TrackSourceCLI) + if err != nil || tracked.CatalogErr != nil { + t.Fatalf("register other family: %v %v", err, tracked.CatalogErr) + } + stuck := filepath.Join(base, "stuck") + builderGit(t, primary, "worktree", "add", "-b", "stuck", stuck) + late := filepath.Join(base, "late") + builderGit(t, other, "worktree", "add", "-b", "late", late) + outside := []string{filepath.Join(base, "plain")} + for i := range 40 { + outside = append(outside, filepath.Join(base, fmt.Sprintf("unknown-%d", i))) + } + for i, dir := range outside { + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if i > 0 { + builderGit(t, dir, "init", "--initial-branch=main") + } + } + release := make(chan struct{}) + defer close(release) + stuckRoot := pathkey.CanonicalExistingRoot(stuck) + var entered atomic.Int32 + lc.observationInventory = func(ctx context.Context, path string) (*gitstate.FamilyInventory, error) { + if pathkey.HasPathPrefix(path, stuckRoot) { + entered.Add(1) + <-release // Git or a mount that ignores cancellation. + return nil, context.Canceled + } + return gitstate.Inventory(ctx, path) + } + activeJobs := func() int { + lc.observationMu.Lock() + defer lc.observationMu.Unlock() + return len(lc.observationJobs) + } + observeOutside := func(phase string, wantJobs int) { + t.Helper() + for _, dir := range outside { + _, found, err := lc.ObserveCheckoutPath(t.Context(), dir) + if jobs := activeJobs(); found || err != nil || jobs != wantJobs { + t.Errorf("%s: %s is in no known family: found=%v err=%v jobs=%d, want %d", phase, dir, found, err, jobs, wantJobs) + return + } + } + } + observeOutside("idle", 0) + + requests := make([]string, 40) + errs := make([]error, len(requests)) + for i := range requests { + requests[i] = filepath.Join(stuck, fmt.Sprintf("request-%d", i)) + if err := os.Mkdir(requests[i], 0o755); err != nil { + t.Fatal(err) + } + } + var wg sync.WaitGroup + for i := range requests { + wg.Go(func() { _, _, errs[i] = lc.ObserveCheckoutPath(t.Context(), requests[i]) }) + } + wg.Wait() + for i, err := range errs { + if !errors.Is(err, ErrCheckoutMutationBusy) { + t.Fatalf("stuck family request %d: %v", i, err) + } + } + lc.observationMu.Lock() + jobs := make([]*checkoutObservationJob, 0, len(lc.observationJobs)) + draining := "" + for path, job := range lc.observationJobs { + jobs = append(jobs, job) + draining = path + } + lc.observationMu.Unlock() + if len(jobs) != checkoutObservationFamilyCapacity { + t.Errorf("one family holds %d discovery jobs, want its own bound of %d", len(jobs), checkoutObservationFamilyCapacity) + } + deadline := time.Now().Add(30 * time.Second) + for int(entered.Load()) < len(jobs) { + if time.Now().After(deadline) { + t.Fatalf("workers did not reach the stalled inventory: %d of %d", entered.Load(), len(jobs)) + } + time.Sleep(time.Millisecond) + } + for _, job := range jobs { + job.cancel() // Past the work deadline; the workers keep running. + } + + observeOutside("stuck", len(jobs)) + checkout, found, err := observeCheckoutUntilSettled(t, lc, late) + if err != nil || !found || checkout.FamilyID != tracked.FamilyID || !pathkey.EqualPaths(checkout.RootPath, pathkey.CanonicalExistingRoot(late)) { + t.Errorf("another family's new worktree was refused: %+v found=%v err=%v", checkout, found, err) + } + extra := filepath.Join(stuck, "extra") + if err := os.Mkdir(extra, 0o755); err != nil { + t.Fatal(err) + } + _, _, err = lc.ObserveCheckoutPath(t.Context(), extra) + var pending *CheckoutDiscoveryPendingError + if !errors.Is(err, ErrCheckoutMutationBusy) || !errors.As(err, &pending) || !pathkey.EqualPaths(pending.Root, stuckRoot) || pending.Family != familyID || pending.Reason != checkoutDiscoveryFamilyCapacity { + t.Errorf("stuck family's new path: %v", err) + } + if _, _, err := lc.ObserveCheckoutPath(t.Context(), draining); !errors.Is(err, ErrCheckoutMutationBusy) || checkoutDiscoveryReason(err) != checkoutDiscoveryDraining { + t.Errorf("stuck path did not keep its draining refusal: %v", err) + } +} + func TestObserveCheckoutPathCancellationAndClosingDoNotAllocate(t *testing.T) { lc, catalog, primary, familyID, _, closeFixture := newCheckoutObservationFixture(t) worktree := filepath.Join(filepath.Dir(primary), "canceled") @@ -390,54 +512,84 @@ func TestObserveCheckoutPathSlowObservationRunsOnceAcrossRetries(t *testing.T) { } } -func TestObserveCheckoutPathExpiredJobsStayBoundedUntilTheyDrain(t *testing.T) { +// checkoutDiscoveryReason is the reason of a pending discovery refusal, "" +// for any other outcome. +func checkoutDiscoveryReason(err error) string { + var pending *CheckoutDiscoveryPendingError + if errors.As(err, &pending) { + return pending.Reason + } + return "" +} + +func TestObserveCheckoutPathStuckJobsStayBoundedWithoutRefusingOtherFamilies(t *testing.T) { lc, catalog, primary, familyID, _, _ := newCheckoutObservationFixture(t) release := make(chan struct{}) defer close(release) - entered := make(chan struct{}) - var started atomic.Int32 + var entered atomic.Int32 lc.observationInventory = func(context.Context, string) (*gitstate.FamilyInventory, error) { - if started.Add(1) == checkoutObservationCapacity { - close(entered) - } + entered.Add(1) <-release // Deliberately model a dependency that ignores cancellation. return nil, context.Canceled } - jobs := make([]*checkoutObservationJob, 0, checkoutObservationCapacity) - for i := 0; i < checkoutObservationCapacity; i++ { - path := filepath.Join(primary, fmt.Sprintf("request-%d", i)) - if err := os.Mkdir(path, 0o755); err != nil { + // Capacity is accounted to the family the pre-check names. Every worker + // proves the fixture's real family through Git, then stalls in Inventory. + observe := func(family string, i int) (*checkoutObservationJob, error) { + path := filepath.Join(primary, fmt.Sprintf("%s-%d", family, i)) + if err := os.MkdirAll(path, 0o755); err != nil { t.Fatal(err) } - job, err := lc.checkoutObservation(path) - if err != nil { - t.Fatal(err) + return lc.checkoutObservation(path, checkoutObservationTarget{root: primary, familyID: family}, true) + } + var jobs []*checkoutObservationJob + for f := 0; f < checkoutObservationStuckCapacity/checkoutObservationFamilyCapacity; f++ { + family := fmt.Sprintf("full-%d", f) + for i := 0; i < checkoutObservationFamilyCapacity; i++ { + job, err := observe(family, i) + if err != nil { + t.Fatal(err) + } + jobs = append(jobs, job) + } + if _, err := observe(family, checkoutObservationFamilyCapacity); checkoutDiscoveryReason(err) != checkoutDiscoveryFamilyCapacity { + t.Fatalf("full family admitted another discovery: %v", err) } - jobs = append(jobs, job) } + partial, err := observe("partial", 0) + if err != nil { + t.Fatal(err) + } + jobs = append(jobs, partial) ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() - select { - case <-entered: - case <-ctx.Done(): - t.Fatalf("workers did not enter inventory: started=%d: %v", started.Load(), ctx.Err()) - } - if got := started.Load(); got != checkoutObservationCapacity { - t.Fatalf("workers did not start: %d", got) + for int(entered.Load()) < len(jobs) { + if ctx.Err() != nil { + t.Fatalf("workers did not enter inventory: %d of %d", entered.Load(), len(jobs)) + } + time.Sleep(time.Millisecond) } for _, job := range jobs { job.cancel() // Expiry must not release capacity before the worker exits. } - for i := 0; i < 2*checkoutObservationCapacity; i++ { - if _, err := lc.checkoutObservation(filepath.Join(primary, fmt.Sprintf("extra-%d", i))); !errors.Is(err, ErrCheckoutMutationBusy) { - t.Fatalf("canceled but running jobs released capacity: %v", err) + // Past the daemon-wide stuck bound, a family that owns stuck work is + // refused below its own bound, and a stuck path keeps draining. + for i := 1; i < 2*checkoutObservationStuckCapacity; i++ { + if _, err := observe("partial", i); checkoutDiscoveryReason(err) != checkoutDiscoveryStuckCapacity { + t.Fatalf("family with stuck work escaped the stuck bound: %v", err) } } + if _, err := observe("partial", 0); checkoutDiscoveryReason(err) != checkoutDiscoveryDraining { + t.Fatalf("stuck path was not draining: %v", err) + } + // A family without stuck work is never refused for another family's. + if job, err := observe("fresh", 0); err != nil || job == nil { + t.Fatalf("family without stuck work was refused: %v", err) + } lc.observationMu.Lock() active := len(lc.observationJobs) lc.observationMu.Unlock() - if active != checkoutObservationCapacity || started.Load() != checkoutObservationCapacity { - t.Fatalf("discovery bound escaped: jobs=%d workers=%d", active, started.Load()) + if want := len(jobs) + 1; active != want || int(entered.Load()) > want { + t.Fatalf("discovery bound escaped: jobs=%d workers=%d, want %d", active, entered.Load(), want) } rows, err := catalog.ListCheckouts(t.Context(), familyID) if err != nil || len(rows) != 1 { @@ -455,7 +607,7 @@ func TestObserveCheckoutPathCloseCancelsAndJoinsMetadata(t *testing.T) { close(exited) return nil, ctx.Err() } - if _, err := lc.checkoutObservation(primary); err != nil { + if _, err := lc.checkoutObservation(primary, checkoutObservationTarget{}, true); err != nil { t.Fatal(err) } <-entered @@ -473,7 +625,7 @@ func TestObserveCheckoutPathCloseCancelsAndJoinsMetadata(t *testing.T) { if active != 0 { t.Fatalf("Close retained %d discovery jobs", active) } - if _, err := lc.checkoutObservation(primary); !errors.Is(err, ErrCheckoutRefreshStopped) { + if _, err := lc.checkoutObservation(primary, checkoutObservationTarget{}, true); !errors.Is(err, ErrCheckoutRefreshStopped) { t.Fatalf("closed lifecycle admitted new work: %v", err) } } @@ -504,7 +656,7 @@ func TestObserveCheckoutPathRejectsInventoryWithReboundGitMarker(t *testing.T) { // Judge the job that received the pre-rebind inventory, not a request retry // loop. Slow Git can outlast the request wait, the refused job is then // forgotten, and a retry's fresh inventory legitimately sees the new binding. - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } @@ -531,7 +683,7 @@ func TestObserveCheckoutPathRejectsBindingChangesWhileProofIsPending(t *testing. lc, catalog, primary, familyID, _, _ := newCheckoutObservationFixture(t) worktree := filepath.Join(filepath.Dir(primary), "pending-proof") builderGit(t, primary, "worktree", "add", "-b", "pending-proof", worktree) - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } @@ -630,7 +782,7 @@ func completedCheckoutObservationForTest(t *testing.T) (*CheckoutLifecycle, *che calls.Add(1) return gitstate.Inventory(ctx, path) } - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(worktree), checkoutObservationTarget{}, true) if err != nil { t.Fatal(err) } diff --git a/internal/indexer/checkout_observe_unknown_family_test.go b/internal/indexer/checkout_observe_unknown_family_test.go index 2a1a25258..d69545bc7 100644 --- a/internal/indexer/checkout_observe_unknown_family_test.go +++ b/internal/indexer/checkout_observe_unknown_family_test.go @@ -56,7 +56,7 @@ func TestObserveCheckoutPathTrackedNonGitUnknownAncestorSkipsInventory(t *testin return gitstate.Inventory(ctx, path) } for range 3 { - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root), checkoutObservationTarget{}, true) require.NoError(t, err) select { case <-job.done: @@ -90,7 +90,7 @@ func TestObserveCheckoutPathTrackedNonGitUnknownAncestorSkipsInventory(t *testin // worktree of the known family. No remembered negative may hide its identity. require.NoError(t, os.Rename(root, root+"-before-worktree")) builderGit(t, primary, "worktree", "add", "-b", "competitor-worktree", root) - job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root)) + job, err := lc.checkoutObservation(pathkey.CanonicalExistingRoot(root), checkoutObservationTarget{}, true) require.NoError(t, err) select { case <-job.proofReady: From 213c320e6f8bfe45d0f5b225e1da04122f98d564 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 18:05:20 +0100 Subject: [PATCH 18/60] daemon: name the pending checkout root in discovery refusals A discovery-pending admission refusal reported only the session cwd, so an agent or hook could not tell one pending worktree from an unavailable daemon. It now carries the checkout root as data.path, plus data.family, data.reason and state discovery_pending, and stays retryable. A daemon-level test pins the containment: with one family's discovery hung past its deadline, sessions in its primary checkout, in a directory beside an unregistered nested worktree and in an unrelated tracked repository keep answering, another family's new worktree gets its identity, and only paths inside the pending worktrees are refused. checkoutForRequestPath now states that guard. --- cmd/gortex/daemon_mcp.go | 13 +- .../daemon_mcp_checkout_admission_test.go | 10 + cmd/gortex/daemon_mcp_discovery_scope_test.go | 175 ++++++++++++++++++ internal/mcp/checkout_binding.go | 5 + 4 files changed, 198 insertions(+), 5 deletions(-) create mode 100644 cmd/gortex/daemon_mcp_discovery_scope_test.go diff --git a/cmd/gortex/daemon_mcp.go b/cmd/gortex/daemon_mcp.go index 7e5c20e90..29035320f 100644 --- a/cmd/gortex/daemon_mcp.go +++ b/cmd/gortex/daemon_mcp.go @@ -410,17 +410,20 @@ func checkoutAdmissionError(cwd string, inbound []byte, err error) []byte { } _ = json.Unmarshal(inbound, &req) code, retryable := checkoutAdmissionFailure(err) + data := map[string]any{"error_code": code, "path": cwd, "retryable": retryable} + // A pending discovery names the checkout root it applies to, so a caller + // can tell one pending worktree from an unavailable daemon. + var pending *indexer.CheckoutDiscoveryPendingError + if errors.As(err, &pending) { + data["path"], data["family"], data["reason"], data["state"] = pending.Root, pending.Family, pending.Reason, "discovery_pending" + } out, _ := json.Marshal(map[string]any{ "jsonrpc": "2.0", "id": req.ID, "error": map[string]any{ "code": -32000, "message": "checkout discovery unavailable: " + err.Error(), - "data": map[string]any{ - "error_code": code, - "path": cwd, - "retryable": retryable, - }, + "data": data, }, }) return out diff --git a/cmd/gortex/daemon_mcp_checkout_admission_test.go b/cmd/gortex/daemon_mcp_checkout_admission_test.go index 4a38224da..024e07ed3 100644 --- a/cmd/gortex/daemon_mcp_checkout_admission_test.go +++ b/cmd/gortex/daemon_mcp_checkout_admission_test.go @@ -65,6 +65,16 @@ func TestDispatcher_CheckoutAdmissionFailurePreservesClassification(t *testing.T } } +// Hooks and agents key on this refusal to tell one pending worktree from an +// unavailable daemon. +func TestCheckoutAdmissionErrorPendingDiscoveryGolden(t *testing.T) { + err := fmt.Errorf("observe checkout: %w", &indexer.CheckoutDiscoveryPendingError{Root: "/repos/app-wt", Family: "family-0123", Reason: "family_capacity"}) + got := checkoutAdmissionError("/repos/app-wt/internal", []byte(`{"jsonrpc":"2.0","id":7,"method":"tools/call"}`), err) + const want = `{"error":{"code":-32000,"data":{"error_code":"view_building","family":"family-0123","path":"/repos/app-wt","reason":"family_capacity","retryable":true,"state":"discovery_pending"},` + + `"message":"checkout discovery unavailable: observe checkout: indexer: checkout mutation lane is busy; retry: checkout discovery is pending for /repos/app-wt (family family-0123, reason family_capacity)"},"id":7,"jsonrpc":"2.0"}` + require.Equal(t, want, string(got)) +} + type checkoutAdmissionSQLiteError int func (e checkoutAdmissionSQLiteError) Error() string { return fmt.Sprintf("SQLite error %d", e) } diff --git a/cmd/gortex/daemon_mcp_discovery_scope_test.go b/cmd/gortex/daemon_mcp_discovery_scope_test.go new file mode 100644 index 000000000..dc81ae215 --- /dev/null +++ b/cmd/gortex/daemon_mcp_discovery_scope_test.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/daemon" + "github.com/zzet/gortex/internal/gitstate" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/graphview" + "github.com/zzet/gortex/internal/indexer" + gortexmcp "github.com/zzet/gortex/internal/mcp" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" + "github.com/zzet/gortex/internal/pathkey" + "github.com/zzet/gortex/internal/query" + "github.com/zzet/gortex/internal/reconcile" + "github.com/zzet/gortex/internal/search" +) + +func discoveryScopeRepo(t *testing.T, dir string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "internal"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "fixture.go"), []byte("package fixture\n\nfunc Value() int { return 1 }\n"), 0o600)) + guidanceGit(t, dir, "init", "-q", "-b", "main") + guidanceGit(t, dir, "add", ".") + guidanceGit(t, dir, "commit", "-q", "-m", "fixture") +} + +// A checkout discovery that hangs refuses only its own worktree. The primary +// checkout, its directories outside a pending nested worktree, an unrelated +// tracked repository and another family's new worktree keep answering. +func TestDispatcher_PendingDiscoveryStaysInItsWorktree(t *testing.T) { + base, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + primary, other := filepath.Join(base, "primary"), filepath.Join(base, "other") + discoveryScopeRepo(t, primary) + discoveryScopeRepo(t, other) + gc := &config.GlobalConfig{} + gc.SetConfigPath(filepath.Join(base, "config.yaml")) + require.NoError(t, gc.Save()) + cm, err := config.NewConfigManager(gc.ConfigPath()) + require.NoError(t, err) + store, err := store_sqlite.Open(filepath.Join(base, "store.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) + registry := parser.NewRegistry() + registry.Register(languages.NewGoExtractor()) + mi := indexer.NewMultiIndexer(store, registry, search.NewNull(), cm, zap.NewNop()) + t.Cleanup(func() { _ = mi.Close(context.Background()) }) + lifecycle, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: zap.NewNop()}) + require.NoError(t, err) + t.Cleanup(func() { _ = lifecycle.Close() }) + // The primary family's new worktrees hang in Git metadata and ignore + // cancellation, like Git on a hung mount. + stuck, nested := filepath.Join(base, "stuck"), filepath.Join(primary, "nested") + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + reconcile.WithHEADSampler(func(ctx context.Context, root string) (gitstate.HEADState, error) { + if root = pathkey.CanonicalExistingRoot(root); pathkey.HasPathPrefix(root, stuck) || pathkey.HasPathPrefix(root, nested) { + <-release + return gitstate.HEADState{}, errors.New("released hung HEAD sample") + } + return gitstate.SampleHEAD(ctx, root) + })(lifecycle.Reconciler()) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + families := map[string]string{} + for name, root := range map[string]string{"primary": primary, "other": other} { + registered, err := lifecycle.Register(ctx, config.RepoEntry{Path: root, Name: name}, indexer.TrackSourceCLI) + require.NoError(t, err) + require.NoError(t, registered.CatalogErr) + families[name] = registered.FamilyID + } + srv := gortexmcp.NewServer(query.NewEngine(store), store, nil, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm, CheckoutLifecycle: lifecycle}) + srv.SetMaterializer(&graphview.Materializer{Store: store, Catalog: store.Catalog()}) + d := newMCPDispatcher(srv, mi, zap.NewNop()) + + late := filepath.Join(base, "late") + guidanceGit(t, primary, "worktree", "add", "-q", "-b", "stuck", stuck) + guidanceGit(t, primary, "worktree", "add", "-q", "-b", "nested", nested) + guidanceGit(t, other, "worktree", "add", "-q", "-b", "late", late) + const initializeFrame = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"test","version":"1"}}}` + const searchFrame = `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"search","arguments":{"query":"Value"}}}` + searchFrom := func(id, cwd string) map[string]any { + sess := &daemon.Session{ID: id, CWD: cwd} + dispatchJSON(t, d, sess, initializeFrame) + return dispatchJSON(t, d, sess, searchFrame) + } + + // Sessions in many directories of the hung worktree. The second round + // joins the first round's discoveries, so their workers hang. + stuckSessions := make([]*daemon.Session, 40) + for i := range stuckSessions { + cwd := filepath.Join(stuck, fmt.Sprintf("session-%d", i)) + require.NoError(t, os.Mkdir(cwd, 0o700)) + stuckSessions[i] = &daemon.Session{ID: fmt.Sprintf("stuck-%d", i), CWD: cwd} + } + stuckReplies := make([][]byte, len(stuckSessions)) + stuckErrs := make([]error, len(stuckSessions)) + for range 2 { + var wg sync.WaitGroup + for i, sess := range stuckSessions { + wg.Go(func() { stuckReplies[i], stuckErrs[i] = d.Dispatch(context.Background(), sess, []byte(searchFrame)) }) + } + wg.Wait() + } + + // Another family's new worktree gets its identity; it may wait for its own + // discovery, never for the hung one. + deadline := time.Now().Add(30 * time.Second) + for { + reply := searchFrom("late", late) + refusal, refused := reply["error"].(map[string]any) + if !refused { + break + } + data, _ := refusal["data"].(map[string]any) + require.Equal(t, "in_progress", data["reason"], "another family's discovery refused this worktree: %v", reply) + require.True(t, time.Now().Before(deadline), "the other family's worktree discovery did not finish: %v", reply) + time.Sleep(10 * time.Millisecond) + } + overview, err := lifecycle.FamiliesOverview(ctx, families["other"]) + require.NoError(t, err) + allocated := false + for _, family := range overview.Families { + for _, checkout := range family.Checkouts { + allocated = allocated || pathkey.EqualPaths(checkout.RootPath, late) + } + } + require.True(t, allocated, "the other family's new worktree has no identity: %+v", overview.Families) + + for _, cwd := range []string{primary, filepath.Join(primary, "internal"), other} { + reply := searchFrom("answer-"+cwd, cwd) + require.Contains(t, toolResultText(t, reply), "Value", "%s must answer while another checkout's discovery hangs: %v", cwd, reply) + require.NotEqual(t, true, reply["result"].(map[string]any)["isError"], "%s: %v", cwd, reply) + } + + // Only a path inside the unregistered nested worktree waits for discovery. + reply := searchFrom("nested", nested) + text := toolResultText(t, reply) + require.Equal(t, true, reply["result"].(map[string]any)["isError"], "nested worktree answered from its parent: %v", reply) + require.Contains(t, text, "view_building") + require.Contains(t, text, nested, "the refusal names the pending nested worktree") + + familyCapacity := 0 + for i, raw := range stuckReplies { + require.NoError(t, stuckErrs[i]) + var parsed map[string]any + require.NoError(t, json.Unmarshal(raw, &parsed)) + data := errorData(t, parsed) + require.Equal(t, "view_building", data["error_code"], "%v", parsed) + require.Equal(t, true, data["retryable"], "%v", parsed) + path, _ := data["path"].(string) + require.True(t, pathkey.EqualPaths(stuck, path), "the refusal names the pending checkout root, not the session directory: %v", parsed) + require.Equal(t, families["primary"], data["family"], "%v", parsed) + require.Contains(t, []any{"in_progress", "family_capacity"}, data["reason"], "%v", parsed) + if data["reason"] == "family_capacity" { + familyCapacity++ + } + } + require.NotZero(t, familyCapacity, "one family's discoveries are bounded") +} diff --git a/internal/mcp/checkout_binding.go b/internal/mcp/checkout_binding.go index 1e2391137..b1517450a 100644 --- a/internal/mcp/checkout_binding.go +++ b/internal/mcp/checkout_binding.go @@ -759,6 +759,11 @@ func exactnessWithdrawnRefusal(reason string) error { // session scope and request view selection. A cache miss observes only this // checkout of an already known Git family; it never tracks a new repository or // waits for parsing. Existing identities take the catalog-only fast path. +// +// A path inside a registered root, its subdirectories included, is answered +// by that root and is never refused for a discovery pending elsewhere. The one +// exception is a path inside a nested worktree that discovery has not +// registered yet: only such a path is observed, so only it can be refused. func (s *Server) checkoutForRequestPath(ctx context.Context, path string) (store_sqlite.Checkout, bool, error) { checkout, found, err := s.registeredCheckoutForPath(ctx, path) if err != nil { From b3532db83d2fc74daf18ed448a40b044755dcdca Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:07:38 +0100 Subject: [PATCH 19/60] test(indexer): fold the first-index flush panic test into the seal-panic row TestFirstIndexFlushPanicPropagates repeated the "seal panic" row of TestFirstIndexFailureRestoresStore: the same injected seal panic, the same backstop retry, synchronous, index-list, disk-target and receipt assertions. Only its memory-lease checks were its own, so they move into that row and the standalone test goes. --- internal/indexer/first_index_bulk_test.go | 39 +++-------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 2ca45f673..475eb3818 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -451,6 +451,9 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { ctx = progress.WithReporter(ctx, firstIndexPanicReporter{}) } idx := newFirstIndexBulkIndexer(t, store, "repo") + if mode == "seal panic" { + idx.indexMemoryAdmission = newIndexMemoryAdmissionBudget(defaultIndexMemoryAdmissionBytes) + } dir := firstIndexFixtureRepo(t) switch mode { @@ -463,6 +466,8 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { case "seal panic": require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") + assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") + assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") case "cancel before drain": _, err := idx.IndexCtx(ctx, dir) require.ErrorIs(t, err, context.Canceled) @@ -506,40 +511,6 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { } } -func TestFirstIndexFlushPanicPropagates(t *testing.T) { - // Given a real store whose dense-index seal panics once, inside the - // drain's explicit FlushBulk. - store, db := newFirstIndexStore(t) - indexes := firstIndexSchemaIndexes(t, db) - writerSync := firstIndexWriterSynchronous(t, store) - seals := 0 - defer store.SetBulkSealHookForTest(func(string) error { - seals++ - if seals == 1 { - panic("injected seal panic") - } - return nil - })() - idx := newFirstIndexBulkIndexer(t, store, "repo") - idx.indexMemoryAdmission = newIndexMemoryAdmissionBudget(defaultIndexMemoryAdmissionBytes) - - // When the completed shadow drain flushes, the panic reaches the caller. - require.PanicsWithValue(t, "injected seal panic", func() { - _, _ = idx.IndexCtx(context.Background(), firstIndexFixtureRepo(t)) - }) - - // Then the backstop retried the flush once while the panic unwound, and - // the retry sealed and closed the window: the write gate is free, the - // writer is back at its own durability, and every index is rebuilt. - assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") - assert.Equal(t, writerSync, firstIndexWriterSynchronous(t, store)) - assertFirstIndexRestored(t, store, db, indexes) - assert.Same(t, store, idx.graph) - assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") - assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") - assertFirstIndexNoReceipts(t, store, "repo") -} - type firstIndexDrainPanicReporter struct{} func (firstIndexDrainPanicReporter) Report(stage string, _, _ int) { From 408aafbb89b401c17cb37b03847baab04137db81 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 17:26:23 +0100 Subject: [PATCH 20/60] hooks: bound the briefing's detect_changes call and report lagging relations The Stop briefing and the Codex PostToolUse briefing call detect_changes with only the 5 s transport bound, and fold every failure into an empty string. While edits were pending the change tool refused, the hook could not decode the error, and the agent was told nothing about its change set. detect_changes now gets 300 ms through callServerToolWithin, which tells a reply that did not arrive in time apart from an unreachable bridge. The call carries a wait_deadline 150 ms out, so the daemon labels the edits it is still publishing instead of waiting 250 ms for them on top of the diff and the walk, which would land the labelled answer past the budget. An empty reply after that deadline, which is how the daemon refuses a deadline that passed before it took the call up, counts as a stall. A stall, or an answer labelled pending or failed (graph_freshness on a primary checkout, graph_status on a worktree), yields a short briefing with the relations line and the changed files instead of silence or diagnostics over lagging relations. A stall says only that detect_changes did not answer, and stays silent for a session that recorded no writes, so a slow daemon does not put a warning into every read-only turn. renderDeadCodeHits never reports a symbol listed on a "pending_unknown " line of analyze dead_code as dead. --- internal/hooks/posttask.go | 132 ++++++++++++++++++++++++++++---- internal/hooks/posttask_test.go | 111 +++++++++++++++++++++++++++ internal/hooks/precompact.go | 17 ++++ 3 files changed, 244 insertions(+), 16 deletions(-) diff --git a/internal/hooks/posttask.go b/internal/hooks/posttask.go index f8494cb04..c3ab29170 100644 --- a/internal/hooks/posttask.go +++ b/internal/hooks/posttask.go @@ -133,25 +133,106 @@ type changeSet struct { Scope string `json:"scope"` Repo string `json:"repo"` RepoRoot string `json:"repo_root"` + // The label of an answer whose relations lag: graph_freshness on a + // primary checkout, graph_status on a worktree. + GraphFreshness string `json:"graph_freshness"` + GraphStatus string `json:"graph_status"` + PendingFileCount int `json:"pending_file_count"` } +// detectChangesBudget bounds the detect_changes round trip of a briefing. A +// daemon still publishing edits answers inside it with a labelled partial; a +// stalled one is reported rather than waited on. +const detectChangesBudget = 300 * time.Millisecond + +// detectChangesWait is how far into the budget the briefing's wait_deadline +// lets the daemon wait for pending edits before it labels them. The rest of +// the budget is for the diff walk and the transport. +const detectChangesWait = detectChangesBudget / 2 + // detectChangeSet asks the bridge for the current diff and decodes it. -// Returns nil when the bridge is unreachable or the payload is unusable. -func detectChangeSet(port int) *changeSet { - raw := callServerTool(port, "detect_changes", map[string]any{ +// answered is false when the bridge did not reply within detectChangesBudget, +// or replied with nothing once detectChangesWait had passed; changes is nil +// when it is unreachable or the payload is unusable. +func detectChangeSet(port int) (changes *changeSet, answered bool) { + started := time.Now() + raw, answered := callServerToolWithin(port, "detect_changes", map[string]any{ "scope": postTaskDiffScope, // The briefings never read by_depth; summary_only drops the // largest part of the response. - "summary_only": true, - }) + "summary_only": true, + "wait_deadline": started.Add(detectChangesWait).UTC().Format(time.RFC3339Nano), + }, detectChangesBudget) if raw == "" { - return nil + // The daemon refuses a wait_deadline that passed before it took + // the call up: a busy daemon, not an unreachable one. + return nil, answered && time.Since(started) < detectChangesWait } - var changes changeSet if err := json.Unmarshal([]byte(raw), &changes); err != nil { - return nil + return nil, true } - return &changes + return changes, true +} + +// relationsLine is the label line of a change set whose relations lag, or "" +// when the graph has published every change it reports. +func (c *changeSet) relationsLine() string { + state := c.GraphFreshness + if state == "" && c.GraphStatus != "" && c.GraphStatus != "ready" { + state = c.GraphStatus + } + if state == "" { + return "" + } + files := c.PendingFileCount + if files == 0 { + files = len(c.ChangedFiles) + } + return fmt.Sprintf("relations: %s (%d files)", state, files) +} + +// laggingChangesBriefing is the briefing for a change set the graph has not +// caught up with: the relations line and the changed files, with the +// symbol-level diagnostics left to a turn that can trust them. A stalled +// detect_changes (nil changes) is reported the same way instead of as +// silence, except to a session that recorded no writes: it has no change set +// to warn about. lagging is false when the change set is complete. +func laggingChangesBriefing(heading string, scope sessionScope, changes *changeSet) (briefing string, lagging bool) { + line := fmt.Sprintf("relations: pending (detect_changes did not answer within %v)", detectChangesBudget) + why := "" + var files []string + if changes == nil { + if scope.SessionID != "" && len(loadSessionState(scope.SessionID).WrittenPaths) == 0 { + return "", true + } + } else { + if line = changes.relationsLine(); line == "" { + return "", false + } + why = " the graph is still publishing edits, so" + files = changes.ChangedFiles + if attr := attributeChanges(scope, changes); attr.Attributed { + if len(attr.OwnedFiles) == 0 { + return "", true + } + files = attr.OwnedFiles + } + } + var sb strings.Builder + sb.WriteString(heading + "\n\n") + sb.WriteString(line + " —" + why + " symbol-level diagnostics are skipped; run detect_changes again before handoff.\n") + if len(files) > 0 { + sb.WriteString("\nChanged files:\n") + const show = 10 + for i, f := range files { + if i == show { + fmt.Fprintf(&sb, "- (+%d more)\n", len(files)-show) + break + } + fmt.Fprintf(&sb, "- `%s`\n", f) + } + } + return sb.String(), true } // briefingScope names the working tree detect_changes actually diffed. @@ -245,10 +326,13 @@ func renderUnattributedFiles(files []string, capped bool) string { // named rather than dropped. When the session cannot be identified at all the // briefing falls back to the whole tree and labels itself as such. func buildPostTaskBriefing(port int, scope sessionScope) string { - changes := detectChangeSet(port) - if changes == nil { + changes, answered := detectChangeSet(port) + if answered && changes == nil { return "" } + if briefing, lagging := laggingChangesBriefing("## Gortex Working-Tree Diagnostics", scope, changes); lagging { + return briefing + } if len(changes.ChangedSymbols) == 0 { // No indexed symbols touched — skip silently. The agent doesn't need @@ -365,8 +449,14 @@ func buildPostTaskBriefing(port int, scope sessionScope) string { // so nothing is ever recorded for it and it always resolves to the labeled // whole-tree fallback — this does not give Codex attribution coverage. func buildMutationBriefing(port int, scope sessionScope) string { - changes := detectChangeSet(port) - if changes == nil || len(changes.ChangedSymbols) == 0 { + changes, answered := detectChangeSet(port) + if answered && changes == nil { + return "" + } + if briefing, lagging := laggingChangesBriefing("## Gortex mutation follow-up", scope, changes); lagging { + return briefing + } + if len(changes.ChangedSymbols) == 0 { return "" } @@ -461,7 +551,9 @@ func renderGuardViolations(port int, idsCSV string) string { // renderDeadCodeHits filters analyze:dead_code results to the intersection // with the currently-changed symbols (i.e. "did this task leave anything -// orphaned"). Emits nothing when the intersection is empty. +// orphaned"). Emits nothing when the intersection is empty. A symbol listed +// on a "pending_unknown " line has unknown liveness while edits are pending +// and is never reported as dead. func renderDeadCodeHits(port int, ids []string) string { raw := callServerTool(port, "analyze", map[string]any{ "kind": "dead_code", @@ -472,14 +564,22 @@ func renderDeadCodeHits(port int, ids []string) string { return "" } + lines := strings.Split(raw, "\n") idSet := make(map[string]bool, len(ids)) for _, id := range ids { idSet[id] = true } + for _, line := range lines { + if rest, unknown := strings.CutPrefix(line, "pending_unknown "); unknown { + for _, field := range strings.Fields(rest) { + delete(idSet, field) + } + } + } var hits []string - for _, line := range strings.Split(raw, "\n") { - if line == "" { + for _, line := range lines { + if line == "" || strings.HasPrefix(line, "pending_unknown ") { continue } // Each compact line starts with the symbol descriptor including the diff --git a/internal/hooks/posttask_test.go b/internal/hooks/posttask_test.go index b55f14b90..2dda76944 100644 --- a/internal/hooks/posttask_test.go +++ b/internal/hooks/posttask_test.go @@ -3,9 +3,12 @@ package hooks import ( "encoding/json" "fmt" + "net/http" + "net/http/httptest" "path/filepath" "strings" "testing" + "time" "github.com/zzet/gortex/internal/daemon" "github.com/zzet/gortex/internal/localizationauth" @@ -189,6 +192,114 @@ func TestRunPostTask_DeadCodeFiltersToChanged(t *testing.T) { if strings.Contains(out, "API Contract Issues") { t.Errorf("contract section should be omitted when clean:\n%s", out) } + + // A changed symbol whose liveness is unknown while edits are pending is + // never reported as dead. + pending := newFakeServer(map[string]string{ + "analyze": "relations: pending (1 files)\npending_unknown function foo.go::Foo foo.go:3\n", + }) + defer pending.Close() + if hits := renderDeadCodeHits(portFromURL(t, pending.URL), []string{"foo.go::Foo"}); hits != "" { + t.Errorf("pending_unknown symbol reported as dead:\n%s", hits) + } +} + +// While the graph is still publishing edits, detect_changes answers with a +// labelled partial. Both briefings must say so and name the changed files +// rather than render diagnostics over lagging relations or fall silent. +func TestPostTaskBriefingPrintsPendingRelations(t *testing.T) { + for name, detect := range map[string]string{ + "primary": `{"changed_files":["a.go","b.go"],"changed_symbols":[{"id":"a.go::A","name":"A","kind":"function"}], + "risk":"LOW","complete":false,"graph_freshness":"pending","pending_file_count":2,"pending_files":["a.go","b.go"]}`, + "worktree": `{"changed_files":["a.go"],"changed_symbols":[],"risk":"UNKNOWN","complete":false,"graph_status":"pending"}`, + } { + srv := newRecordingFakeServer(map[string]string{"detect_changes": detect, "get_test_targets": "a_test.go TestA\n"}) + port := portFromURL(t, srv.URL) + started := time.Now() + for builder, briefing := range map[string]string{ + "stop": buildPostTaskBriefing(port, sessionScope{}), + "mutation": buildMutationBriefing(port, sessionScope{}), + } { + want := "relations: pending (2 files)" + if name == "worktree" { + want = "relations: pending (1 files)" + } + if !strings.Contains(briefing, want) || !strings.Contains(briefing, "`a.go`") { + t.Errorf("%s %s briefing missing %q and the changed files:\n%s", name, builder, want, briefing) + } + } + if n := srv.callCount("get_test_targets"); n != 0 { + t.Errorf("%s: diagnostics ran over lagging relations (%d get_test_targets calls)", name, n) + } + // The daemon waits for pending edits until wait_deadline and labels + // the rest. Without one it waits past the budget, and the labelled + // answer arrives after the briefing has given up on it. + for _, args := range srv.argsFor("detect_changes") { + deadline, err := time.Parse(time.RFC3339Nano, fmt.Sprint(args["wait_deadline"])) + if err != nil || !deadline.After(started) || !deadline.Before(started.Add(detectChangesBudget)) { + t.Errorf("%s: detect_changes wait_deadline %v does not end inside the %v budget", name, args["wait_deadline"], detectChangesBudget) + } + } + srv.Close() + } +} + +// A detect_changes that does not answer within its budget is reported, and +// the briefing returns well inside the half-second hook budget. So is one the +// daemon reached only after its wait_deadline had passed, which it refuses. +// A session that recorded no writes is not told about a slow daemon. +func TestPostTaskBriefingBoundedWhenDetectStalls(t *testing.T) { + t.Setenv(hookSessionDirEnvVar, t.TempDir()) + saveSessionState("writer", sessionState{WrittenPaths: []string{"a.go"}}) + release := make(chan struct{}) + stalled := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-time.After(5 * time.Second): + } + http.NotFound(w, r) + })) + defer stalled.Close() + defer close(release) + late := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + Arguments map[string]any `json:"arguments"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if deadline, err := time.Parse(time.RFC3339Nano, fmt.Sprint(req.Arguments["wait_deadline"])); err == nil { + time.Sleep(time.Until(deadline) + 10*time.Millisecond) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"content":[{"type":"text","text":"wait_deadline is not in the future"}],"isError":true}`)) + })) + defer late.Close() + + for name, srv := range map[string]*httptest.Server{"stalled": stalled, "refused late": late} { + port := portFromURL(t, srv.URL) + for builder, build := range map[string]func(int, sessionScope) string{ + "stop": buildPostTaskBriefing, + "mutation": buildMutationBriefing, + } { + for session, report := range map[string]bool{"": true, "writer": true, "reader": false} { + if name != "stalled" && session != "" { + continue + } + started := time.Now() + briefing := build(port, sessionScope{SessionID: session}) + if elapsed := time.Since(started); elapsed > 500*time.Millisecond { + t.Errorf("%s %s briefing took %v", name, builder, elapsed) + } + switch { + case !report && briefing != "": + t.Errorf("%s %s briefing warned session %q, which wrote nothing:\n%q", name, builder, session, briefing) + case report && !strings.Contains(briefing, "relations: pending"): + t.Errorf("%s %s briefing did not report the busy detect_changes to session %q:\n%q", name, builder, session, briefing) + case report && strings.Contains(briefing, "publishing"): + t.Errorf("%s %s briefing claims edits are publishing without an answer:\n%q", name, builder, briefing) + } + } + } + } } // contractsCompact builds a realistic `contracts check --compact` payload: diff --git a/internal/hooks/precompact.go b/internal/hooks/precompact.go index a086c2393..a7cc8ddd4 100644 --- a/internal/hooks/precompact.go +++ b/internal/hooks/precompact.go @@ -232,6 +232,23 @@ func callServerTool(port int, name string, args map[string]any) string { return callServerToolHTTP(port, name, args) } +// callServerToolWithin is callServerTool bounded by timeout. answered is false +// when the call did not return in time: the bridge is reachable but busy, +// which a briefing reports instead of folding it into the silence of an +// unreachable bridge. The abandoned call ends with the hook process. +func callServerToolWithin(port int, name string, args map[string]any, timeout time.Duration) (raw string, answered bool) { + done := make(chan string, 1) + go func() { done <- callServerTool(port, name, args) }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case raw = <-done: + return raw, true + case <-timer.C: + return "", false + } +} + // callServerToolHTTP issues a POST /v1/tools/{name} against the server and // returns the text content from the first response block. Returns empty string // on any error (server unreachable, non-200, missing content). From 1761fd034509311ea18ac1a6ede1e5d793bdeb17 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:08:05 +0100 Subject: [PATCH 21/60] fix(indexer): end the edge drain after a refused node write When the store refused a node batch, the node drain stopped but the edge drain still went on writing: at least one more batch that would be refused the same way, or, if the refusal was transient or specific to that node batch, the full edge set of a build that had already failed. The failed build publishes nothing and the retry evicts what it wrote, so those writes were wasted I/O against a store that had just reported a storage failure, often a full volume. A failed node write now ends the whole drain. A symbol FTS failure still lets the graph rows drain, as before. --- internal/indexer/first_index_bulk_test.go | 1 + internal/indexer/indexer.go | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 475eb3818..04b8c4fbc 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -479,6 +479,7 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { var storageErr *store_sqlite.StorageError require.ErrorAs(t, err, &storageErr) assert.Contains(t, store_sqlite.SafeStorageFailureReason(err), "SQLite code 13") + assert.Equal(t, 1, batches, "a refused write must be the drain's last") case "seal error": _, err := idx.IndexCtx(ctx, dir) require.ErrorContains(t, err, "nodes_by_kind") diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index ae6e744be..818189951 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3519,6 +3519,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index // The builtin sentinels the drain is about to move, kept aside for // the re-assert below. See restampedBuiltins. var restampedBuiltins []*graph.Node + nodeDrainStopped := false for nodes := range inMemShadow.DrainNodeBatches(persistChunkRows, persistChunkBytes) { for _, node := range nodes { if node != nil && graph.IsBuiltinStub(node.ID) { @@ -3528,10 +3529,12 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index } if err := drainAddBatch(ctx, diskTarget, nodes, nil); err != nil { // A refused write fails every later batch the same way, and - // a cooperative stop is final; either ends the node drain. + // a cooperative stop is final; either ends the drain, so the + // edge drain below writes nothing for this failed build. if retErr == nil { retErr = drainBatchError(ctx, err) } + nodeDrainStopped = true break } if retErr != nil && drainChecksCancellation(ctx) && ctx.Err() != nil { @@ -3586,6 +3589,9 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index drainPressure.afterNodeBatch(nodeRows) } for edges := range inMemShadow.DrainEdgeBatches(persistChunkRows, persistChunkBytes) { + if nodeDrainStopped { + break + } if drainChecksCancellation(ctx) && ctx.Err() != nil { if retErr == nil { retErr = fmt.Errorf("indexer: cooperative drain stopped: %w", ctx.Err()) From 549b0bbee748e080ef1f34d0f1e1652fa2c5537f Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:08:39 +0100 Subject: [PATCH 22/60] mcp: label detect_changes for pending edits to the changed symbols' callers detect_changes scoped its wait and label to the diff's changed files, but its risk and blast radius come from walking the changed symbols' callers. With scope staged or compare the working tree is not in the diff, so a pending edit in a caller file left the walk reading the old relations without a label. After the symbol join it now also waits for, and labels, the relation scope of the changed symbols: their files, the files with edges into them and the files that name them. That wait ends at the same deadline as the changed files' wait, so the answer still waits at most once. --- internal/mcp/mutation_freshness_test.go | 17 +++++++++++++++++ internal/mcp/relation_freshness.go | 23 +++++++++++++++++++++++ internal/mcp/tools_analysis.go | 19 ++++++++++++++++--- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/internal/mcp/mutation_freshness_test.go b/internal/mcp/mutation_freshness_test.go index 230398750..011c5e5fa 100644 --- a/internal/mcp/mutation_freshness_test.go +++ b/internal/mcp/mutation_freshness_test.go @@ -546,4 +546,21 @@ func TestDetectChangesLabelsPendingChangedFile(t *testing.T) { t.Fatalf("detect with wait_deadline %v: graph_freshness %v after %v ms", offset, payload["graph_freshness"], waited) } } + + // A staged diff leaves the pending caller.go out of the changed files, + // but the walk still reads it as a caller of the changed Target. + writeUnindexed(t, root, "target.go", "package p\n\nfunc Target() { _ = 0 }\n") + checkoutMutationGit(t, root, "add", "target.go") + req.Params.Arguments = map[string]any{"format": "json", "scope": "staged"} + result, err = s.handleDetectChanges(context.Background(), req) + if err != nil { + t.Fatalf("detect handler error: %v", err) + } + payload = lifecycleResultPayload(t, result) + if files, _ := payload["changed_files"].([]any); len(files) != 1 || files[0] != "target.go" { + t.Fatalf("staged detect changed_files = %v", payload["changed_files"]) + } + if pending, _ := payload["pending_files"].([]any); payload["graph_freshness"] != "pending" || len(pending) != 1 || pending[0] != "caller.go" { + t.Fatalf("staged detect left the pending caller unlabelled: %v", payload) + } } diff --git a/internal/mcp/relation_freshness.go b/internal/mcp/relation_freshness.go index 024c687f0..1b489e247 100644 --- a/internal/mcp/relation_freshness.go +++ b/internal/mcp/relation_freshness.go @@ -230,6 +230,29 @@ func (f relationFreshness) requireFreshError() *mcp.CallToolResult { return mcp.NewToolResultError(message) } +// merge folds the label of another scope the same answer read into f: the +// files of both, failed over pending, and the time spent waiting on each. +func (f relationFreshness) merge(other relationFreshness) relationFreshness { + f.waited += other.waited + if other.state == "" { + return f + } + if f.state != "failed" { + f.state = other.state + } + seen := make(map[string]bool, len(f.files)) + for _, file := range f.files { + seen[file] = true + } + for _, file := range other.files { + if !seen[file] { + f.files = append(f.files, file) + } + } + sort.Strings(f.files) + return f +} + // decorate makes a rendered text answer lead with its label: TOON and plain // text start with the relations line; GCX keeps GCX1 as line 1, carries the // label as header meta and the relations line as a comment on line 2. JSON diff --git a/internal/mcp/tools_analysis.go b/internal/mcp/tools_analysis.go index 942467169..bec7687a4 100644 --- a/internal/mcp/tools_analysis.go +++ b/internal/mcp/tools_analysis.go @@ -357,6 +357,9 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques var freshnessErr error var label relationFreshness var diff *analysis.DiffResult + var freshness requestFreshness + var keys []string + var deadline time.Time if checkoutScoped { pending, freshnessErr = s.mutationFreshnessSummaryForRepos(ctx, repoPrefix) } else { @@ -367,12 +370,13 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques if err != nil { return mcp.NewToolResultError(err.Error()), nil } - keys := make([]string, len(files.ChangedFiles)) + keys = make([]string, len(files.ChangedFiles)) for i, file := range files.ChangedFiles { keys[i] = analysis.GraphKey(repoPrefix, file, analysis.RepoRelativePath) } - freshness := takeRequestFreshness(&req) - label = s.awaitRelationFreshness(ctx, nil, keys, relationWaitDeadline(ctx, freshness)) + freshness = takeRequestFreshness(&req) + deadline = relationWaitDeadline(ctx, freshness) + label = s.awaitRelationFreshness(ctx, nil, keys, deadline) if freshness.requireFresh && label.state != "" { return label.requireFreshError(), nil } @@ -464,6 +468,15 @@ func (s *Server) handleDetectChanges(ctx context.Context, req mcp.CallToolReques for i, cs := range diff.ChangedSymbols { symbolIDs[i] = cs.ID } + if !checkoutScoped { + // The walk reads the changed symbols' callers, and an edit outside + // the diff (a staged or compare scope leaves the working tree out) + // can be changing those. They share what is left of the budget. + label = label.merge(s.awaitRelationFreshness(ctx, symbolIDs, keys, deadline)) + if freshness.requireFresh && label.state != "" { + return label.requireFreshError(), nil + } + } impact, err := s.analyzeDiffImpact(ctx, symbolIDs) if err != nil { From 46da202a562d1dc28077b2111f554dfdaa0c1402 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:08:53 +0100 Subject: [PATCH 23/60] fix(indexer): retry the deferred bulk flush when it panics too The first-index backstop retried a FlushBulk that panicked only when it was the explicit flush after a completed drain. When a drain write panicked instead, the deferred FlushBulk ran while that panic unwound, and if its seal panicked as well, nothing retried it: the standalone window stayed pinned at synchronous=OFF until Close. The deferred flush now marks itself in progress the same way, so the earlier-registered defer gives it the same single retry while the later panic unwinds. No recover is involved, so that panic keeps its stack. --- internal/indexer/first_index_bulk_test.go | 11 ++++++--- internal/indexer/indexer.go | 29 +++++++++++++---------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 04b8c4fbc..3b47dff86 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -405,7 +405,7 @@ BEGIN SELECT RAISE(ABORT, 'injected drain FTS error'); END`) func TestFirstIndexFailureRestoresStore(t *testing.T) { for _, mode := range []string{ "drain error", "cancel before drain", "panic in body", "panic in drain", - "storage full", "seal panic", "seal error", + "storage full", "seal panic", "seal panic after a drain panic", "seal error", } { t.Run(mode, func(t *testing.T) { store, db := newFirstIndexStore(t) @@ -428,7 +428,7 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { defer store.SetBulkSealHookForTest(func(index string) error { seals++ switch { - case mode == "seal panic" && seals == 1: + case (mode == "seal panic" || mode == "seal panic after a drain panic") && seals == 1: panic("injected seal panic") case mode == "seal error" && index == "nodes_by_kind": return errors.New("injected seal failure") @@ -440,7 +440,7 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { switch mode { case "drain error": firstIndexDrainFTSTrigger(t, store) - case "panic in drain": + case "panic in drain", "seal panic after a drain panic": fault = func() error { panic("injected drain panic") } case "storage full": full := sqliteFullError(t) @@ -468,6 +468,11 @@ func TestFirstIndexFailureRestoresStore(t *testing.T) { assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") assert.Positive(t, idx.indexMemoryAdmission.snapshot().peak, "fixture must acquire the memory lease") assert.Zero(t, idx.indexMemoryAdmission.snapshot().used, "earlier defers must release the memory lease") + case "seal panic after a drain panic": + // The flush the drain panic's unwind runs panics too; it is + // retried the same way, and the later panic reaches the caller. + require.PanicsWithValue(t, "injected seal panic", func() { _, _ = idx.IndexCtx(ctx, dir) }) + assert.Greater(t, seals, 1, "the backstop did not retry the panicked flush") case "cancel before drain": _, err := idx.IndexCtx(ctx, dir) require.ErrorIs(t, err, context.Canceled) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 818189951..b14bf161c 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3471,21 +3471,26 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index bulkFlushPending := true bulkFlushInProgress := false defer func() { - if !bulkFlushPending { - return + // A FlushBulk panicked: the explicit one below, or the one the + // next defer runs while a drain panic unwinds. The store releases + // its write gate on every exit and keeps a window whose seal did + // not finish retryable, so one more flush closes it (restoring + // durability, or recording the indexes it could not rebuild) + // instead of leaving a pinned synchronous=OFF writer behind. + if bulkFlushPending && bulkFlushInProgress { + retryBulkFlushAfterPanic(bl, idx.logger, idx.RepoPrefix()) } - if !bulkFlushInProgress { - if err := bl.FlushBulk(); err != nil && retErr == nil { - retErr = fmt.Errorf("indexer: persist bulk graph: %w", err) - } + }() + defer func() { + if !bulkFlushPending || bulkFlushInProgress { return } - // The explicit FlushBulk below panicked. The store releases its - // write gate on every exit and keeps a window whose seal did not - // finish retryable, so one more flush closes it (restoring - // durability, or recording the indexes it could not rebuild) - // instead of leaving a pinned synchronous=OFF writer behind. - retryBulkFlushAfterPanic(bl, idx.logger, idx.RepoPrefix()) + bulkFlushInProgress = true + err := bl.FlushBulk() + bulkFlushPending = false + if err != nil && retErr == nil { + retErr = fmt.Errorf("indexer: persist bulk graph: %w", err) + } }() // Transfer the shadow in explicit row/byte-capped batches. The // process-wide shadow admission caps all live shadows at 1 GiB; these From ae55e629f7d42bc0f3fb4ea5aaeffabf8b3821fc Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:09:30 +0100 Subject: [PATCH 24/60] test(indexer): check that a first-index drain seals its window once When TestFirstIndexDrainUsesBulkWindow moved to the real store, it lost the probe assertions that the drain opened and flushed exactly one window. The witness only counted batches written inside an open window, so a drain that opened a second window would still pass. The witness now also counts seals per dense index through the store's seal hook and requires each index to be sealed exactly once per IndexCtx, on every route the test covers, the retry included. --- internal/indexer/first_index_bulk_test.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 3b47dff86..61b4f6fd6 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -203,17 +203,23 @@ func TestDrainAddBatchReturnsStorageFailure(t *testing.T) { } } -// firstIndexWindowWitness counts a store's batch writes, and how many of them -// landed inside an engaged bulk window with the dense indexes deferred. fail, -// when set, decides whether the store refuses a batch. +// firstIndexWindowWitness counts a store's batch writes, how many of them +// landed inside an engaged bulk window with the dense indexes deferred, and how +// often each dense index was sealed. fail, when set, decides whether the store +// refuses a batch. type firstIndexWindowWitness struct { batches, engaged, deferred int + seals map[string]int fail func(edgeRows int) error } func watchFirstIndexWindow(t *testing.T, store *store_sqlite.Store, db *sql.DB, indexes []string) *firstIndexWindowWitness { t.Helper() - w := &firstIndexWindowWitness{} + w := &firstIndexWindowWitness{seals: map[string]int{}} + t.Cleanup(store.SetBulkSealHookForTest(func(index string) error { + w.seals[index]++ + return nil + })) t.Cleanup(store.SetBatchWriteHookForTest(func(_, edgeRows int) error { w.batches++ if firstIndexBulkActive(t, store) { @@ -242,6 +248,11 @@ func (w *firstIndexWindowWitness) assertEngaged(t *testing.T) { require.Positive(t, w.batches, "test must observe the shadow drain") assert.Positive(t, w.engaged, "cold shadow drain must engage standalone bulk mode") assert.Positive(t, w.deferred, "dense indexes must be deferred during drain") + // One window per drain: a second one would seal every index again. + require.NotEmpty(t, w.seals, "the window must seal the indexes it deferred") + for index, n := range w.seals { + assert.Equal(t, 1, n, "index %s sealed %d times", index, n) + } } func assertFirstIndexReceipts(t *testing.T, store *store_sqlite.Store, result *IndexResult) { @@ -318,7 +329,7 @@ func TestFirstIndexDrainUsesBulkWindow(t *testing.T) { assertFirstIndexRestored(t, store, db, indexes) // The retry evicts them, which leaves the store cold again. - w.batches, w.engaged, w.deferred = 0, 0, 0 + w.batches, w.engaged, w.deferred, w.seals = 0, 0, 0, map[string]int{} result, err := newFirstIndexBulkIndexer(t, store, "repo").IndexCtx(context.Background(), dir) require.NoError(t, err) w.assertEngaged(t) From c60eb826ea092c1c0db7e207748c33763f8005bb Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 17:29:40 +0100 Subject: [PATCH 25/60] mcp: label relation-reading mutators and block deletes on pending callers change_contract, preview_edit and safe_delete_symbol read callers and references from the graph and act on them or give a verdict, with no word about edits the graph has not published yet. Once edit acknowledgements stop waiting for the reindex, those windows become routine and the tools would answer from the old graph without saying so; safe_delete could delete a symbol whose only caller was added a moment ago in another file. Each tool now snapshots the unpublished edits before it walks, and once the walk has named the symbols involved labels the answer with the in-scope ones (graph_freshness, freshness_scope, pending_files, pending_file_count), leading TOON and plain text with the relations line. There is no default wait. safe_delete_symbol also scans the files of those edits for lines that name the target as a whole identifier and reports each one as a provisional reference, so the delete is rejected through the existing rejected_has_references status, and in propagate mode as a manual patch that blocks the propagation. An edit to the target's own file is one provisional reference for the whole file: it may add a caller there, and the graph's span of the target, which the delete cuts by, may no longer hold. A line the graph already holds a reference for is left to the graph, so it is neither counted twice nor turned into a manual patch. With cascade:"apply" every closure member gets the same scan, since the closure is orphaned by the graph's edges only, and a hit refuses the cascade. The scan stands in for the unresolved call stubs of a pending projection until those exist. preview_edit maps the touched files to graph keys before scoping the label, so a file named by an absolute URI still counts. --- internal/mcp/change_contract.go | 7 +- internal/mcp/relation_freshness.go | 101 ++++++++++++++++++- internal/mcp/relation_freshness_test.go | 127 ++++++++++++++++++++++++ internal/mcp/tools_safe_delete.go | 76 ++++++++++++-- internal/mcp/tools_simulate.go | 17 ++++ 5 files changed, 315 insertions(+), 13 deletions(-) diff --git a/internal/mcp/change_contract.go b/internal/mcp/change_contract.go index 81d47c99e..325397fba 100644 --- a/internal/mcp/change_contract.go +++ b/internal/mcp/change_contract.go @@ -897,6 +897,9 @@ func (s *Server) handleChangeContract(ctx context.Context, req mcp.CallToolReque if s == nil || s.graph == nil { return mcp.NewToolResultError("change_contract: server not fully initialised"), nil } + // Snapshot the unpublished edits before the walk; the changed symbols + // that scope the label are known only once it has run. + gaps := s.laggingMutationReceipts(ctx) p, err := s.lowerChange(ctx, req) if err != nil { if ctxErr := requestContextError(ctx, err); ctxErr != nil { @@ -909,5 +912,7 @@ func (s *Server) handleChangeContract(ctx context.Context, req mcp.CallToolReque } violations := s.evaluateChange(ctx, p) env := s.assembleEnvelope(ctx, p, violations) - return s.respondJSONOrTOON(ctx, req, env) + label := s.labelRelations(ctx, gaps, p.changedIDs, nil) + res, err := s.respondJSONOrTOON(ctx, req, label.labelled(env)) + return label.decorate(res), err } diff --git a/internal/mcp/relation_freshness.go b/internal/mcp/relation_freshness.go index 1b489e247..b586cf68b 100644 --- a/internal/mcp/relation_freshness.go +++ b/internal/mcp/relation_freshness.go @@ -3,6 +3,7 @@ package mcp import ( "bytes" "context" + "encoding/json" "fmt" "os" "sort" @@ -80,20 +81,41 @@ func (s *Server) awaitRelationFreshness(ctx context.Context, ids, files []string } lagging = s.laggingMutationReceipts(ctx) } + label := s.relationLabel(lagging, inScope) + label.waited = time.Since(started) + return label +} +// labelRelations labels an answer that read relations without waiting. gaps +// is the laggingMutationReceipts snapshot taken before the walk; the scope of +// ids and files, known only once the walk has run, filters it afterwards. An +// edit that published during the walk still labels the answer, which may +// have read around it. +func (s *Server) labelRelations(ctx context.Context, gaps []*mutationReceipt, ids, files []string) relationFreshness { + if len(gaps) == 0 { + return relationFreshness{} + } + return s.relationLabel(gaps, s.relationScope(ctx, ids, files)) +} + +// relationLabel labels the receipts that fall in scope: failed when the edit +// failed to publish, pending otherwise. One pending receipt for a path +// outweighs an older failed one, and any failed path makes the label failed. +func (s *Server) relationLabel(receipts []*mutationReceipt, inScope func(absPath string) bool) relationFreshness { states := make(map[string]string) - for _, receipt := range lagging { + for _, receipt := range receipts { if !inScope(receipt.path) { continue } key := s.repoRelative(receipt.path) - if receipt.outcome(true).Pending { + outcome := receipt.outcome(true) + if outcome.Pending || (outcome.Err == nil && outcome.Reindexed) { states[key] = "pending" } else if states[key] == "" { states[key] = "failed" } } - label := relationFreshness{waited: time.Since(started)} + var label relationFreshness for path, state := range states { label.files = append(label.files, path) if label.state != "failed" { @@ -214,6 +236,25 @@ func (f relationFreshness) apply(result map[string]any) { } } +// labelled is payload with the label's fields merged in, a struct payload +// being re-read as its JSON object first. A fresh scope returns payload +// unchanged. +func (f relationFreshness) labelled(payload any) any { + if f.state == "" { + return payload + } + data, err := json.Marshal(payload) + if err != nil { + return payload + } + var fields map[string]any + if json.Unmarshal(data, &fields) != nil || fields == nil { + return payload + } + f.apply(fields) + return fields +} + // line is the relations line a text answer leads with. func (f relationFreshness) line() string { return fmt.Sprintf("relations: %s (%d files)", f.state, len(f.files)) @@ -279,3 +320,57 @@ func (f relationFreshness) decorate(res *mcp.CallToolResult) *mcp.CallToolResult } return rebuildTextResult(res, body) } + +// provisionalSite is a line of a file whose edit the graph has not published +// that names a symbol as a whole identifier: a reference the graph cannot show +// yet, reported so a mutator neither misses it nor rewrites it. Line is 0 for +// the symbol's own file, which is provisional as a whole. +type provisionalSite struct { + File string `json:"file"` + Line int `json:"line"` + Text string `json:"text"` +} + +// provisionalSites scans the files of gaps for lines naming node, leaving out +// the lines where g already holds a reference to it. When the node's own file +// is among them it is one site without a line: until that edit publishes, the +// graph's span of the node may not hold, and a line that names the node there +// may be a new caller or the definition itself. +func (s *Server) provisionalSites(g graph.Reader, gaps []*mutationReceipt, node *graph.Node) []provisionalSite { + names := []string{identifierName(node.Name)} + known := make(map[string]bool) + for _, e := range g.GetInEdges(node.ID) { + if caller := g.GetNode(e.From); caller != nil && isReferencingEdgeKind(e.Kind) { + known[fmt.Sprintf("%s:%d", caller.FilePath, e.Line)] = true + } + } + seen := make(map[string]bool) + var sites []provisionalSite + for _, receipt := range gaps { + file := s.repoRelative(receipt.path) + if seen[file] { + continue + } + seen[file] = true + if file == node.FilePath { + sites = append(sites, provisionalSite{File: file}) + continue + } + data, err := os.ReadFile(receipt.path) + if err != nil || !namesAnyIdentifier(data, names) { + continue + } + for i, line := range strings.Split(string(data), "\n") { + if namesAnyIdentifier([]byte(line), names) && !known[fmt.Sprintf("%s:%d", file, i+1)] { + sites = append(sites, provisionalSite{File: file, Line: i + 1, Text: strings.TrimSpace(line)}) + } + } + } + sort.Slice(sites, func(i, j int) bool { + if sites[i].File != sites[j].File { + return sites[i].File < sites[j].File + } + return sites[i].Line < sites[j].Line + }) + return sites +} diff --git a/internal/mcp/relation_freshness_test.go b/internal/mcp/relation_freshness_test.go index 4eef99956..1414642c2 100644 --- a/internal/mcp/relation_freshness_test.go +++ b/internal/mcp/relation_freshness_test.go @@ -192,3 +192,130 @@ func TestChangeAnswersStartWithRelationsLine(t *testing.T) { map[string]any{"format": "toon"})) require.Equal(t, line, strings.SplitN(detect, "\n", 2)[0], detect) } + +func nodeID(t *testing.T, s *Server, name string) string { + t.Helper() + for _, n := range s.graph.AllNodes() { + if n.Name == name && n.Kind == graph.KindFunction { + return n.ID + } + } + t.Fatalf("%s is not indexed", name) + return "" +} + +// A call to a symbol that an edit added and the graph has not read yet must +// block safe_delete like a graph reference, in plain and propagate mode. An +// edit to the symbol's own file blocks it as a whole: it may add a caller +// there, and the graph's span of the symbol may no longer hold. +func TestSafeDeleteRejectsTargetCalledFromPendingFile(t *testing.T) { + for name, edit := range map[string]struct { + file, src string + ref map[string]any + }{ + "caller file": {"namer.go", "package p\n\nfunc Namer() {\n\tOther()\n}\n", + map[string]any{"kind": "provisional", "file_path": "namer.go", "line": float64(4)}}, + "definition file": {"other.go", "package p\n\nfunc Extra() {\n\tOther()\n}\n\nfunc Other() {}\n", + map[string]any{"kind": "provisional", "file_path": "other.go"}}, + } { + t.Run(name, func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-"+edit.file, "", writeUnindexed(t, root, edit.file, edit.src), 1) + otherPath := filepath.Join(root, "other.go") + before, err := os.ReadFile(otherPath) + require.NoError(t, err) + id := nodeID(t, s, "Other") + + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": id, "dry_run": false, "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"]) + require.Equal(t, []any{edit.ref}, payload["references"]) + require.Equal(t, "pending", payload["graph_freshness"]) + + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": id, "propagate": true, "dry_run": false, "format": "json"})) + require.Equal(t, "propagation_blocked_manual", payload["status"]) + require.Equal(t, float64(1), payload["manual"]) + + after, err := os.ReadFile(otherPath) + require.NoError(t, err) + require.Equal(t, string(before), string(after), "the target was deleted with a pending caller") + }) + } + + // A call the graph already holds is listed once, as the graph's + // reference, when its file has an unrelated pending edit; propagate can + // still remove it. + t.Run("graph reference", func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-caller", "", + writeUnindexed(t, root, "caller.go", "package p\n\nfunc Caller() {\n\tTarget()\n}\n\nfunc More() {}\n"), 1) + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": targetID(t, s), "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"]) + require.Equal(t, float64(1), payload["reference_count"], "%v", payload["references"]) + payload = lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": targetID(t, s), "propagate": true, "format": "json"})) + require.Equal(t, "propagation_plan", payload["status"]) + require.Equal(t, float64(0), payload["manual"], "%v", payload["caller_patches"]) + }) + + // Target is orphaned with Caller by the graph's edges, but a pending edit + // now calls it: the cascade must not delete it with Caller. + t.Run("cascade member", func(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + sources := map[string]string{} + for _, name := range []string{"caller.go", "target.go"} { + data, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + sources[name] = string(data) + } + payload := lifecycleResultPayload(t, callChangeTool(t, s.handleSafeDeleteSymbol, "safe_delete_symbol", + map[string]any{"id": nodeID(t, s, "Caller"), "cascade": "apply", "dry_run": false, "format": "json"})) + require.Equal(t, "rejected_has_references", payload["status"], "%v", payload) + require.Equal(t, []any{map[string]any{"kind": "provisional", "file_path": "namer.go", "line": float64(4)}}, payload["references"]) + for name, src := range sources { + data, err := os.ReadFile(filepath.Join(root, name)) + require.NoError(t, err) + require.Equal(t, src, string(data), "%s changed although the cascade was refused", name) + } + }) +} + +// The relation-reading mutators answer at once and label the answer while an +// edit in their scope never publishes; they never wait it out or refuse. +func TestRelationToolsLabelAndNeverRefuseWithUnresolvedReceipt(t *testing.T) { + s, root := newRelationFixture(t) + pendingFreshnessReceipt(s, "receipt-namer", "", + writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) + id := targetID(t, s) + rename := `{"changes":{"target.go":[{"range":{"start":{"line":2,"character":5},"end":{"line":2,"character":11}},"newText":"Renamed"}]}}` + comment := `{"changes":{"file://` + filepath.ToSlash(filepath.Join(root, "namer.go")) + + `":[{"range":{"start":{"line":0,"character":0},"end":{"line":0,"character":0}},"newText":"// Namer names Target.\n"}]}}` + + for tool, call := range map[string]func() *mcpgo.CallToolResult{ + "change_contract": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handleChangeContract, "change_contract", + map[string]any{"source": "symbols", "symbols": id, "format": "json"}) + }, + "preview_edit": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handlePreviewEdit, "preview_edit", + map[string]any{"workspace_edit": rename, "diagnostics": false}) + }, + // No symbol is removed or renamed: only the touched file, named by + // an absolute file URI, brings the pending edit into scope. + "preview_edit file uri": func() *mcpgo.CallToolResult { + return callChangeTool(t, s.handlePreviewEdit, "preview_edit", + map[string]any{"workspace_edit": comment, "diagnostics": false}) + }, + } { + started := time.Now() + payload := lifecycleResultPayload(t, call()) + require.Less(t, time.Since(started), 2*time.Second, "%s waited on a receipt that never publishes", tool) + require.Equal(t, "pending", payload["graph_freshness"], "%s: %v", tool, payload) + require.Equal(t, []any{"namer.go"}, payload["pending_files"], tool) + require.Equal(t, false, payload["complete"], tool) + } +} diff --git a/internal/mcp/tools_safe_delete.go b/internal/mcp/tools_safe_delete.go index 17b194870..08ac47ad4 100644 --- a/internal/mcp/tools_safe_delete.go +++ b/internal/mcp/tools_safe_delete.go @@ -52,10 +52,11 @@ func (s *Server) registerSafeDeleteSymbolTool() { // information for the caller to navigate to it and remove it before // retrying the delete. type safeDeleteReference struct { - FromID string `json:"from_id"` + FromID string `json:"from_id,omitempty"` Kind string `json:"kind"` FromName string `json:"from_name,omitempty"` FilePath string `json:"file_path,omitempty"` + Line int `json:"line,omitempty"` } // cascadeClosureEntry describes a symbol that the cascade pass added @@ -109,6 +110,20 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq // EdgeDefines / EdgeMemberOf are skipped (they don't represent // "someone calls this"). refs := collectReferencingEdges(g, id) + // A call that an edit added and the graph has not read yet blocks the + // delete like a graph reference does: a provisional reference here, a + // manual patch in propagate mode. The same snapshot labels the answer. + gaps := s.laggingMutationReceipts(ctx) + provisional := s.provisionalSites(g, gaps, node) + for _, site := range provisional { + refs = append(refs, safeDeleteReference{Kind: "provisional", FilePath: site.File, Line: site.Line}) + } + respond := func(result map[string]any) (*mcp.CallToolResult, error) { + label := s.labelRelations(ctx, gaps, []string{id}, nil) + label.apply(result) + res, err := s.respondJSONOrTOON(ctx, req, result) + return label.decorate(res), err + } // SRN-7: propagate-delete — patch the surviving call sites instead of // refusing. Builds a per-caller plan, applies the removable standalone @@ -116,9 +131,18 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq var appliedPatches, failedPatches []callerPatch if req.GetBool("propagate", false) && len(refs) > 0 { plan := s.buildPropagationPlan(ctx, g, node) + for _, site := range provisional { + reason := "provisional: named in a file whose edit the graph has not read yet — patch by hand" + if site.Line == 0 { + reason = "provisional: the graph has not read the latest edit to the symbol's own file — retry once it publishes" + } + plan = append(plan, callerPatch{ + File: site.File, Line: site.Line, CurrentText: site.Text, Action: "manual", Reason: reason, + }) + } manual := countManual(plan) if dryRun { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_plan", "symbol": id, "file": node.FilePath, @@ -131,7 +155,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq }) } if manual > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_blocked_manual", "symbol": id, "caller_patches": plan, @@ -145,7 +169,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq } appliedPatches, failedPatches = applied, failed if len(failed) > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "propagation_partial", "symbol": id, "patches_applied": applied, @@ -158,7 +182,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq force = true node = g.GetNode(id) if node == nil { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "deleted_by_propagation", "symbol": id, "patches_applied": appliedPatches, @@ -171,7 +195,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq } if len(refs) > 0 && !force { - return s.respondJSONOrTOON(ctx, req, map[string]any{ + return respond(map[string]any{ "status": "rejected_has_references", "symbol": id, "file": node.FilePath, @@ -195,6 +219,40 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if cascadeMode != cascadeModeOff { closure, cascadeTruncated = computeCascadeClosure(g, node, cascadeIntoTests) } + if cascadeMode == cascadeModeApply { + // The closure is orphaned by the graph's edges only. A call that an + // unpublished edit added keeps a member alive, and force does not + // cover the members, so the cascade waits for that edit. + var blocked []safeDeleteReference + var members []string + for _, entry := range closure { + member := g.GetNode(entry.ID) + if member == nil { + continue + } + sites := s.provisionalSites(g, gaps, member) + if len(sites) > 0 { + members = append(members, entry.ID) + } + for _, site := range sites { + blocked = append(blocked, safeDeleteReference{Kind: "provisional", FilePath: site.File, Line: site.Line}) + } + } + if len(blocked) > 0 { + return respond(map[string]any{ + "status": "rejected_has_references", + "symbol": id, + "file": node.FilePath, + "references": blocked, + "reference_count": len(blocked), + "dry_run": dryRun, + "force": force, + "cascade_mode": cascadeMode, + "cascade_closure": closure, + "hint": "cascade member(s) " + strings.Join(members, ", ") + " are named in files whose edits the graph has not read yet; retry once those publish, or delete without cascade", + }) + } + } absPath, err := s.resolveNodePath(ctx, node) if err != nil { @@ -242,7 +300,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if dryRun { result["status"] = "preview" - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // Commit phase. cascadeModeApply expands the work to every entry @@ -282,7 +340,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq result["status"] = "partial_failure" result["cascade_deleted"] = deletedIDs result["error"] = err.Error() - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // Persist session state and re-index every touched file. @@ -310,7 +368,7 @@ func (s *Server) handleSafeDeleteSymbol(ctx context.Context, req mcp.CallToolReq if len(failedPatches) > 0 { result["patches_failed"] = failedPatches } - return s.respondJSONOrTOON(ctx, req, result) + return respond(result) } // normaliseCascadeMode coerces the cascade parameter to a known diff --git a/internal/mcp/tools_simulate.go b/internal/mcp/tools_simulate.go index fdfadea7c..4156dfb43 100644 --- a/internal/mcp/tools_simulate.go +++ b/internal/mcp/tools_simulate.go @@ -107,6 +107,9 @@ func (s *Server) handlePreviewEdit(ctx context.Context, req mcp.CallToolRequest) wantDiag := req.GetBool("diagnostics", true) diagTimeout := time.Duration(req.GetInt("diagnostics_timeout_ms", 1500)) * time.Millisecond + // Snapshot the unpublished edits before the simulation reads relations; + // the symbols that scope the label are known only once it has run. + gaps := s.laggingMutationReceipts(ctx) sim, simErr := s.buildSimulation(ctx, []lsp.WorkspaceEdit{edit}, inherit) if simErr != nil { if ctxErr := requestContextError(ctx, simErr); ctxErr != nil { @@ -138,6 +141,20 @@ func (s *Server) handlePreviewEdit(ctx context.Context, req mcp.CallToolRequest) result["diagnostics_summary"] = summariseDiagnostics(diags) } + // broken_callers walks the callers of the removed and renamed symbols. + seeds := append([]string{}, step.symbolsRemoved...) + for _, renamed := range step.symbolsRenamed { + seeds = append(seeds, renamed["from"]) + } + // touched_files keeps each path as the client spelled it; the scope + // takes graph keys. + var touched []string + for _, path := range step.touchedFiles { + if abs, err := s.resolveOverlayAbsPath(path); err == nil && abs != "" { + touched = append(touched, s.repoRelative(abs)) + } + } + s.labelRelations(ctx, gaps, seeds, touched).apply(result) return mcp.NewToolResultText(jsonOK(result)), nil } From da4ce7ec1a94c8b9581f2a65cd18666372376917 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:10:06 +0100 Subject: [PATCH 26/60] fix(sqlite): skip a dense-index repair when the store size is unreadable The repair's free-space gate sizes an attempt by the store's used bytes from CompactStats, which reads 0, 0 when any of its PRAGMAs fails. The need then fell to the 64 MiB floor, so a multi-gigabyte store could start rebuilding its dense indexes on a nearly full volume, the case the gate exists to block. An unreadable free-space figure was already a skip; an unreadable store size is now one too, and the backoff retries later. --- .../graph/store_sqlite/dense_index_repair.go | 7 ++++- .../store_sqlite/dense_index_repair_test.go | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/internal/graph/store_sqlite/dense_index_repair.go b/internal/graph/store_sqlite/dense_index_repair.go index 3752c6c83..a8e5a0764 100644 --- a/internal/graph/store_sqlite/dense_index_repair.go +++ b/internal/graph/store_sqlite/dense_index_repair.go @@ -173,9 +173,14 @@ func (s *Store) runDenseIndexRepair(ctx context.Context) { // denseIndexRepairSpace reports the free space on the store's volume, the // amount an attempt needs, and whether the attempt may run. An unreadable // free-space figure is a skip: the gate exists for the full volume, which is -// exactly where a probe may fail too. +// exactly where a probe may fail too. So is an unreadable store size +// (CompactStats reads 0, 0 when its PRAGMAs fail), which would otherwise +// shrink the need to the floor whatever the store's real size. func (s *Store) denseIndexRepairSpace() (avail, need int64, ok bool) { free, total := s.CompactStats() + if total <= 0 { + return -1, -1, false + } need = max(total-free, denseIndexRepairMinFreeBytes) probe := platform.DiskAvailBytes if override := s.denseRepairDiskAvail.Load(); override != nil && *override != nil { diff --git a/internal/graph/store_sqlite/dense_index_repair_test.go b/internal/graph/store_sqlite/dense_index_repair_test.go index ceca4b105..b65bccec6 100644 --- a/internal/graph/store_sqlite/dense_index_repair_test.go +++ b/internal/graph/store_sqlite/dense_index_repair_test.go @@ -2,7 +2,9 @@ package store_sqlite import ( "context" + "database/sql" "errors" + "path/filepath" "reflect" "strings" "sync/atomic" @@ -205,6 +207,30 @@ func TestDenseIndexRepairBuildsFromTheRecordedDDL(t *testing.T) { } } +// TestDenseIndexRepairSkipsWithoutTheStoreSize: an attempt is sized by the +// store's own used bytes, and CompactStats reads 0, 0 when it cannot read +// them. That must skip the attempt, as an unreadable free-space figure does, +// not shrink the need to the floor and let a large store rebuild on a volume +// that is nearly full. +func TestDenseIndexRepairSkipsWithoutTheStoreSize(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + s := &Store{storeCore: &storeCore{db: db, dbPath: filepath.Join(t.TempDir(), "store.sqlite")}} + diskAvail := func(string) (uint64, error) { return 1 << 40, nil } + s.denseRepairDiskAvail.Store(&diskAvail) + if _, total := s.CompactStats(); total != 0 { + t.Fatalf("CompactStats total = %d on a closed pool, want 0", total) + } + if avail, need, ok := s.denseIndexRepairSpace(); ok { + t.Fatalf("attempt allowed with the store size unreadable: avail=%d need=%d", avail, need) + } +} + // TestDenseIndexRepairWaitsForAPinnedWriter: a repair that comes due while a // window pins the writer defers, as VACUUM does, instead of building under // that window, and builds on a later retry once the window has closed. A From a281f42afb9261fc75dad45eff7b40d99e50e0e3 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:10:30 +0100 Subject: [PATCH 27/60] docs(sqlite): say what a seal error leaves behind in flushBulkLocked The comment said a seal panic leaves the window open "exactly as a seal error does". It does not: after a seal error FlushBulk still restores and releases the pinned writer, and records the indexes it could not rebuild for the maintenance-lane repair. Only a coordinated End keeps its window open after a seal error. --- internal/graph/store_sqlite/bulk_load.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index 1d27b7d74..c9fd33b04 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -973,8 +973,10 @@ func (s *Store) FlushBulk() error { } // flushBulkLocked is FlushBulk's write-gate half. A seal that panics leaves the -// pinned writer and its deferred indexes in place, exactly as a seal error -// does, so the next FlushBulk retries the same window. +// pinned writer and its deferred indexes in place, so the next FlushBulk +// retries the same window. A seal that returns an error closes the window +// anyway: the writer is restored and released, and the indexes it could not +// rebuild stay recorded as missing for the maintenance lane's repair. func (s *Store) flushBulkLocked() (bulkWindowExit, error) { s.writeMu.Lock() defer s.writeMu.Unlock() From 342348bef63dce69d19d48d079cfaa4b5d8e75a5 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 19:10:30 +0100 Subject: [PATCH 28/60] docs(mcp): describe the dense_indexes_missing planner_stats verdict index_health's planner_stats can now report the reason dense_indexes_missing together with a dense_indexes_missing list, but the index_health entry only described bulk_window_active. Say what the new verdict means, what it costs, and what clears it. --- docs/mcp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mcp.md b/docs/mcp.md index a4397a928..c2dc65d8a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -479,7 +479,7 @@ Gortex captures every large tool response into a bounded per-session ring; these |------|-------------| | `analyze` | Unified graph analysis dispatcher. `kind` ∈ `dead_code`, `hotspots`, `cycles`, `would_create_cycle`, `connectivity_health`, `todos`, `blame`, `coverage`, `coverage_gaps`, `coverage_summary`, `stale_code`, `stale_flags`, `ownership`, `releases`, `cgo_users`, `wasm_users`, `orphan_tables`, `unreferenced_tables`, `channel_ops`, `goroutine_spawns`, `field_writers`, `race_writes`, `unclosed_channels`, `unsafe_patterns`, `health_score`, `impact`, `annotation_users`, `config_readers`, `env_var_users`, `sql_call_sites`, `fixes_history`, `edge_audit`, `domain`, `named`, `tests_as_edges`, `clusters`, `event_emitters`, `pubsub`, `string_emitters`, `error_surface`, `log_events`, `sql_rebuild`, `external_calls`, `routes`, `models`, `components`, `k8s_resources`, `images`, `kustomize`, `cross_repo`, `dbt_models`, `synthesizers`, `resolution_outcomes`. `clusters` takes an `algorithm` arg (`leiden` / `louvain` / `spectral`). `impact` takes an optional `target` (`{symbol}` or `{file}`): with one it ranks that target's blast radius — the target row first, then its transitive dependents with their `depth` — and reports the closure width plus whether it is exact; without one it ranks the whole repo. An unresolvable target is a structured error, never a silent fall-back to the repo-wide ranking. `synthesizers` rolls up every framework-dispatch-synthesized edge by the pass that produced it; `resolution_outcomes` classifies unresolved call/reference edges by why the resolver gave up (`ambiguous_multi_match` / `candidate_out_of_scope` / `cross_language_only` / `stub_only` / `no_definition`) | | `find_clones` | Near-duplicate function/method clusters from the MinHash + LSH `similar_to` layer; `dead_only: true` finds dead duplicates of live code | -| `index_health` | Health score, parse failures, stale files, language coverage, tracked-repo path liveness (`tracked_repo_paths_ok` + `missing_repo_paths` — a repo whose directory was deleted still holds its registration and silently drops out of workspace-wide answers), per-(repo, provider) semantic-enrichment lifecycle (`semantic_enrichment`: running / completed / partial / abandoned / failed with edge counts, plus a `semantic_enrichment_ok` rollup) — a green file count with a `partial` enrichment state means LSP-tier edges are incomplete. `path_liveness` asks the same question one level down, per file: it stats the paths the graph itself claims and reports how many indexed files no longer exist on disk (`orphan_files` / `orphan_rate` / `orphans_by_repo`, sampled with `truncated: true` past 20k files). `stale_files` only covers files the daemon still tracks, so a deletion it never witnessed shows up here and nowhere else; a non-zero `orphan_files` caps `health_score`. On a SQLite-backed daemon a `planner_stats` object reports the query planner's own view of the store: `nodes` / `edges` carry `believed` (the leading token of the index's `sqlite_stat1` row) against `actual_from_counters` (the `repo_index_state` counter sum across ALL view generations, flagged by `counters_known`; no counter row describes the external-call bucket or nodes carrying no repo prefix, and a leftover empty-prefix row on a store that later became multi-repo is summed on top of the per-repo rows, so the pair is an approximation of index cardinality and a difference between the two is not a measured discrepancy), `receivers` carries a `bounded: true` LIMIT-capped probe of the Go receiver index plus `complete` (false means the probe stopped at its cap, so `actual` is a lower bound and must not be compared with `believed`) and is omitted entirely when the receiver index is not in the schema — a bulk load has dropped it, or this store never had it, and a `believed: 0` there would read as the poisoned near-zero row rather than as an absent index. `stale` says whether the graph has outgrown what the planner believes — which can invert join order on receiver/edge queries — while `reason` names every verdict, staleness or not: `bulk_window_active` is why a cold load in progress reports zeros. `last_refresh_at` / `last_refresh_reason` appear once this process has rebuilt them. The field is read-only: statistics refresh at the next repository index, commit/HEAD move, whole-graph resolve or generation build, never from reading this payload — and that refresh is cooperative, so a busy writer, a spent per-pass budget or a per-index timeout defers the rest of the work to the following boundary (which resumes where the last one stopped) rather than stalling the pipeline that noticed | +| `index_health` | Health score, parse failures, stale files, language coverage, tracked-repo path liveness (`tracked_repo_paths_ok` + `missing_repo_paths` — a repo whose directory was deleted still holds its registration and silently drops out of workspace-wide answers), per-(repo, provider) semantic-enrichment lifecycle (`semantic_enrichment`: running / completed / partial / abandoned / failed with edge counts, plus a `semantic_enrichment_ok` rollup) — a green file count with a `partial` enrichment state means LSP-tier edges are incomplete. `path_liveness` asks the same question one level down, per file: it stats the paths the graph itself claims and reports how many indexed files no longer exist on disk (`orphan_files` / `orphan_rate` / `orphans_by_repo`, sampled with `truncated: true` past 20k files). `stale_files` only covers files the daemon still tracks, so a deletion it never witnessed shows up here and nowhere else; a non-zero `orphan_files` caps `health_score`. On a SQLite-backed daemon a `planner_stats` object reports the query planner's own view of the store: `nodes` / `edges` carry `believed` (the leading token of the index's `sqlite_stat1` row) against `actual_from_counters` (the `repo_index_state` counter sum across ALL view generations, flagged by `counters_known`; no counter row describes the external-call bucket or nodes carrying no repo prefix, and a leftover empty-prefix row on a store that later became multi-repo is summed on top of the per-repo rows, so the pair is an approximation of index cardinality and a difference between the two is not a measured discrepancy), `receivers` carries a `bounded: true` LIMIT-capped probe of the Go receiver index plus `complete` (false means the probe stopped at its cap, so `actual` is a lower bound and must not be compared with `believed`) and is omitted entirely when the receiver index is not in the schema — a bulk load has dropped it, or this store never had it, and a `believed: 0` there would read as the poisoned near-zero row rather than as an absent index. `stale` says whether the graph has outgrown what the planner believes — which can invert join order on receiver/edge queries — while `reason` names every verdict, staleness or not: `bulk_window_active` is why a cold load in progress reports zeros, and `dense_indexes_missing` means a cold load closed without rebuilding some of its dense indexes (typically because the volume filled up during the final index rebuild): `dense_indexes_missing` in the same object names them, lookups that use them scan until they are back, and the store rebuilds them in the background once its volume has free space at least the size of the store (a daemon restart also rebuilds them). `last_refresh_at` / `last_refresh_reason` appear once this process has rebuilt them. The field is read-only: statistics refresh at the next repository index, commit/HEAD move, whole-graph resolve or generation build, never from reading this payload — and that refresh is cooperative, so a busy writer, a spent per-pass budget or a per-index timeout defers the rest of the work to the following boundary (which resumes where the last one stopped) rather than stalling the pipeline that noticed | | `get_symbol_history` | Symbols modified this session with counts; flags churning (3+ edits) | The `analyze` dispatcher also accepts a set of **facade-aliased kinds** that route to the captured legacy handler instead of the dispatcher switch: `processes` → `get_processes`, `communities` → `get_communities`, `contracts` → `contracts`, `architecture` → `get_architecture`, `clones` → `find_clones`, `health` → `audit_health`, `inspections` → `run_inspections`, `recent_changes` → `get_recent_changes`, and the other entries of the facade analyze migration table (see `mcp-facade-v1.md`). These aliases are **surface-independent**: they work for client (facade-v1), explicitly legacy-preset, and session-less HTTP callers alike, with no `tools_search` promotion — the HTTP dashboard endpoints depend on this under the `core`/`defer` default. From 6073fb0b7293a6f3f189373321a2048770e8c7d3 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 21:21:43 +0100 Subject: [PATCH 29/60] test(indexer): list the drain's checked write in the delta checklist The drain now asserts drainCheckedWriter on its target so a storage failure comes back as an error instead of a panic out of the void AddBatch. TestDeltaWriterCapabilityChecklistIsComplete requires every interface the engine asserts to be implemented by *graph.DeltaWriter or listed with a reason, and this one was neither, so the checklist test failed from that change on. The delta does not need it: the checked write belongs to the whole-repository shadow drain into the durable store, not to a file delta's per-save path, and a store without it still takes AddBatch. --- internal/indexer/edit_delta_capability_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index 390e19b6d..48cd965d4 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -211,6 +211,7 @@ var editDeltaAbsentCapabilities = map[string]string{ "indexer.repoSemanticStateReleaser": "semantic (go/types) state: the enrichment stage runs over the generation handle after the delta (runEnrichment), not over the DeltaWriter", "indexer.repoSemanticStateRetainer": "semantic (go/types) state: the enrichment stage runs over the generation handle after the delta (runEnrichment), not over the DeltaWriter", "indexer.repositoryCleanupPayload": "whole-repository or administrative operation (full index, repository reset, untrack or cleanup, warm start, vector corpus, bulk load windows, WAL maintenance); not on a file delta's per-save path", + "indexer.drainCheckedWriter": "checked write of the whole-repository shadow drain into the durable store (full index, shadow replacement); not on a file delta's per-save path, and a store without it takes the void AddBatch", "resolver.backendResolveWorkIndicator": "whole-graph pass (ResolveAll, a cold warm-up, a whole-repository derived rebuild); a delta resolves and re-derives by its receipt frontier", "resolver.batchClaimingResolver": "framework synthesizer contract, asserted on synthesizers", "resolver.claimEdgeVocabulary": "framework synthesizer contract, asserted on synthesizers", From fdd08a445ec963cd61f853bcf1c7624660e98480 Mon Sep 17 00:00:00 2001 From: protereus <63172654+protereus@users.noreply.github.com> Date: Sat, 31 Oct 2026 21:08:54 +0000 Subject: [PATCH 30/60] fix(hooks): answer shell escapes from inline Python and nested shells Follow-up to #1185 covering the bypasses found in its review: - Literal commands in os.system, os.popen, subprocess.getoutput / getstatusoutput, and subprocess.run / call / check_call / check_output / Popen (a string with literal shell=True, or a list of string literals), plus the -c string of a top-level bash/sh/dash/zsh, go through the full Bash policy one level deep. os / subprocess must be plainly imported and never rebound; non-literals, splats, cwd=, executable= and an unfollowable working directory pass through. - os.open with literal os.O_* flags: read-only flags get the Read deny; write flags get the shell-write answer open(p, 'w') already gets. - Inline Python behind exec, nohup, setsid, nice, command, nvm exec and direnv exec is found. skipCommandPrefix and other shell shapes are unchanged. The existing policy runs first and unchanged, with the new analysis branches off; a deny from it returns at once. The new shapes run after, with their own deadline and step and probe budgets, and never replace a deny. Part of #1184. Co-Authored-By: Claude Opus 5.5 --- docs/agents.md | 2 +- internal/hooks/bash_nested.go | 136 ++++++++++ internal/hooks/bash_script_read.go | 306 ++++++++++++++++++---- internal/hooks/enrich_bash_script_test.go | 164 ++++++++++++ internal/hooks/pretooluse.go | 55 +++- internal/hooks/python_script_facts.go | 222 +++++++++++++++- 6 files changed, 823 insertions(+), 62 deletions(-) create mode 100644 internal/hooks/bash_nested.go diff --git a/docs/agents.md b/docs/agents.md index a28c27c44..5abdb023e 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -413,7 +413,7 @@ Current Codex hook coverage: | ------- | -------- | | `SessionStart` | Matches `startup|resume|clear|compact` and emits graph-tools orientation for new, resumed, cleared, and compacted sessions. | | Hook-observable local-tool `PreToolUse` terminal gate | Uses one match-all installer-owned hook so an enforceable localization `answer_ready` contract blocks every local tool Codex routes through `PreToolUse`, including Bash, `apply_patch`, MCP tools, and other local function tools. Hosted tools such as `WebSearch`, and specialized paths that opt out of the default hook path, are outside this host boundary. Without a terminal marker it is a local no-op; advisory completion still permits non-navigation contract operations. | -| Bash `PreToolUse` | Advises by default; `deny` hard-blocks graph-detectable fallback reads/searches; `rewrite` converts only an unambiguous indexed `cat ` into the exact public `gortex call read` mirror. Compound or ambiguous commands remain advisory. A shell command that would *rewrite* indexed source — `sed -i` / `perl -pi`, a `>` / `>>` redirect, `tee`, or an inline interpreter script opening the path for writing (or reading it and writing it back) — gets the same redirect Edit and Write get, escalating to a hard block under `GORTEX_HOOK_BLOCK_EDIT`. Writing a path the daemon does not know is a new file and passes through. Inline Python (`python3 -c`, a `python3 - <` into the exact public `gortex call read` mirror. Compound or ambiguous commands remain advisory. A shell command that would *rewrite* indexed source — `sed -i` / `perl -pi`, a `>` / `>>` redirect, `tee`, or an inline interpreter script opening the path for writing (or reading it and writing it back) — gets the same redirect Edit and Write get, escalating to a hard block under `GORTEX_HOOK_BLOCK_EDIT`. Writing a path the daemon does not know is a new file and passes through. Inline Python (`python3 -c`, a `python3 - < 1 && (w[0] == '-' || w[0] == '+'): + for _, f := range w[1:] { + switch f { + case 'c': + hasC = hasC || w[0] == '-' + case 'o', 'O': + i++ // -o pipefail, +O extglob + } + } + default: + return w, hasC + } + } + return "", false +} + +// shellQuoteWord quotes one word for a POSIX shell, leaving a word that +// needs no quoting bare, so shellWords reads it back exactly. +func shellQuoteWord(w string) string { + if w != "" && strings.Trim(w, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_./:,+@%-") == "" { + return w + } + return shellSingleQuote(w) +} + +// nestedCommand is one command another command runs, and the directory it +// runs in. +type nestedCommand struct { + cmd, cwd string +} + +// nestedCommands lists the commands cmd runs as text, in order: each shell +// -c string, then each literal command line of its inline Python. A command +// whose working directory the hook cannot tell — after a cd it cannot +// follow, under a launcher that moves it, or from a script that may chdir — +// is left out rather than read with absolute paths only: it fails open. +func nestedCommands(shells []inlineScript, scripts []analyzedScript, dirs commandDirs, cwd string) []nestedCommand { + var out []nestedCommand + add := func(cmd, dir string) { + switch { + case filepath.IsAbs(dir): + case dir == "": + dir = cwd + case cwd == "": + return + default: + dir = filepath.Join(cwd, dir) + } + out = append(out, nestedCommand{cmd, dir}) + } + for _, shell := range shells { + if dir, ok := dirs.at(shell); ok { + add(shell.code, dir) + } + } + for _, script := range scripts { + if dir, ok := dirs.pythonDir(script.inlineScript); ok { + for _, line := range script.facts.commands { + add(line, dir) + } + } + } + return out +} + +// enrichNestedCommands runs the Bash policy over the nested commands of a +// command whose own answer is outer, which is not a deny. A nested deny wins; +// otherwise outer stands, or the first nested answer when outer is empty. +// The commands share the outer command's deadline, and those past the bound +// or the deadline are not examined. +func enrichNestedCommands(outer enrichResult, nested []nestedCommand, probe fileScopeProbe, depth int, deadline time.Time) enrichResult { + answer := outer + for i, n := range nested { + if i >= maxNestedCommands || !time.Now().Before(deadline) { + break + } + if len(n.cmd) > maxNestedCommand { + continue + } + r := enrichBashCommand(n.cmd, n.cwd, probe, depth+1, deadline) + if r.deny { + return r + } + if answer == (enrichResult{}) { + answer = r + } + } + return answer +} diff --git a/internal/hooks/bash_script_read.go b/internal/hooks/bash_script_read.go index 05a9b7ead..953511067 100644 --- a/internal/hooks/bash_script_read.go +++ b/internal/hooks/bash_script_read.go @@ -31,12 +31,14 @@ const ( inlineScriptTime = 500 * time.Millisecond ) -// pythonScript is one inline Python program and where it starts in the -// command, so a `cd` before it can be told from one after it. -type pythonScript struct { +// inlineScript is one inline program — a Python script, or the command +// string of `bash -c` — and where it starts in the command, so a `cd` before +// it can be told from one after it. +type inlineScript struct { code string pos int cwdUnknown bool // a launcher ran it elsewhere (env -C, uv run --directory) + wrapped bool // found only by looking through skipLaunchWrappers' wrappers } // pythonBinaryRE matches python, python3, python3.12, pypy3 and the like. @@ -64,10 +66,21 @@ var interpreterLaunchers = map[string]struct { // Python. cwdChanged reports a launcher option that runs it in another // directory. func pythonCommand(words []string) (python []string, cwdChanged bool) { - words = skipCommandPrefix(words) - for len(words) > 0 { + return launchedCommand(words, pythonBinaryRE.MatchString, false) +} + +// launchedCommand returns the words from the first binary `is` accepts on, +// looking past the same prefixes and launchers as pythonCommand, and with +// wrappers also past skipLaunchWrappers' wrappers before every command. +func launchedCommand(words []string, is func(name string) bool, wrappers bool) (command []string, cwdChanged bool) { + for words = skipCommandPrefix(words); len(words) > 0; { + if wrappers { + if words = skipLaunchWrappers(words); len(words) == 0 { + break + } + } name := baseCommandName(words[0]) - if pythonBinaryRE.MatchString(name) { + if is(name) { return words, cwdChanged } launcher, ok := interpreterLaunchers[name] @@ -105,6 +118,80 @@ func pythonCommand(words []string) (python []string, cwdChanged bool) { return nil, false } +// findPython is pythonCommand, then, with extra and when that finds no +// Python, the same search through skipLaunchWrappers' wrappers; wrapped +// reports the second. +func findPython(words []string, extra bool) (python []string, cwdChanged, wrapped bool) { + if python, cwdChanged = pythonCommand(words); python != nil || !extra { + return python, cwdChanged, false + } + python, cwdChanged = launchedCommand(words, pythonBinaryRE.MatchString, true) + return python, cwdChanged, python != nil +} + +// skipLaunchWrappers is skipCommandPrefix that also looks through the +// wrappers that run the command after them in the same filesystem and +// directory — `exec`, `nohup`, `setsid`, `nice`, `command`, `nvm exec +// `, `direnv exec ` — when finding an inline interpreter or a +// nested shell. The shell read and write shapes do not use it: a wrapped +// first statement would stop their classification short of a later one. +// Wrappers that run the command against another root or host (chroot, +// docker, ssh) are not looked through, and neither is `command -v`, which +// runs nothing. +func skipLaunchWrappers(tokens []string) []string { + for tokens = skipCommandPrefix(tokens); len(tokens) > 0; tokens = skipCommandPrefix(tokens) { + var opts []string + switch t := tokens[0]; { + case t == "nohup": + tokens = tokens[1:] + case t == "exec": + tokens, _ = skipWrapperOptions(tokens[1:], "a") + case t == "setsid": + tokens, _ = skipWrapperOptions(tokens[1:], "") + case t == "nice": + tokens, _ = skipWrapperOptions(tokens[1:], "n") + case t == "command": + rest := tokens + if tokens, opts = skipWrapperOptions(tokens[1:], ""); strings.ContainsAny(strings.Join(opts, ""), "vV") { + return rest + } + case t == "nvm" && len(tokens) > 1 && tokens[1] == "exec": + if tokens, _ = skipWrapperOptions(tokens[2:], ""); len(tokens) > 0 && nvmVersionRE.MatchString(tokens[0]) { + tokens = tokens[1:] + } + case t == "direnv" && len(tokens) > 2 && tokens[1] == "exec": + tokens = tokens[3:] + default: + return tokens + } + } + return tokens +} + +// nvmVersionRE matches the version `nvm exec` takes before the command: 20, +// v20.1.0, node, lts/iron. The version is optional, so a word that is not +// one is the command. +var nvmVersionRE = regexp.MustCompile(`^(?:v?[0-9][0-9.]*|node|iojs|stable|lts/\S+)$`) + +// skipWrapperOptions drops the options a wrapper takes before its command +// and returns them. A short option in valued takes the next token as its +// value unless the value is attached; `--` ends the options. +func skipWrapperOptions(tokens []string, valued string) (rest, opts []string) { + for len(tokens) > 0 && strings.HasPrefix(tokens[0], "-") && tokens[0] != "-" { + opt := tokens[0] + tokens = tokens[1:] + if opt == "--" { + break + } + opts = append(opts, opt) + if last := opt[len(opt)-1:]; !strings.HasPrefix(opt, "--") && strings.Contains(valued, last) && len(tokens) > 0 { + opts = append(opts, tokens[0]) + tokens = tokens[1:] + } + } + return tokens, opts +} + // pythonInline reads a Python command line: the code of `-c CODE`, or // fromStdin when the program comes from standard input (`python3 -` or no // script at all). `-m module` and a script file are not inline code. @@ -207,15 +294,24 @@ func shellWords(s string) []string { } // pythonScriptsAt returns every inline Python program in cmd with its -// position: each `-c` statement, then each heredoc body fed to a Python -// reading standard input. Positions come from cursors that only move +// position. +func pythonScriptsAt(cmd string) []inlineScript { + scripts, _ := inlineScriptsAt(cmd, false) + return scripts +} + +// inlineScriptsAt returns every inline Python program in cmd with its +// position — each `-c` statement, then each heredoc body fed to a Python +// reading standard input — and the command string of each `bash -c` (or +// sh, dash, zsh) statement. Positions come from cursors that only move // forward, so a long command costs one pass and a repeated statement gets -// its own position. -func pythonScriptsAt(cmd string) []pythonScript { - if !strings.Contains(cmd, "python") && !strings.Contains(cmd, "pypy") { - return nil +// its own position. Without extra it finds only what pythonCommand finds, +// and no shells: no Python found only through a wrapper. +func inlineScriptsAt(cmd string, extra bool) (scripts, shells []inlineScript) { + hasPython := strings.Contains(cmd, "python") || strings.Contains(cmd, "pypy") + if !hasPython && (!extra || !strings.Contains(cmd, "sh")) { + return nil, nil } - var scripts []pythonScript // A text not found verbatim (a segment the splitter rewrote, such as // a backslash-newline) loses the cursor: what follows has no position, // which only gives up on relative paths, and the scan stays one pass. @@ -236,12 +332,20 @@ func pythonScriptsAt(cmd string) []pythonScript { for _, segment := range allCommandSegments(cmd) { pos := at(&segCursor, segment) command, _ := splitSegmentRedirects(segment) - if words, moved := pythonCommand(shellWords(command)); words != nil { - if code, fromStdin, ok := pythonInline(words); ok && !fromStdin && code != "" { - scripts = append(scripts, pythonScript{code: code, pos: pos, cwdUnknown: moved}) + words := shellWords(command) + if python, moved, wrapped := findPython(words, extra); python != nil { + if code, fromStdin, ok := pythonInline(python); ok && !fromStdin && code != "" { + scripts = append(scripts, inlineScript{code: code, pos: pos, cwdUnknown: moved, wrapped: wrapped}) + } + } else if shell, moved := launchedCommand(words, isNestingShell, true); extra && shell != nil { + if code, ok := shellInline(shell); ok && code != "" { + shells = append(shells, inlineScript{code: code, pos: pos, cwdUnknown: moved}) } } } + if !hasPython { + return nil, shells + } _, heredocs := splitHeredocs(cmd) docCursor := 0 for _, heredoc := range heredocs { @@ -265,13 +369,13 @@ func pythonScriptsAt(cmd string) []pythonScript { continue } command, _ := splitSegmentRedirects(segments[len(segments)-1]) - if words, moved := pythonCommand(shellWords(command)); words != nil { + if words, moved, wrapped := findPython(shellWords(command), extra); words != nil { if _, fromStdin, ok := pythonInline(words); ok && fromStdin { - scripts = append(scripts, pythonScript{code: heredoc.body, pos: pos, cwdUnknown: moved}) + scripts = append(scripts, inlineScript{code: heredoc.body, pos: pos, cwdUnknown: moved, wrapped: wrapped}) } } } - return scripts + return scripts, shells } // segmentChangesDir reports whether a segment runs cd or pushd anywhere in @@ -313,60 +417,146 @@ func cdPositions(cmd string, leading int) []int { return positions } +// analyzedScript is an inline Python script and what its analysis found. +type analyzedScript struct { + inlineScript + facts pyScriptFacts +} + +// analyzeInlineScripts analyses up to maxInlineScripts scripts by deadline, +// dropping each one the analysis has no opinion on; extra is the +// analysis mode (see analyzePythonScript). The scripts found only through a +// wrapper come last and are capped on their own. +func analyzeInlineScripts(scripts []inlineScript, deadline time.Time, extra bool) []analyzedScript { + var out []analyzedScript + for _, wrapped := range []bool{false, true} { + n := 0 + for _, script := range scripts { + if script.wrapped != wrapped || n >= maxInlineScripts { + continue + } + n++ + if facts, ok := analyzePythonScript(script.code, deadline, extra); ok { + out = append(out, analyzedScript{script, facts}) + } + } + } + return out +} + +// commandDirs is where the statements of one command run: the directory its +// leading `cd` chain ends in, and where each cd the chain cannot follow +// starts. +type commandDirs struct { + dir string + cds []int +} + +func newCommandDirs(cmd string) commandDirs { + dir, segments := leadingCdChain(cmd) + return commandDirs{dir: dir, cds: cdPositions(cmd, segments)} +} + +// at returns the directory a script runs in — relative to the hook's cwd +// unless absolute, "" for the cwd itself — or false when a cd the chain +// cannot follow comes before it, or a launcher ran it elsewhere. +func (d commandDirs) at(script inlineScript) (string, bool) { + for _, at := range d.cds { + if at < 0 || script.pos < 0 || at < script.pos { + return "", false + } + } + if script.cwdUnknown { + return "", false + } + return d.dir, true +} + +// pythonDir is at for a Python script, which may also change its own +// working directory (os.chdir, fchdir, chdir imported or aliased from os): +// then nothing relative in it resolves against the hook's cwd. +func (d commandDirs) pythonDir(script inlineScript) (string, bool) { + if strings.Contains(script.code, "chdir") { + return "", false + } + return d.at(script) +} + +// resolve returns the path a script's literal path p names, relative to +// the hook's cwd unless absolute, or false when it cannot be told. +func (d commandDirs) resolve(script inlineScript, p string) (string, bool) { + if filepath.IsAbs(p) { + return p, true + } + dir, ok := d.pythonDir(script) + if !ok { + return "", false + } + if dir != "" { + return filepath.Join(dir, p), true + } + return p, true +} + +// scriptWrites returns the files the scripts open for writing through +// os.open, which no shell write pattern sees, as `script write` shapes: such +// a write gets the answer open(p, 'w') gets. +func scriptWrites(scripts []analyzedScript, dirs commandDirs) []BashWrite { + var writes []BashWrite + for _, script := range scripts { + for _, p := range script.facts.writes { + if at, ok := dirs.resolve(script.inlineScript, p); ok && len(writes) < bashWriteProbeLimit { + writes = append(writes, BashWrite{Path: at, Shape: "script write"}) + } + } + } + return writes +} + // enrichBashScriptRead answers inline Python that reads indexed source. // answered is false when the command runs no such script, so the caller // falls through to its own verdict. -func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe) (enrichResult, bool) { +// The deadline bounds the analysis; a zero one is inlineScriptTime from now. +// What only the extra analysis sees is answered by enrichBashCommand after +// this, with its own budget. +func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe, deadline time.Time) (enrichResult, bool) { started := time.Now() - deadline := started.Add(inlineScriptTime) + if deadline.IsZero() { + deadline = started.Add(inlineScriptTime) + } scripts := pythonScriptsAt(cmd) if len(scripts) == 0 { return enrichResult{}, false } - if len(scripts) > maxInlineScripts { - scripts = scripts[:maxInlineScripts] - } - cdDir, cdSegments := leadingCdChain(cmd) - cds := cdPositions(cmd, cdSegments) - resolve := func(script pythonScript, p string) (string, bool) { - if filepath.IsAbs(p) { - return p, true - } - // A cd the chain cannot follow, made before the script runs, or a - // chdir inside it: the relative path is no longer the hook's. - for _, at := range cds { - if at < 0 || script.pos < 0 || at < script.pos { - return "", false - } - } - // A possible working-directory change inside the script (os.chdir, - // fchdir, chdir imported or aliased from os): a relative path no - // longer resolves against the hook's cwd. - if script.cwdUnknown || strings.Contains(script.code, "chdir") { - return "", false - } - if cdDir != "" { - return filepath.Join(cdDir, p), true - } - return p, true - } + dirs := newCommandDirs(cmd) + return scriptReadAnswer(analyzeInlineScripts(scripts, deadline, false), false, dirs, cwd, scopeProbe, started) +} +// scriptReadAnswer answers the reads and edits of the scripts: with second +// false, the open()/pathlib reads and edits of the scripts found without a +// wrapper; with second true, os.open reads and the reads and edits of the +// scripts found only through a wrapper. Each call has its own probe budget. +func scriptReadAnswer(scripts []analyzedScript, second bool, dirs commandDirs, cwd string, scopeProbe fileScopeProbe, started time.Time) (enrichResult, bool) { type candidate struct { - script pythonScript + script inlineScript path string guessed bool // the write target is unnamed: it may be another file } var edits, reads []candidate for _, script := range scripts { - facts, ok := analyzePythonScript(script.code, deadline) - if !ok { - continue + facts := script.facts + if script.wrapped == second { + for _, p := range facts.reads { + if facts.writeBack[p] { + edits = append(edits, candidate{script.inlineScript, p, facts.guessed[p]}) + } else { + reads = append(reads, candidate{script.inlineScript, p, false}) + } + } } - for _, p := range facts.reads { - if facts.writeBack[p] { - edits = append(edits, candidate{script, p, facts.guessed[p]}) - } else { - reads = append(reads, candidate{script, p, false}) + if second { + for _, p := range facts.osReads { + reads = append(reads, candidate{script.inlineScript, p, false}) } } } @@ -379,7 +569,7 @@ func enrichBashScriptRead(cmd, cwd string, scopeProbe fileScopeProbe) (enrichRes if probes >= bashWriteProbeLimit { return fileIndexStatus{}, false } - at, ok := resolve(c.script, c.path) + at, ok := dirs.resolve(c.script, c.path) if !ok { return fileIndexStatus{}, false } diff --git a/internal/hooks/enrich_bash_script_test.go b/internal/hooks/enrich_bash_script_test.go index 19b8c6bfe..84136c995 100644 --- a/internal/hooks/enrich_bash_script_test.go +++ b/internal/hooks/enrich_bash_script_test.go @@ -29,6 +29,11 @@ func TestEnrichBash_InlinePython(t *testing.T) { silent = "silent" // nothing at all allow = "allow" // no deny (another door may still advise) catDeny = "catDeny" // the shell read's own deny + readA = "readA" // the shell read's deny, for a.go + sedA = "sedA" // the shell-write advisory, for sed -i a.go + scriptW = "scriptW" // the shell-write advisory, for a script write to a.go + + pyRead = `python3 -c "print(open('a.go').read())"` ) for _, tc := range []struct { name, cmd, want string @@ -69,6 +74,7 @@ func TestEnrichBash_InlinePython(t *testing.T) { // Everything else passes. {"pure write", `python3 -c "open('a.go', \"w\").write('package a')"`, allow}, + {"pure write, plain quotes", `python3 -c "open('a.go', 'w').write('package a')"`, scriptW}, {"syntax check", `python -c 'import ast; ast.parse(open("a.go").read()); print("syntax ok")'`, silent}, {"data file", `python3 -c "import json; print(json.load(open('package.json')))"`, silent}, {"unindexed source", `python3 -c "print(open('c.go').read())"`, silent}, @@ -119,6 +125,110 @@ func TestEnrichBash_InlinePython(t *testing.T) { {"env -C", `env -C /tmp python3 -c "print(open('a.go').read())"`, silent}, {"uv run --directory", `uv run --directory=/tmp python -c "print(open('a.go').read())"`, silent}, {"env -C attached", `env -iC/tmp python3 -c "print(open('a.go').read())"`, silent}, + + // Wrappers that run the command in the same filesystem and directory + // are looked through; those that run it against another root, and + // `command -v`, which runs nothing, are not. + {"exec", "exec " + pyRead, deny}, + {"nohup", "nohup " + pyRead, deny}, + {"setsid", "setsid -f " + pyRead, deny}, + {"nice -n", "nice -n 5 " + pyRead, deny}, + {"nice -N", "nice -5 " + pyRead, deny}, + {"nice --adjustment", "nice --adjustment=3 " + pyRead, deny}, + {"command", "command " + pyRead, deny}, + {"nvm exec", "nvm exec 20 " + pyRead, deny}, + {"nvm exec without a version", "nvm exec " + pyRead, deny}, + {"direnv exec", "direnv exec . " + pyRead, deny}, + {"wrapped launcher", "timeout 5 nohup " + pyRead, deny}, + {"command -v", "command -v " + pyRead, silent}, + // The shell read and write shapes do not look through them: a wrapped + // first statement must not stop the classification before a later one. + // What the hook newly sees (os.open reads, wrapped scripts) has its + // own probe budget: it never spends the one a later read needs. + {"os.open reads before a read", `python3 -c "import os; os.open('x0.py', os.O_RDONLY); os.open('x1.py', os.O_RDONLY); os.open('x2.py', os.O_RDONLY); print(open('a.go').read())"`, deny}, + {"os.open creates before a read", `python3 -c "import os; os.open('/tmp/x0.py', os.O_CREAT); os.open('/tmp/x1.py', os.O_CREAT); os.open('/tmp/x2.py', os.O_CREAT); print(open('a.go').read())"`, deny}, + {"os.open reads, then a read", `python3 -c "import os; os.open('x0.py', os.O_RDONLY); os.open('x1.py', os.O_RDONLY); os.open('x2.py', os.O_RDONLY)" && ` + pyRead, deny}, + {"wrapped reads before a read", `nohup python3 -c "print(open('x0.py').read())"; nohup python3 -c "print(open('x1.py').read())"; nohup python3 -c "print(open('x2.py').read())"; ` + pyRead, deny}, + {"wrapped script reading three files before a read", `nice -n 5 python3 -c "print(open('x0.py').read(), open('x1.py').read(), open('x2.py').read())"; ` + pyRead, deny}, + {"wrapped rewrites before a rewrite", `exec python3 -c "p='x0.py'; s=open(p).read(); open(p,'w').write(s)"; exec python3 -c "p='x1.py'; s=open(p).read(); open(p,'w').write(s)"; exec python3 -c "p='x2.py'; s=open(p).read(); open(p,'w').write(s)"; python3 -c "p='a.go'; s=open(p).read(); open(p,'w').write(s)"`, edit}, + {"wrapped command, then a cat", `nohup ls && cat a.go`, readA}, + {"command, then a tail", `command ls; tail a.go`, readA}, + {"wrapped cat of other source, then a cat", `nice -n 5 cat c.go && cat a.go`, readA}, + {"wrapped cat", `nohup cat a.go`, silent}, + {"chroot", "chroot /x " + pyRead, silent}, + + // os.open reads and writes by its literal flags. + {"os.open read", `python3 -c "import os; fd = os.open('a.go', os.O_RDONLY); print(os.read(fd, 99))"`, deny}, + {"aliased os.open", `python3 -c "from os import open as o, read, O_RDONLY; print(read(o('a.go', O_RDONLY), 99))"`, silent}, + {"os.open bare flag", `python3 -c "import os; from os import O_RDONLY, O_CLOEXEC; print(os.read(os.open('a.go', O_RDONLY | O_CLOEXEC), 99))"`, deny}, + // An os.open write is a write target only: never a read, never what + // turns a read into an edit, so it hides no Read deny. + {"os.open write and a read of the same file", `python3 -c "import os; print(open('a.go').read()); os.open('a.go', os.O_WRONLY)"`, deny}, + {"os.open read-write and a read of another file", `python3 -c "import os; os.open('a.go', os.O_RDWR); print(open('b.go').read())"`, deny}, + {"Python read, then an os.open read-write", `python3 -c "print(open('b.go').read())" && python3 -c "import os; os.open('a.go', os.O_RDWR)"`, deny}, + {"os.open O_PATH", `python3 -c "import os; os.open('a.go', os.O_RDONLY | os.O_PATH)"`, silent}, + {"os.open O_DIRECTORY", `python3 -c "import os; os.open('a.go', os.O_DIRECTORY)"`, silent}, + {"os.open read-modify-write", "python3 - <<'PY'\nimport os\nfd = os.open('a.go', os.O_RDWR)\ns = os.read(fd, 1 << 20)\nos.lseek(fd, 0, 0)\nos.write(fd, s)\nPY", scriptW}, + {"os.open write", `python3 -c "import os; os.write(os.open('a.go', os.O_WRONLY | os.O_TRUNC), b'package a')"`, scriptW}, + {"os.open variable flags", `python3 -c "import os, sys; f = int(sys.argv[1]); print(os.read(os.open('a.go', f), 99))"`, silent}, + {"os.open flag not from os", `python3 -c "import os; print(os.read(os.open('a.go', O_RDONLY), 99))"`, silent}, + {"os.open create", `python3 -c "import os; print(os.read(os.open('a.go', os.O_RDONLY | os.O_CREAT), 99))"`, deny}, + {"os.open dir_fd", `python3 -c "import os; d = os.open('src', os.O_RDONLY); print(os.read(os.open('a.go', os.O_RDONLY, dir_fd=d), 99))"`, silent}, + {"os aliased onto another module", `python3 -c "import mymod as os; print(os.read(os.open('a.go', os.O_RDONLY), 99))"`, silent}, + + // An os.open write never softens a deny the rest of the command earns. + {"os.open write, then a cat", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && cat b.go`, catDeny}, + {"os.open write after a cat", `cat b.go && python3 -c "import os; os.open('a.go', os.O_WRONLY)"`, catDeny}, + {"os.open write, then a head", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && head b.go`, deny}, + {"os.open write and a read", `python3 -c "import os; os.open('a.go', os.O_WRONLY); print(open('b.go').read())"`, deny}, + {"os.open write, then a Python read", `python3 -c "import os; os.open('a.go', os.O_WRONLY)" && python3 -c "print(open('b.go').read())"`, deny}, + {"open write, then a cat", `python3 -c "open('a.go', 'w').write('x')" && cat b.go`, scriptW}, + {"os.open write, unindexed", `python3 -c "import os; os.write(os.open('c.go', os.O_WRONLY), b'x')"`, silent}, + + // A command run as text gets the answer it would get typed: the + // -c string of a shell, and a literal command line Python runs. + {"bash -c", `bash -c "cat a.go"`, readA}, + {"bash -lc", `bash -lc 'head -5 a.go'`, readA}, + {"sh with options", `sh -o pipefail -ec 'cat a.go'`, readA}, + {"wrapped shell", `nohup bash -c 'cat a.go'`, readA}, + {"python in a shell", `sh -c 'python3 -c "print(open(\"a.go\").read())"'`, deny}, + {"shell write in a shell", `bash -c "sed -i s/x/y/ a.go"`, sedA}, + {"nested deny beats an outer advisory", `sed -i s/x/y/ a.go && bash -c 'cat b.go'`, catDeny}, + {"os.popen", `python3 -c "import os; print(os.popen('cat a.go').read())"`, readA}, + {"os.system", `python3 -c "import os; os.system('cat a.go')"`, readA}, + {"subprocess list", `python3 -c "import subprocess; subprocess.run(['cat', 'a.go'])"`, readA}, + {"check_output tuple", `python3 -c "import subprocess; print(subprocess.check_output(('tail', '-n', '5', 'a.go')))"`, readA}, + {"getoutput", `python3 -c "import subprocess; print(subprocess.getoutput('head a.go'))"`, readA}, + {"subprocess shell write", `python3 -c "import subprocess; subprocess.run('sed -i s/x/y/ a.go', shell=True)"`, sedA}, + {"subprocess python", `python3 -c "import subprocess; subprocess.run(['python3', '-c', 'print(open(\'a.go\').read())'])"`, silent}, + {"subprocess python, quoted", "python3 - <<'PY'\nimport subprocess\nsubprocess.run(['python3', '-c', \"print(open('a.go').read())\"])\nPY", deny}, + + {"module imported with a submodule", `python3 -c "import os, os.path; os.system('cat a.go')"`, readA}, + {"subprocess string with shell=True", `python3 -c "import subprocess; print(subprocess.check_output('cat a.go', shell=True))"`, readA}, + {"subprocess string without a shell", `python3 -c "import subprocess; subprocess.run('cat a.go')"`, silent}, + {"os.system list", `python3 -c "import os; os.system(['cat', 'a.go'])"`, silent}, + {"os aliased onto another module, system", `python3 -c "import mylib as os; os.system('cat a.go')"`, silent}, + {"subprocess rebound", "python3 - <<'PY'\nimport subprocess\nsubprocess = None\nsubprocess.run(['cat', 'a.go'])\nPY", silent}, + {"local class os", "python3 - <<'PY'\nclass os:\n def system(c): pass\nos.system('cat a.go')\nPY", silent}, + {"os never imported", `python3 -c "os.system('cat a.go')"`, silent}, + {"nested script file", `bash run.sh a.go`, silent}, + {"nested unindexed", `bash -c "cat c.go" && python3 -c "import os; os.system('cat c.go')"`, silent}, + {"two levels", `bash -c "bash -c 'cat a.go'"`, silent}, + {"two levels through python", `bash -c 'python3 -c "import os; os.system(\"cat a.go\")"'`, silent}, + {"shell list from python", `python3 -c "import subprocess; subprocess.run(['bash', '-c', 'cat a.go'])"`, silent}, + {"subprocess cwd", `python3 -c "import subprocess; subprocess.run(['cat', 'a.go'], cwd='/tmp')"`, silent}, + {"subprocess executable", `python3 -c "import subprocess; subprocess.run('cat a.go', shell=True, executable='/bin/zsh')"`, silent}, + {"subprocess *args", `python3 -c "import subprocess, sys; subprocess.run(['cat', 'a.go'], *sys.argv[1:])"`, silent}, + {"subprocess **kwargs", `python3 -c "import subprocess; kw = {'cwd': '/tmp'}; subprocess.run(['cat', 'a.go'], **kw)"`, silent}, + {"subprocess variable", `python3 -c "import subprocess; c = ['cat', 'a.go']; subprocess.run(c)"`, silent}, + {"os.system f-string", `python3 -c "import os; f = 'a.go'; os.system(f'cat {f}')"`, silent}, + {"list handed to the shell", `python3 -c "import subprocess; subprocess.run(['cat a.go'], shell=True)"`, silent}, + {"nested after cd to a variable", `cd $V && bash -c "cat a.go"`, silent}, + {"python nested after cd to a variable", `cd $V && python3 -c "import os; os.system('cat a.go')"`, silent}, + {"nested after chdir", `python3 -c "import os; os.chdir('/tmp'); os.system('cat a.go')"`, silent}, + {"nested under env -C", `env -C /tmp bash -c "cat a.go"`, silent}, + {"within the nested bound", strings.Repeat(`bash -c "cat c.go"; `, 7) + `bash -c "cat a.go"`, readA}, + {"past the nested bound", strings.Repeat(`bash -c "cat c.go"; `, 8) + `bash -c "cat a.go"`, silent}, } { t.Run(tc.name, func(t *testing.T) { redirectTelemetry(t) @@ -138,6 +248,15 @@ func TestEnrichBash_InlinePython(t *testing.T) { if !r.deny || !strings.Contains(r.reason, "Bash `cat b.go` reads indexed source") { t.Errorf("want the cat deny, got %+v", r) } + case readA: + if !r.deny || !strings.Contains(r.reason, " a.go` reads indexed source") { + t.Errorf("want the shell read deny, got %+v", r) + } + case sedA, scriptW: + shape := map[string]string{sedA: "sed -i", scriptW: "script write"}[tc.want] + if r.deny || !strings.Contains(r.context, "Bash `"+shape+"` writes a.go") { + t.Errorf("want the %s advisory, got %+v", shape, r) + } case maybe: if r.deny || !strings.Contains(r.context, "Python script may rewrite") { t.Errorf("want the hedged edit advisory, got %+v", r) @@ -167,6 +286,51 @@ func TestEnrichBash_InlinePythonEditBlocking(t *testing.T) { } } +// A write-only os.open gets the shell-write deny under edit blocking, as +// open(p, 'w') does. +func TestEnrichBash_InlinePythonOSOpenEditBlocking(t *testing.T) { + redirectTelemetry(t) + stubIndexedPaths(t, "a.go") + t.Setenv("GORTEX_HOOK_BLOCK_EDIT", "1") + for _, cmd := range []string{ + `python3 -c "open('a.go', 'w').write('x')"`, + `python3 -c "import os; os.open('a.go', os.O_WRONLY)"`, + } { + r := enrichBash(map[string]any{"command": cmd}, "/repo") + if !r.deny || !strings.Contains(r.reason, "Bash `script write` writes a.go") { + t.Errorf("%s: want the shell-write deny, got %+v", cmd, r) + } + } +} + +// What only the extra analysis looks for (os.open, command lines) costs +// the read analysis nothing: a script it covers just under its step budget +// keeps the Read deny however many os.open calls it carries. +func TestEnrichBash_InlinePythonExtraAnalysisBudget(t *testing.T) { + redirectTelemetry(t) + stubIndexedPaths(t, "a.go") + t.Setenv("GORTEX_HOOK_BLOCK_EDIT", "") + lines := []string{"import os", "print(open('a.go').read())"} + for i := 0; i < 90; i++ { + lines = append(lines, "os.open("+strings.Repeat("'a'+", 199)+"'a', os.O_RDONLY)") + } + code := strings.Join(lines, "\n") + if _, ok := analyzePythonScript(code, time.Now().Add(time.Minute), true); ok { + t.Fatal("the extra analysis of this script should exceed its own step budget") + } + facts, ok := analyzePythonScript(code, time.Now().Add(time.Minute), false) + if !ok || len(facts.reads) != 1 || facts.reads[0] != "a.go" { + t.Fatalf("the read analysis must cover the script, got %v %+v", ok, facts) + } + // The whole policy, under a deadline that a slow (-race) build cannot + // miss. + cmd := "python3 - <<'EOF'\n" + code + "\nEOF" + r := enrichBashCommand(cmd, "/repo", queryFileIndexScope, 0, time.Now().Add(time.Minute)) + if !r.deny || !strings.Contains(r.reason, "reads indexed source") { + t.Errorf("want the Read deny, got %+v", r) + } +} + // A redirected Python patch script is counted with the other shell writes. func TestEnrichBash_InlinePythonEditTelemetry(t *testing.T) { log := redirectTelemetry(t) diff --git a/internal/hooks/pretooluse.go b/internal/hooks/pretooluse.go index 665c2dee5..fcae4a2bb 100644 --- a/internal/hooks/pretooluse.go +++ b/internal/hooks/pretooluse.go @@ -1455,7 +1455,60 @@ func enrichBashWithProbe(toolInput map[string]any, cwd string, probe fileScopePr if cmd == "" { return enrichResult{} } + return enrichBashCommand(cmd, cwd, probe, 0, time.Time{}) +} + +// enrichBashCommand answers one command at the given nesting depth: 0 for +// the command the agent typed, 1 for a command it runs as text (`bash -c`, +// a Python os.system), whose own nested commands are not examined. +// +// The shapes the hook has always answered come first, exactly as before: +// their analysis and probes share nothing with what follows. Only a +// command they do not deny is looked at again for what they cannot see — +// os.open, scripts behind a wrapper, nested commands — in a second +// analysis with its own budget, starting when it begins. A zero deadline +// gives each part its own; a nested command gets the deadline of the +// command it is nested in. +func enrichBashCommand(cmd, cwd string, probe fileScopeProbe, depth int, deadline time.Time) enrichResult { + r := enrichBashShapes(cmd, cwd, probe, deadline) + if r.deny { + return r + } + if deadline.IsZero() { + deadline = time.Now().Add(inlineScriptTime) + } + pyScripts, shells := inlineScriptsAt(cmd, true) + scripts := analyzeInlineScripts(pyScripts, deadline, true) + if len(scripts) == 0 && len(shells) == 0 { + return r + } + dirs := newCommandDirs(cmd) + if extra, ok := scriptReadAnswer(scripts, true, dirs, cwd, probe, time.Now()); ok { + if extra.deny { + return extra + } + if r == (enrichResult{}) { + r = extra + } + } + // A Python os.open write is a write shape the patterns cannot see: it + // gets the shell-write answer. + if writes := scriptWrites(scripts, dirs); len(writes) > 0 { + if w, answered := enrichBashWrite(BashClassification{Action: BashActionWriteSource, Writes: writes}, cwd); answered { + if w.deny { + return w + } + r = w + } + } + if depth > 0 { + return r + } + return enrichNestedCommands(r, nestedCommands(shells, scripts, dirs, cwd), probe, depth, deadline) +} +// enrichBashShapes answers the shell shapes and inline Python of a command. +func enrichBashShapes(cmd, cwd string, probe fileScopeProbe, deadline time.Time) enrichResult { c := classifyBashCommand(cmd) if c.Action == BashActionWriteSource { if result, answered := enrichBashWrite(c, cwd); answered { @@ -1488,7 +1541,7 @@ func enrichBashWithProbe(toolInput map[string]any, cwd string, probe fileScopePr // read-only or loading allowance) must not cost the command a deny its // shell read earns on its own, so the shell answer is taken first when // it denies. - py, pyAnswered := enrichBashScriptRead(cmd, cwd, probe) + py, pyAnswered := enrichBashScriptRead(cmd, cwd, probe, deadline) if pyAnswered && (py.deny || c.Action == BashActionPassthrough) { return py } diff --git a/internal/hooks/python_script_facts.go b/internal/hooks/python_script_facts.go index 2a34847ca..e9bae7e41 100644 --- a/internal/hooks/python_script_facts.go +++ b/internal/hooks/python_script_facts.go @@ -2,6 +2,7 @@ package hooks import ( "context" + "slices" "sort" "strings" "time" @@ -25,6 +26,9 @@ type pyScriptFacts struct { writeBack map[string]bool // literal paths written back guessed map[string]bool // written back only by a write it cannot name seen map[string]bool + osReads []string // literal source paths os.open opens read-only + writes []string // literal paths os.open opens for writing + commands []string // literal command lines the script runs (os.system, subprocess) } // pyMaxSteps bounds the analysis of one script and pyMaxDepth its expression @@ -54,6 +58,12 @@ type pyAnalysis struct { classes map[uintptr]bool ambiguous map[string]bool bound map[string]bool // every name bound anywhere, built on first use + osImports map[string]bool // names imported by `from os import NAME` + imports map[string]int // bindings of a name by a plain `import name` + fromNames map[string]int // bindings of a name as part of `from module import …` + bindCount map[string]int // bindings of a name in any scope, built on first use + osStar bool // `from os import *` + extra bool // collect what only the extra analysis answers loopEnd uint32 // end of the outermost loop around the node collect is at deadline time.Time root uintptr @@ -78,12 +88,19 @@ var ( "os.path.normpath": true, "os.fspath": true, "str": true, "Path": true, "pathlib.Path": true, "PurePath": true, "pathlib.PurePath": true} pyOpenModules = map[string]bool{"io": true, "codecs": true, "builtins": true} + // pyCommandCalls run a command line, or an argument vector as a list. + pyCommandCalls = map[string]bool{"os.system": true, "os.popen": true, + "subprocess.run": true, "subprocess.call": true, "subprocess.check_call": true, + "subprocess.check_output": true, "subprocess.Popen": true, + "subprocess.getoutput": true, "subprocess.getstatusoutput": true} ) // analyzePythonScript parses one script and collects what it reads and // writes. ok is false when the script does not parse cleanly or exceeds the -// analysis budget. -func analyzePythonScript(code string, deadline time.Time) (pyScriptFacts, bool) { +// analysis budget. With extra it also collects os.open accesses and the +// command lines the script runs; without, it does no work for them at all, +// so the reads and edits the hook has always answered cost what they did. +func analyzePythonScript(code string, deadline time.Time, extra bool) (pyScriptFacts, bool) { if len(code) > maxInlineScript || !time.Now().Before(deadline) { return pyScriptFacts{}, false } @@ -114,7 +131,11 @@ func analyzePythonScript(code string, deadline time.Time) (pyScriptFacts, bool) compr: map[uintptr]bool{}, classes: map[uintptr]bool{}, ambiguous: map[string]bool{}, + osImports: map[string]bool{}, + imports: map[string]int{}, + fromNames: map[string]int{}, deadline: deadline, + extra: extra, root: root.Id(), facts: pyScriptFacts{writeBack: map[string]bool{}, guessed: map[string]bool{}, seen: map[string]bool{}}, } @@ -238,6 +259,10 @@ func (a *pyAnalysis) bind(n *sitter.Node, scope uintptr, depth int) { } case "augmented_assignment": a.bindTargets(n.ChildByFieldName("left"), scope, "") + case "import_statement", "import_from_statement": + if a.extra { + a.recordImport(n) + } case "parameters", "lambda_parameters": // Parameters hold no path. Their defaults run where the function is // written, so a walrus in one binds the scope around it. @@ -616,6 +641,35 @@ func (a *pyAnalysis) collectCall(n *sitter.Node, scope uintptr) { return m, ok } switch { + case a.extra && (name == "os.open" || pyCommandCalls[name]) && !a.realModule(strings.SplitN(name, ".", 2)[0]): + // A name the script binds to something other than the module. + case a.extra && name == "os.open" && (kw["dir_fd"] != nil || hasSplat(pos)): + // Relative to a directory the hook cannot see, or flags it cannot. + case a.extra && name == "os.open": + path := kw["path"] + if len(pos) > 0 { + path = pos[0] + } + flags := kw["flags"] + if len(pos) > 1 { + flags = pos[1] + } + // A read is a read; a write is only a write target, answered after + // the rest of the command: it never makes a read an edit. + if write, ok := a.osOpenWrites(flags); ok && path != nil { + switch lit := a.eval(path, scope, 0); { + case !write: + if lit != "" && looksLikeSourceFile(lit) && !strings.ContainsAny(lit, "*?$") && !slices.Contains(a.facts.osReads, lit) && len(a.facts.osReads) < bashWriteProbeLimit { + a.facts.osReads = append(a.facts.osReads, lit) + } + case lit != "" && looksLikeSourceFile(lit) && !strings.ContainsAny(lit, "*?$") && len(a.facts.writes) < bashWriteProbeLimit: + a.facts.writes = append(a.facts.writes, lit) + } + } + case a.extra && pyCommandCalls[name]: + if line, ok := a.commandLine(name, pos, kw); ok && len(a.facts.commands) < maxNestedCommands { + a.facts.commands = append(a.facts.commands, line) + } case name == "open" && a.shadowed("open"): // The script defines or imports its own open. case name == "open" || (object != nil && method == "open" && pyOpenModules[a.dotted(object)]): @@ -641,6 +695,170 @@ func (a *pyAnalysis) collectCall(n *sitter.Node, scope uintptr) { } } +// osOpenWrites reads the flags of os.open built from os.O_* names joined by +// `|`: write is true when they open the file for writing (O_WRONLY, O_RDWR, +// O_TRUNC, O_APPEND), false for a read — O_CREAT alone changes no file that +// exists. ok is false when the flags are anything else, since a value the +// hook cannot read could be a write, and for O_PATH or O_DIRECTORY, which +// read no file content. +func (a *pyAnalysis) osOpenWrites(flags *sitter.Node) (write, ok bool) { + names := map[string]bool{} + var walk func(n *sitter.Node, depth int) bool + walk = func(n *sitter.Node, depth int) bool { + if n == nil || depth > pyMaxDepth || !a.step(depth) { + return false + } + var flag string + switch n.Type() { + case "binary_operator": + op := n.ChildByFieldName("operator") + return op != nil && a.text(op) == "|" && + walk(n.ChildByFieldName("left"), depth+1) && walk(n.ChildByFieldName("right"), depth+1) + case "parenthesized_expression": + return walk(n.NamedChild(0), depth+1) + case "attribute": + var ok bool + if flag, ok = strings.CutPrefix(a.dotted(n), "os."); !ok || !a.realModule("os") { + return false + } + case "identifier": + // Imported from os once and never rebound. + flag = a.text(n) + imported := a.osImports[flag] && a.bindings[pyBindingKey{flag, a.root}] != nil && a.bindingCount(flag) == 1 + if !imported && (!a.osStar || a.shadowed(flag)) { + return false + } + default: + return false + } + if !strings.HasPrefix(flag, "O_") || strings.Contains(flag, ".") { + return false + } + names[flag] = true + return true + } + if !walk(flags, 0) || names["O_PATH"] || names["O_DIRECTORY"] { + return false, false + } + return names["O_WRONLY"] || names["O_RDWR"] || names["O_TRUNC"] || names["O_APPEND"], true +} + +// commandLine returns the command line a pyCommandCalls call runs when it +// is literal: a string the shell runs as it is, or a list or tuple of +// strings joined with shell quoting. false for anything else, and for a +// call that runs it in another directory (cwd=) or with another program +// (executable=). os.system, os.popen and getoutput always run a string +// through the shell; the Popen family runs a string through it only with +// shell=True (without, the string is a program name), and a list only +// without it (with, the items past the first are arguments to the shell). +func (a *pyAnalysis) commandLine(name string, pos []*sitter.Node, kw map[string]*sitter.Node) (string, bool) { + // *args and **kwargs may carry anything, cwd= and shell= included. + if len(pos) == 0 || kw["cwd"] != nil || kw["executable"] != nil || hasSplat(pos) { + return "", false + } + alwaysShell := name == "os.system" || name == "os.popen" || strings.HasPrefix(name, "subprocess.get") + shell, hasShell := kw["shell"] + // An escape in the literal reads differently to Python and the shell. + literal := func(n *sitter.Node) (string, bool) { + s, ok := a.stringLiteral(n) + return s, ok && s != "" && !strings.Contains(s, `\`) + } + switch arg := pos[0]; arg.Type() { + case "string", "concatenated_string": + if !alwaysShell && (!hasShell || shell.Type() != "true") { + return "", false + } + return literal(arg) + case "list", "tuple": + if alwaysShell || hasShell && shell.Type() != "false" { + return "", false + } + var words []string + for c := range arg.NamedChildren() { + w, ok := literal(c) + if !ok || len(words) >= 256 { + return "", false + } + words = append(words, shellQuoteWord(w)) + } + return strings.Join(words, " "), len(words) > 0 + } + return "", false +} + +// recordImport counts the names an import binds for realModule: a plain +// `import os` / `import os.path` binds the module itself, and the module +// of a `from module import …` is recorded as bound too. Bare os.O_* flag +// names imported from os are recorded for os.open. +func (a *pyAnalysis) recordImport(n *sitter.Node) { + if n.Type() == "import_statement" { + for c := range n.NamedChildren() { + if c.Type() == "dotted_name" && c.NamedChildCount() > 0 { + a.imports[a.text(c.NamedChild(0))]++ + } + } + return + } + module := n.ChildByFieldName("module_name") + if module == nil { + return + } + a.countFromModule(module) + fromOS := a.text(module) == "os" + for c := range n.NamedChildren() { + switch { + case c.Id() == module.Id(): + case fromOS && c.Type() == "wildcard_import": + a.osStar = true + case fromOS && c.Type() == "dotted_name": + a.osImports[a.text(c)] = true + } + } +} + +// countFromModule counts the names bind records for the module of a +// `from module import …`, a dotted name alone or inside a relative import. +func (a *pyAnalysis) countFromModule(n *sitter.Node) { + for c := range n.NamedChildren() { + switch c.Type() { + case "identifier": + if n.Type() == "dotted_name" { + a.fromNames[a.text(c)]++ + } + case "dotted_name": + a.countFromModule(c) + } + } +} + +// hasSplat reports whether a call's arguments include *args or **kwargs. +func hasSplat(args []*sitter.Node) bool { + for _, arg := range args { + if t := arg.Type(); t == "list_splat" || t == "dictionary_splat" { + return true + } + } + return false +} + +// realModule reports whether name is bound only by plain `import name` +// statements, so name.attr is the module's own: never aliased onto +// another module, assigned, defined, deleted or declared global. +func (a *pyAnalysis) realModule(name string) bool { + return a.imports[name] > 0 && !a.ambiguous[name] && a.bindingCount(name) == a.imports[name]+a.fromNames[name] +} + +// bindingCount returns how many times the script binds name, in any scope. +func (a *pyAnalysis) bindingCount(name string) int { + if a.bindCount == nil { + a.bindCount = make(map[string]int, len(a.bindings)) + for key, bs := range a.bindings { + a.bindCount[key.name] += len(bs) + } + } + return a.bindCount[name] +} + // reboundLater reports whether the name read at pos is bound again further // down a loop that runs the read: on the next pass it holds that value. func (a *pyAnalysis) reboundLater(name string, pos uint32, scope uintptr) bool { From 926c2bb1c4ffee190c3fcd6e249da6d8a3e5c354 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sat, 31 Oct 2026 23:44:48 +0100 Subject: [PATCH 31/60] wip: refusal identity and idle-job accounting fixes (untested) Unfinished review fixes: build the join and draining refusals from the pending job's own root and family, leave idle authorization waits out of the family bound, and tighten two tests. The focused tests have not run on this commit; squash into the indexer and daemon commits once they pass. --- cmd/gortex/daemon_mcp.go | 5 +- .../daemon_mcp_checkout_admission_test.go | 6 ++ cmd/gortex/daemon_mcp_discovery_scope_test.go | 24 +++++ internal/indexer/checkout_observe.go | 26 +++-- internal/indexer/checkout_observe_test.go | 102 +++++++++++++++++- 5 files changed, 152 insertions(+), 11 deletions(-) diff --git a/cmd/gortex/daemon_mcp.go b/cmd/gortex/daemon_mcp.go index 29035320f..577be77c6 100644 --- a/cmd/gortex/daemon_mcp.go +++ b/cmd/gortex/daemon_mcp.go @@ -415,7 +415,10 @@ func checkoutAdmissionError(cwd string, inbound []byte, err error) []byte { // can tell one pending worktree from an unavailable daemon. var pending *indexer.CheckoutDiscoveryPendingError if errors.As(err, &pending) { - data["path"], data["family"], data["reason"], data["state"] = pending.Root, pending.Family, pending.Reason, "discovery_pending" + data["family"], data["reason"], data["state"] = pending.Family, pending.Reason, "discovery_pending" + if pending.Root != "" { + data["path"] = pending.Root + } } out, _ := json.Marshal(map[string]any{ "jsonrpc": "2.0", diff --git a/cmd/gortex/daemon_mcp_checkout_admission_test.go b/cmd/gortex/daemon_mcp_checkout_admission_test.go index 024e07ed3..21195c028 100644 --- a/cmd/gortex/daemon_mcp_checkout_admission_test.go +++ b/cmd/gortex/daemon_mcp_checkout_admission_test.go @@ -73,6 +73,12 @@ func TestCheckoutAdmissionErrorPendingDiscoveryGolden(t *testing.T) { const want = `{"error":{"code":-32000,"data":{"error_code":"view_building","family":"family-0123","path":"/repos/app-wt","reason":"family_capacity","retryable":true,"state":"discovery_pending"},` + `"message":"checkout discovery unavailable: observe checkout: indexer: checkout mutation lane is busy; retry: checkout discovery is pending for /repos/app-wt (family family-0123, reason family_capacity)"},"id":7,"jsonrpc":"2.0"}` require.Equal(t, want, string(got)) + + // A refusal that names no checkout root keeps the session's path. + got = checkoutAdmissionError("/repos/app-wt/internal", []byte(`{"jsonrpc":"2.0","id":8,"method":"tools/call"}`), &indexer.CheckoutDiscoveryPendingError{Reason: "in_progress"}) + var parsed map[string]any + require.NoError(t, json.Unmarshal(got, &parsed)) + require.Equal(t, "/repos/app-wt/internal", errorData(t, parsed)["path"], "%s", got) } type checkoutAdmissionSQLiteError int diff --git a/cmd/gortex/daemon_mcp_discovery_scope_test.go b/cmd/gortex/daemon_mcp_discovery_scope_test.go index dc81ae215..8ce7d85c0 100644 --- a/cmd/gortex/daemon_mcp_discovery_scope_test.go +++ b/cmd/gortex/daemon_mcp_discovery_scope_test.go @@ -118,6 +118,30 @@ func TestDispatcher_PendingDiscoveryStaysInItsWorktree(t *testing.T) { wg.Wait() } + // The checks below run once the hung discovery is past its work deadline + // and still running: a session waiting on it is then refused as draining. + waiting := -1 + for i, raw := range stuckReplies { + require.NoError(t, stuckErrs[i]) + var parsed map[string]any + require.NoError(t, json.Unmarshal(raw, &parsed)) + if errorData(t, parsed)["reason"] == "in_progress" { + waiting = i + break + } + } + require.NotEqual(t, -1, waiting, "no session waits on a hung discovery") + drainBy := time.Now().Add(6 * time.Second) + for { + data := errorData(t, dispatchJSON(t, d, stuckSessions[waiting], searchFrame)) + if data["reason"] == "draining" { + break + } + require.Equal(t, "in_progress", data["reason"], "%v", data) + require.True(t, time.Now().Before(drainBy), "the hung discovery did not pass its work deadline: %v", data) + time.Sleep(50 * time.Millisecond) + } + // Another family's new worktree gets its identity; it may wait for its own // discovery, never for the hung one. deadline := time.Now().Add(30 * time.Second) diff --git a/internal/indexer/checkout_observe.go b/internal/indexer/checkout_observe.go index 8bdd9ac25..5e18ec3b3 100644 --- a/internal/indexer/checkout_observe.go +++ b/internal/indexer/checkout_observe.go @@ -92,7 +92,9 @@ type checkoutObservationMarker struct { } type checkoutObservationJob struct { - familyID string + // The checkout and family the job was started for. Refusals of requests + // that join it name these, whatever those requests' markers read now. + checkoutObservationTarget ctx context.Context cancel context.CancelFunc proofReady chan struct{} @@ -154,6 +156,8 @@ func (l *CheckoutLifecycle) ObserveCheckoutPath(ctx context.Context, path string if err != nil || job == nil { return checkout, false, err } + // Refusals from here on name the joined discovery's checkout. + target = job.checkoutObservationTarget select { case <-ctx.Done(): return checkout, false, ctx.Err() @@ -236,7 +240,7 @@ func (l *CheckoutLifecycle) checkoutObservation(path string, target checkoutObse if job.ctx.Err() == nil { return job, nil } - return nil, target.refusal(checkoutDiscoveryDraining) + return nil, job.refusal(checkoutDiscoveryDraining) } if !start { return nil, nil @@ -244,10 +248,8 @@ func (l *CheckoutLifecycle) checkoutObservation(path string, target checkoutObse // One family's burst or stall never refuses another family's paths. inFlight, stuck, familyStuck := 0, 0, false for _, job := range l.observationJobs { - select { - case <-job.done: - continue // Finished; at most a retained result. - default: + if !job.holdsSlot() { + continue } expired := job.ctx.Err() != nil if expired { @@ -268,13 +270,23 @@ func (l *CheckoutLifecycle) checkoutObservation(path string, target checkoutObse l.observationJobs = make(map[string]*checkoutObservationJob) } ctx, cancel := context.WithTimeout(context.Background(), checkoutObservationWorkTimeout) - job := &checkoutObservationJob{familyID: target.familyID, ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} + job := &checkoutObservationJob{checkoutObservationTarget: target, ctx: ctx, cancel: cancel, proofReady: make(chan struct{}), authorized: make(chan struct{}), done: make(chan struct{})} l.observationJobs[path] = job l.observationWG.Add(1) go l.runCheckoutObservation(path, job) return job, nil } +// holdsSlot reports whether job counts against discovery capacity. A finished +// job keeps at most its result, and a proven job that no caller has authorized +// runs no Git work: it only waits for one until its deadline. +func (job *checkoutObservationJob) holdsSlot() bool { + if channelClosed(job.done) { + return false + } + return channelClosed(job.authorized) || !channelClosed(job.proofReady) +} + func (l *CheckoutLifecycle) runCheckoutObservation(path string, job *checkoutObservationJob) { defer l.observationWG.Done() defer job.cancel() diff --git a/internal/indexer/checkout_observe_test.go b/internal/indexer/checkout_observe_test.go index 7ebe34e15..4a8069659 100644 --- a/internal/indexer/checkout_observe_test.go +++ b/internal/indexer/checkout_observe_test.go @@ -368,9 +368,24 @@ func TestObserveCheckoutPathStuckFamilyLeavesOtherRequestsAlone(t *testing.T) { } observeOutside("stuck", len(jobs)) - checkout, found, err := observeCheckoutUntilSettled(t, lc, late) - if err != nil || !found || checkout.FamilyID != tracked.FamilyID || !pathkey.EqualPaths(checkout.RootPath, pathkey.CanonicalExistingRoot(late)) { - t.Errorf("another family's new worktree was refused: %+v found=%v err=%v", checkout, found, err) + // Another family's new worktree may wait for its own discovery only. + lateRoot := pathkey.CanonicalExistingRoot(late) + for deadline := time.Now().Add(checkoutObservationWorkTimeout); ; { + checkout, found, err := lc.ObserveCheckoutPath(t.Context(), late) + var pending *CheckoutDiscoveryPendingError + if !errors.As(err, &pending) { + if err != nil || !found || checkout.FamilyID != tracked.FamilyID || !pathkey.EqualPaths(checkout.RootPath, lateRoot) { + t.Errorf("another family's new worktree was not admitted: %+v found=%v err=%v", checkout, found, err) + } + break + } + if pending.Reason != checkoutDiscoveryInProgress || pending.Family != tracked.FamilyID || !pathkey.EqualPaths(pending.Root, lateRoot) { + t.Fatalf("another family's new worktree was refused: %v", err) + } + if time.Now().After(deadline) { + t.Fatalf("another family's new worktree discovery did not finish: %v", err) + } + time.Sleep(time.Millisecond) } extra := filepath.Join(stuck, "extra") if err := os.Mkdir(extra, 0o755); err != nil { @@ -386,6 +401,87 @@ func TestObserveCheckoutPathStuckFamilyLeavesOtherRequestsAlone(t *testing.T) { } } +// A request that joins a running discovery is refused for that discovery's +// checkout, whatever its own .git marker read finds now. Here the marker of a +// worktree nested in the primary checkout is removed, so a fresh read names +// the primary checkout instead. +func TestObserveCheckoutPathJoinedRefusalNamesItsOwnCheckout(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + nested := filepath.Join(primary, "nested") + builderGit(t, primary, "worktree", "add", "-b", "nested", nested) + nestedRoot := pathkey.CanonicalExistingRoot(nested) + release := make(chan struct{}) + defer close(release) + entered := make(chan struct{}) + var once sync.Once + lc.observationInventory = func(context.Context, string) (*gitstate.FamilyInventory, error) { + once.Do(func() { close(entered) }) + <-release // Ignores cancellation, so the job outlives its deadline. + return nil, context.Canceled + } + refused := func(phase, reason string) { + t.Helper() + _, found, err := lc.ObserveCheckoutPath(t.Context(), nested) + var pending *CheckoutDiscoveryPendingError + if found || !errors.As(err, &pending) || pending.Reason != reason || pending.Family != familyID || !pathkey.EqualPaths(pending.Root, nestedRoot) { + t.Fatalf("%s: want a %s refusal for %s in family %s, got found=%v err=%v", phase, reason, nestedRoot, familyID, found, err) + } + } + refused("first request", checkoutDiscoveryInProgress) + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("discovery did not reach the stalled inventory") + } + if err := os.Remove(filepath.Join(nested, ".git")); err != nil { + t.Fatal(err) + } + refused("joined request", checkoutDiscoveryInProgress) + job := checkoutObservationJobForTest(lc, nested) + if job == nil { + t.Fatal("the stalled discovery is not registered") + } + job.cancel() // Past its work deadline; the worker keeps running. + refused("draining request", checkoutDiscoveryDraining) +} + +// A proven discovery that no caller was authorized to use runs no Git work; +// it only waits for an authorized caller until its deadline. Requests denied +// in a family therefore never hold that family's discovery slots. +func TestObserveCheckoutPathDeniedDiscoveriesLeaveFamilyCapacity(t *testing.T) { + lc, _, primary, familyID, _, _ := newCheckoutObservationFixture(t) + worktree := filepath.Join(filepath.Dir(primary), "own") + builderGit(t, primary, "worktree", "add", "-b", "own", worktree) + denied := errors.New("outside authorized workspace") + for i := range checkoutObservationFamilyCapacity { + dir := filepath.Join(primary, fmt.Sprintf("denied-%d", i)) + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if _, found, err := observeCheckoutUntilSettled(t, lc, dir, func(string) error { return denied }); found || !errors.Is(err, denied) { + t.Fatalf("denied observation %d: found=%v err=%v", i, found, err) + } + } + lc.observationMu.Lock() + waiting := 0 + for _, job := range lc.observationJobs { + if job.familyID == familyID && job.ctx.Err() == nil && !channelClosed(job.authorized) { + waiting++ + } + } + lc.observationMu.Unlock() + if waiting != checkoutObservationFamilyCapacity { + t.Fatalf("%d denied discoveries still wait for authorization, want %d", waiting, checkoutObservationFamilyCapacity) + } + if _, _, err := lc.ObserveCheckoutPath(t.Context(), worktree); checkoutDiscoveryReason(err) == checkoutDiscoveryFamilyCapacity { + t.Fatalf("denied discoveries held the family's slots: %v", err) + } + checkout, found, err := observeCheckoutUntilSettled(t, lc, worktree) + if err != nil || !found || checkout.FamilyID != familyID || !pathkey.EqualPaths(checkout.RootPath, pathkey.CanonicalExistingRoot(worktree)) { + t.Fatalf("the family's own new worktree was not admitted: %+v found=%v err=%v", checkout, found, err) + } +} + func TestObserveCheckoutPathCancellationAndClosingDoNotAllocate(t *testing.T) { lc, catalog, primary, familyID, _, closeFixture := newCheckoutObservationFixture(t) worktree := filepath.Join(filepath.Dir(primary), "canceled") From f8b1c2513e91cbe821c2dc1bad2864ce1d20c68f Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 15:07:34 +0100 Subject: [PATCH 32/60] feat(load): bind edit probes to pinned repository routes --- bench/load/client.go | 3 +- bench/load/client_test.go | 2 + bench/load/edit_probe_absence.go | 6 +- bench/load/edit_probe_plan.go | 145 +++++++-- bench/load/edit_probe_route_readback_test.go | 211 +++++++++++++ bench/load/edit_probe_routes.go | 74 +++++ bench/load/edit_probe_routes_test.go | 297 +++++++++++++++++++ bench/load/edit_probe_run.go | 153 +++++++++- bench/load/run.go | 16 +- bench/load/scenario.go | 3 + bench/load/testdata/paired-v1.schema.json | 109 +++++++ bench/load/testdata/report-v1.schema.json | 109 +++++++ 12 files changed, 1088 insertions(+), 40 deletions(-) create mode 100644 bench/load/edit_probe_route_readback_test.go create mode 100644 bench/load/edit_probe_routes.go create mode 100644 bench/load/edit_probe_routes_test.go diff --git a/bench/load/client.go b/bench/load/client.go index 66f5371a5..0e5b45046 100644 --- a/bench/load/client.go +++ b/bench/load/client.go @@ -64,6 +64,7 @@ type socketClient struct { notifies []notificationObservation notifyDropped int version string + cwd string // immutable checkout supplied by the successful handshake } // Per-call values returned to its caller, never a shared last-ID cache. Times @@ -142,7 +143,7 @@ func openMCP(ctx context.Context, socket, cwd string) (*socketClient, error) { if err != nil { return nil, err } - c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, writeGate: make(chan struct{}, 1)} + c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: cwd, writeGate: make(chan struct{}, 1)} go c.readLoop() if _, err := c.call(ctx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "gortex-load", "version": "1"}}); err != nil { _ = c.Close() diff --git a/bench/load/client_test.go b/bench/load/client_test.go index 2f3c9c701..5aea18254 100644 --- a/bench/load/client_test.go +++ b/bench/load/client_test.go @@ -22,6 +22,7 @@ type stubFrame struct { Params map[string]any `json:"params"` Kind string `json:"kind"` Connection int `json:"-"` + Cwd string `json:"-"` } // A real newline-framed daemon stub exercises the public CLI's transport; @@ -78,6 +79,7 @@ func startStub(t *testing.T, handle func(stubFrame) (any, *rpcError)) string { return } frame.Connection = connectionID + frame.Cwd, _ = handshake["cwd"].(string) if frame.Kind != "" { result, _ := handle(frame) encode(map[string]any{"ok": true, "result": result}) diff --git a/bench/load/edit_probe_absence.go b/bench/load/edit_probe_absence.go index c1de43bad..946a992da 100644 --- a/bench/load/edit_probe_absence.go +++ b/bench/load/edit_probe_absence.go @@ -63,7 +63,11 @@ func (c *editProbeCaller) expectedAbsence(data json.RawMessage, identity wireCal if !proven || editStarted.IsZero() || identity.WriteStarted.IsZero() || identity.WriteStarted.Before(editStarted) { return nil } - scoped := c.runtime.p.Repos[0].Prefix + "/" + logical + repo, err := c.runtime.repoFor(c.state.step) + if err != nil { + return err + } + scoped := repo.Prefix + "/" + logical if message, matched := editProbeExactNotFound(data, scoped); matched { return &editProbeExpectedAbsence{symbol: scoped, message: message} } diff --git a/bench/load/edit_probe_plan.go b/bench/load/edit_probe_plan.go index d10cd7249..84830db96 100644 --- a/bench/load/edit_probe_plan.go +++ b/bench/load/edit_probe_plan.go @@ -28,6 +28,7 @@ type editProbeStep struct { ActivateNS int64 `json:"activate_ns"` DueNS int64 `json:"due_ns"` Editor int `json:"editor,omitempty"` + Route string `json:"route,omitempty"` Marker string `json:"marker"` MutationID string `json:"mutation_id"` Request request `json:"request"` @@ -46,28 +47,41 @@ type editProbeStep struct { File string `json:"file,omitempty"` } +type editProbeRoute struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` +} + type editProbePlan struct { - Version int `json:"version"` - Arrival string `json:"arrival"` - MeanNS int64 `json:"mean_interarrival_ns"` - ProbePeriodNS int64 `json:"probe_period_ns"` - Count int `json:"count"` - Editors int `json:"editors,omitempty"` - Readers int `json:"readers,omitempty"` - NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` - Seed int64 `json:"seed"` - Kind string `json:"kind"` - Tool string `json:"tool"` - Language string `json:"language"` - Mode string `json:"mode"` - Sender string `json:"sender,omitempty"` - Repo string `json:"repo"` - Commit string `json:"commit"` - File string `json:"file"` - Callee string `json:"callee"` - SessionModel string `json:"session_model"` - NotSelected []string `json:"not_selected_not_tested"` - Steps []editProbeStep `json:"steps"` + Version int `json:"version"` + Arrival string `json:"arrival"` + MeanNS int64 `json:"mean_interarrival_ns"` + ProbePeriodNS int64 `json:"probe_period_ns"` + Count int `json:"count"` + Editors int `json:"editors,omitempty"` + Readers int `json:"readers,omitempty"` + NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` + Seed int64 `json:"seed"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` + Sender string `json:"sender,omitempty"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + SessionModel string `json:"session_model"` + NotSelected []string `json:"not_selected_not_tested"` + Steps []editProbeStep `json:"steps"` + Routes []editProbeRoute `json:"routes,omitempty"` } type editProbeArrival struct { @@ -127,6 +141,95 @@ func planEditProbe(c Corpus, o runOptions) (editProbePlan, error) { return p, nil } +// Routed plans reuse each corpus's existing finite plan. Every corpus sees +// the same global ordinals/arrivals; each fixed editor takes only its own +// corpus's request at that ordinal, so no secondary stream is reseeded. +func planEditProbeRoutes(corpora []Corpus, o runOptions) (editProbePlan, error) { + if len(corpora) == 0 { + return editProbePlan{}, errors.New("edit-and-probe requires a pinned corpus") + } + if o.EditRepos < 0 { + return editProbePlan{}, errors.New("invalid edit repository count") + } + if o.EditRepos <= 1 { + return planEditProbe(corpora[0], o) + } + if o.EditRepos != 6 || len(corpora) != 6 || o.Editors < 6 || o.EditCount != 100 || o.Agents != 0 || o.Worktrees != 0 { + return editProbePlan{}, errors.New("six-repository planning requires exactly six pinned corpora, at least six fixed editors, 100 edits, no pure readers and no linked worktrees") + } + plans := make([]editProbePlan, len(corpora)) + routes := make([]editProbeRoute, len(corpora)) + labels, sources := map[string]bool{}, map[string]bool{} + for i, c := range corpora { + if c.Label == "" || labels[c.Label] || sources[c.Source] { + return editProbePlan{}, errors.New("six-repository routes require distinct corpus labels and source repositories") + } + labels[c.Label], sources[c.Source] = true, true + p, err := planEditProbe(c, o) + if err != nil { + return editProbePlan{}, fmt.Errorf("route %s: %w", c.Label, err) + } + if p.Kind != "add_intrafile_call" || p.Tool != "edit.file" || p.Language != "go" || p.Mode != "json" || p.Sender != "" { + return editProbePlan{}, errors.New("six-repository planning currently supports ordinary Go add_intrafile_call/edit.file JSON fixtures only; native, lifecycle, move and other output cells are unsupported") + } + plans[i] = p + routes[i] = editProbeRoute{ID: c.Label, Repo: c.Label, Commit: c.Commit, File: p.File, Callee: p.Callee, Kind: p.Kind, Tool: p.Tool, Language: p.Language, Mode: p.Mode} + } + out := plans[0] + out.Routes = routes + out.Steps = append([]editProbeStep(nil), out.Steps...) + for i, step := range out.Steps { + route := step.Editor % len(plans) + out.Steps[i] = plans[route].Steps[i] + out.Steps[i].Route = routes[route].ID + } + out.NotSelected = nil + for _, limitation := range plans[0].NotSelected { + if limitation != "several editors share one repository; 6-repository and linked-worktree routing are not implemented" { + out.NotSelected = append(out.NotSelected, limitation) + } + } + out.NotSelected = append(out.NotSelected, "multi-repository execution remains refused until routed oracle and restore ownership are validated") + return out, nil +} + +func editProbeRouteOperations(plan editProbePlan, corpora []Corpus) ([]request, error) { + if len(corpora) == 0 { + return nil, errors.New("edit-and-probe operations require a pinned corpus") + } + if len(plan.Routes) == 0 { + for _, step := range plan.Steps { + if step.Route != "" { + return nil, errors.New("edit route has no pinned route metadata") + } + } + return editProbeOperations(plan, corpora[0]), nil + } + if len(plan.Routes) != len(corpora) { + return nil, errors.New("edit route corpus denominator differs from pinned metadata") + } + byRoute := make(map[string]Corpus, len(corpora)) + for i, c := range corpora { + route := plan.Routes[i] + if _, duplicate := byRoute[route.ID]; duplicate || route.ID != c.Label || route.Repo != c.Label || route.Commit != c.Commit || route.File != c.File { + return nil, errors.New("edit route metadata differs from its pinned corpus") + } + byRoute[route.ID] = c + } + var operations []request + for _, step := range plan.Steps { + c, found := byRoute[step.Route] + if !found || step.Route == "" || step.Editor < 0 || step.Editor >= plan.Editors || step.Route != plan.Routes[step.Editor%len(plan.Routes)].ID { + return nil, errors.New("offered edit has an unknown or foreign repository route") + } + operations = append(operations, step.Request) + for _, oracle := range editProbeOracles(c, step) { + operations = append(operations, oracle.call) + } + } + return operations, nil +} + func makeEditProbeTrafficPlan(c Corpus, seed int64, count, editors int) (editProbePlan, error) { if err := validateCorpus(c); err != nil { return editProbePlan{}, err diff --git a/bench/load/edit_probe_route_readback_test.go b/bench/load/edit_probe_route_readback_test.go new file mode 100644 index 000000000..b63ca0037 --- /dev/null +++ b/bench/load/edit_probe_route_readback_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// Two selected ordinals are a unit diagnostic, never a hundred-edit campaign. +// The stub performs actual owned writes and binds each receipt to its cwd. +func TestEditProbeRoutedReadbacksAndRestoreKeepEachCheckoutOriginal(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + p := &privateRun{Root: t.TempDir()} + originals := make([]string, 2) + steps := make([]editProbeStep, 2) + replies := make(map[string]func(stubFrame) (any, *rpcError)) + for i := range 2 { + c := editProbeCorpus() + c.Label = fmt.Sprintf("checkout-%d", i) + callee := fmt.Sprintf("Example%d", i) + c.Symbol = c.File + "::" + callee + c.EditAnchor = "func Caller() int { return " + callee + "() }" + c.EditTemplate = "\nfunc {{marker}}() int { return " + callee + "() }\n" + root := filepath.Join(p.Root, c.Label) + if err := os.Mkdir(root, 0700); err != nil { + t.Fatal(err) + } + originals[i] = fmt.Sprintf("package sample\n// sentinel-%d\nfunc %s() int { return %d }\n%s\n", i, callee, i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(originals[i]), 0600); err != nil { + t.Fatal(err) + } + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: "fixture", Corpus: c}) + plan, err := makeEditProbeTrafficPlan(c, 42, 100, 2) + if err != nil { + t.Fatal(err) + } + for _, step := range plan.Steps { + if step.Editor == i { + steps[i] = step + break + } + } + replies[root] = editProbeDiskStub(t, c, root) + } + var mu sync.Mutex + receipts := make(map[string]string) + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + repo := -1 + for i, r := range p.Repos { + if frame.Cwd == r.Path { + repo = i + break + } + } + if repo < 0 { + return nil, &rpcError{Code: -32000, Message: "unknown checkout"} + } + args, _ := frame.Params["arguments"].(map[string]any) + options, _ := args["options"].(map[string]any) + if frame.Params["name"] == "edit" && args["operation"] == "file" { + target, _ := args["target"].(map[string]any) + if target["file"] != "fixture/"+p.Repos[repo].Corpus.File { + return nil, &rpcError{Code: -32000, Message: "foreign file"} + } + data, err := os.ReadFile(p.Repos[repo].File) + if err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "read failed"} + } + match, _ := args["match"].(string) + replacement, _ := args["replacement"].(string) + if match == "" || strings.Count(string(data), match) != 1 { + return nil, &rpcError{Code: -32000, Message: "exact match missing"} + } + if err := os.WriteFile(p.Repos[repo].File, []byte(strings.Replace(string(data), match, replacement, 1)), 0600); err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "write failed"} + } + mutation, _ := options["mutation_id"].(string) + receipt := fmt.Sprintf("checkout-%d-%s", repo, mutation) + mu.Lock() + receipts[frame.Cwd+"|"+receipt] = mutation + mu.Unlock() + return stubContent(map[string]any{"mutation_receipt": receipt}), nil + } + if frame.Params["name"] == "change" && args["operation"] == "receipt" { + receipt, _ := options["receipt"].(string) + mu.Lock() + mutation := receipts[frame.Cwd+"|"+receipt] + mu.Unlock() + if mutation == "" { + return nil, &rpcError{Code: -32000, Message: "receipt belongs to another checkout"} + } + return stubContent(map[string]any{"graph_status_terminal": true, "graph_status": "fresh", "mutation_id": mutation}), nil + } + return replies[frame.Cwd](frame) + }) + phase := &Phase{Metrics: map[string]Metric{}} + var clients []*socketClient + t.Cleanup(func() { closeClients(clients) }) + for _, repo := range p.Repos { + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + clients = append(clients, client) + } + run, probers, err := prepareEditProbe(ctx, runOptions{Seed: 42, EditCount: 100, Editors: 2, Deadline: time.Second, editProbeBaseline: true}, p, nil, clients, phase) + clients = append(clients, probers...) + if run != nil { + t.Cleanup(func() { _ = run.close() }) + } + if err != nil { + t.Fatal(err) + } + run.origin = time.Now() + selected := make([]*editProbeState, 2) + for i, step := range steps { + state := run.states[step.Ordinal] + // Corpus-specific requests are independently pinned by the planner + // test. This diagnostic selects one real ordinal per checkout. + state.step, state.record = step, newEditProbeRecord(p.Repos[i].Corpus, step) + state.record.Errors = nil + selected[i] = state + } + phase.EditProbe.Records = nil + for i, state := range selected { + raw, _, err := run.perform(ctx, state, run.editorFor(state.step), state.step.Request, "edit", "workload", 0) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil { + t.Fatal(err) + } + receipt, _ := body["mutation_receipt"].(string) + status := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": receipt}}} + raw, _, err = run.perform(ctx, state, state.client, status, "status", "probe", 1) + if err != nil { + t.Fatal(err) + } + body, err = decodeEditProbeResult(raw) + if err != nil || body["mutation_id"] != state.step.MutationID || body["graph_status_terminal"] != true { + t.Fatalf("own receipt missing: %v %v", body, err) + } + state.record.Receipt, state.record.GraphTerminal = receipt, true + _, correct := run.immediateReadbacks(ctx, state, state.step, []string{"file", "source"}, nil) + if !correct { + t.Errorf("checkout%d same-session readbacks used another fixture/callee: %+v", i, state.record.Readbacks) + } + foreign := selected[1-i].client + if _, _, err := run.perform(ctx, state, foreign, status, "status", "probe", 2); err == nil { + t.Error("foreign checkout session borrowed a receipt") + } + phase.EditProbe.Records = append(phase.EditProbe.Records, state.record) + } + foreignReceipt := request{Tool: "change", Operation: "receipt", Arguments: map[string]any{"operation": "receipt", "options": map[string]any{"receipt": selected[1].record.Receipt}}} + if _, _, err := run.perform(ctx, selected[0], selected[0].client, foreignReceipt, "status", "probe", 3); err == nil { + t.Fatal("same-prefix foreign receipt borrowed another checkout's mutation") + } + // A completed sibling cannot justify rewriting an unknown/pending route. + beforeRestore := make([][]byte, len(p.Repos)) + for i, repo := range p.Repos { + beforeRestore[i], err = os.ReadFile(repo.File) + if err != nil { + t.Fatal(err) + } + } + phase.EditProbe.Records[1].GraphTerminal = false + if err := run.restore(ctx); err == nil { + t.Fatal("pending route permitted unsafe cleanup") + } + for i, repo := range p.Repos { + actual, err := os.ReadFile(repo.File) + if err != nil || string(actual) != string(beforeRestore[i]) { + t.Fatalf("pending cleanup changed checkout%d: %v", i, err) + } + } + phase.EditProbe.Records[1].GraphTerminal = true + if err := run.restore(ctx); err != nil { + t.Fatal(err) + } + var restores []struct { + Route string `json:"route"` + Editor int `json:"editor"` + } + if err := json.Unmarshal(phase.EditProbe.Restore, &restores); err != nil || len(restores) != 2 { + t.Fatalf("restore evidence lost a checkout: %s %v", phase.EditProbe.Restore, err) + } + for i, row := range restores { + if row.Route != p.Repos[i].Corpus.Label || row.Editor != i { + t.Fatalf("restore borrowed another checkout's owner: %+v", row) + } + } + for i, repo := range p.Repos { + actual, err := os.ReadFile(repo.File) + if err != nil { + t.Fatal(err) + } + if string(actual) != originals[i] { + t.Errorf("checkout%d original not restored through its own view: %q", i, actual) + } + } +} diff --git a/bench/load/edit_probe_routes.go b/bench/load/edit_probe_routes.go new file mode 100644 index 000000000..f6bbd9388 --- /dev/null +++ b/bench/load/edit_probe_routes.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" +) + +// Multi-checkout cleanup never combines common-prefix paths into editor 0's +// view. Each admitted original has its own editor session and transaction. +func (r *editProbeRuntime) restoreRoutes(ctx context.Context) (resultErr error) { + if r.plan.Kind != "add_intrafile_call" || r.plan.Tool != "edit.file" || r.plan.Mode != "json" || r.plan.Sender != "" { + return errors.New("routed restore supports ordinary add_intrafile_call/edit.file JSON only") + } + type restoredRoute struct { + Route string `json:"route"` + Editor int `json:"editor"` + File string `json:"file"` + Response json.RawMessage `json:"response"` + Error string `json:"error,omitempty"` + } + var rows []restoredRoute + started := time.Now() + defer func() { + r.phase.EditProbe.RestoreMS = measured(float64(time.Since(started))/float64(time.Millisecond), "ms") + data, err := json.Marshal(rows) + r.phase.EditProbe.Restore = data + resultErr = errors.Join(resultErr, err) + }() + for ordinal, repo := range r.p.Repos { + original, found := r.originals[repo.Path] + if !found { + return errors.New("routed restore has no admitted original for checkout") + } + editor := -1 + for i, client := range r.editors { + if client != nil && filepath.Clean(client.cwd) == filepath.Clean(repo.Path) { + editor = i + break + } + } + if editor < 0 { + return errors.New("routed restore has no editor for checkout") + } + file := filepath.Join(repo.Path, filepath.FromSlash(repo.Corpus.File)) + current, err := os.ReadFile(file) + if err != nil { + return err + } + changes := []any{map[string]any{"op": "edit_file", "path": repo.Prefix + "/" + repo.Corpus.File, "old_string": string(current), "new_string": string(original)}} + req := request{Tool: "edit", Operation: "batch", Arguments: map[string]any{"operation": "batch", "changes": changes, "options": map[string]any{"transaction_id": fmt.Sprintf("load-restore-%016x-route-%d", uint64(r.options.Seed), ordinal)}, "output": map[string]any{"format": "json", "max_bytes": 0}}} + data, _, err := callEditProbeTool(ctx, r.editors[editor], req, req.Arguments, "cleanup", ordinal, len(r.states), 0, r.options.Deadline, editProbeRetrySleep(r.options.Seed, len(r.states), ordinal)) + row := restoredRoute{Route: repo.Corpus.Label, Editor: editor, File: repo.Corpus.File, Response: data} + if err == nil { + var restored []byte + restored, err = os.ReadFile(file) + if err == nil && string(restored) != string(original) { + err = errors.New("public routed restore did not restore its checkout's original bytes") + } + } + if err != nil { + row.Error = err.Error() + } + rows = append(rows, row) + if err != nil { + return err + } + } + return nil +} diff --git a/bench/load/edit_probe_routes_test.go b/bench/load/edit_probe_routes_test.go new file mode 100644 index 000000000..5351f56ff --- /dev/null +++ b/bench/load/edit_probe_routes_test.go @@ -0,0 +1,297 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestEditProbeSixRepositoryRoutesAreExplicitAndAffectPlanIdentity(t *testing.T) { + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", "6", "--worktrees", "0", "--edit-count", "100", "--seed", "42", "--edit-repos", "6"} + o, err := parseRun(args) + if err != nil { + t.Fatalf("explicit six-repository edit cell refused: %v", err) + } + corpora := editProbeRoutingCorpora() + report, err := newRunReport(o, corpora, strings.Repeat("a", 40), Host{}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(report.Workload.EditProbe) + if err != nil { + t.Fatal(err) + } + var plan struct { + Routes []struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + } `json:"routes"` + Steps []struct { + Ordinal int `json:"ordinal"` + Editor int `json:"editor"` + Route string `json:"route"` + DueNS int64 `json:"due_ns"` + Marker string `json:"marker"` + MutationID string `json:"mutation_id"` + Request request `json:"request"` + } `json:"steps"` + } + if err := json.Unmarshal(data, &plan); err != nil { + t.Fatal(err) + } + if len(plan.Routes) != 6 || len(plan.Steps) != 100 { + t.Fatalf("selected route/offer denominator lost: routes=%d steps=%d", len(plan.Routes), len(plan.Steps)) + } + for i, route := range plan.Routes { + c := corpora[i] + if route.ID != c.Label || route.Repo != c.Label || route.Commit != c.Commit || route.File != c.File || route.Callee != strings.TrimPrefix(c.Symbol, c.File+"::") { + t.Fatalf("route%d lost its pinned fixture identity: %+v", i, route) + } + } + original, err := makeEditProbeTrafficPlan(corpora[0], 42, 100, 6) + if err != nil { + t.Fatal(err) + } + seenRoutes, seenMutations := map[string]bool{}, map[string]bool{} + for ordinal, step := range plan.Steps { + if step.Editor < 0 || step.Editor >= 6 { + t.Fatalf("invalid fixed editor: %+v", step) + } + c := corpora[step.Editor] + replacement, _ := step.Request.Arguments["replacement"].(string) + if step.Ordinal != ordinal || step.Route != c.Label || step.DueNS != original.Steps[ordinal].DueNS || !strings.Contains(replacement, strings.TrimPrefix(c.Symbol, c.File+"::")+"()") || step.Marker == "" || step.MutationID == "" || seenMutations[step.MutationID] { + t.Fatalf("offer was rerouted, reshaped or lost ownership: %+v", step) + } + seenRoutes[step.Route], seenMutations[step.MutationID] = true, true + } + if len(seenRoutes) != 6 { + t.Fatal("selected repository has no offered edits", seenRoutes) + } + // A secondary repository's pin belongs to the offered workload identity; + // it cannot be ignored merely because the primary fixture is unchanged. + changed := append([]Corpus(nil), corpora...) + changed[3].Commit = strings.Repeat("b", 40) + other, err := newRunReport(o, changed, report.GitSHA, Host{}, t.TempDir()) + if err != nil || other.RequestPlan == report.RequestPlan { + t.Fatalf("secondary route pin absent from plan identity: %v", err) + } +} + +func TestEditProbeSelectedCheckoutsKeepProberSessionsOnTheirEditorRoute(t *testing.T) { + for _, sharedPrefix := range []bool{false, true} { + name := "distinct_repositories" + if sharedPrefix { + name = "shared_prefix_checkouts" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + t.Cleanup(cancel) + p := &privateRun{Root: t.TempDir()} + for i := 0; i < 2; i++ { + c := editProbeCorpus() + c.Label = fmt.Sprintf("route-%d", i) + root := filepath.Join(p.Root, "repos", fmt.Sprintf("route-%d", i)) + if err := os.MkdirAll(root, 0o700); err != nil { + t.Fatal(err) + } + content := fmt.Sprintf("package sample\n// route%d\nfunc Example() int { return %d }\n%s\n", i, i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + prefix := c.Label + if sharedPrefix { + prefix, c.Label = "fixture", "fixture" + } + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: prefix, Corpus: c}) + } + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + args, _ := frame.Params["arguments"].(map[string]any) + target, _ := args["target"].(map[string]any) + return stubContent(map[string]any{"cwd": frame.Cwd, "symbol": target["symbol"]}), nil + }) + var editors []*socketClient + for _, repo := range p.Repos { + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + editors = append(editors, client) + } + t.Cleanup(func() { closeClients(editors) }) + phase := &Phase{Metrics: map[string]Metric{}} + options := runOptions{Seed: 42, EditCount: 100, Editors: 2, Deadline: time.Second} + if sharedPrefix { + options.Worktrees = 1 // one linked checkout in addition to primary + } + runtime, probers, err := prepareEditProbe(ctx, options, p, nil, editors, phase) + if runtime != nil { + t.Cleanup(func() { _ = runtime.close() }) // prepared edits intentionally never run + } + t.Cleanup(func() { closeClients(probers) }) + if err != nil || runtime == nil || len(probers) != 100 { + t.Fatalf("selected route setup unavailable: probers=%d err=%v", len(probers), err) + } + cwd := func(client *socketClient) string { + t.Helper() + raw, err := client.call(ctx, "tools/call", map[string]any{"name": "read", "arguments": map[string]any{"operation": "source"}}) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil { + t.Fatal(err) + } + value, _ := body["cwd"].(string) + return value + } + for _, state := range runtime.states { + want := p.Repos[state.step.Editor].Path + if actual := cwd(editors[state.step.Editor]); actual != want { + t.Fatalf("editor control is on the wrong checkout: got=%s want=%s", actual, want) + } + if actual := cwd(state.client); actual != want { + t.Fatalf("edit%d prober fell back to the primary checkout: editor=%d got=%s want=%s", state.step.Ordinal, state.step.Editor, actual, want) + } + repo := p.Repos[state.step.Editor] + req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": repo.Corpus.Symbol}}} + raw, _, err := runtime.perform(ctx, state, state.client, req, "route_source", "probe", 0) + if err != nil { + t.Fatal(err) + } + body, err := decodeEditProbeResult(raw) + if err != nil || body["cwd"] != want || body["symbol"] != repo.Prefix+"/"+repo.Corpus.Symbol { + t.Fatalf("same-view prober was dispatched with another repository's target: edit%d body=%v err=%v", state.step.Ordinal, body, err) + } + } + }) + } +} + +func TestEditProbeUnknownEditorCheckoutNeverFallsBackToPrimary(t *testing.T) { + run := &editProbeRuntime{ + p: &privateRun{Repos: []privateRepo{{Path: "/owned/primary", Prefix: "shared"}, {Path: "/owned/linked", Prefix: "shared"}}}, + editors: []*socketClient{{cwd: "/owned/primary"}, {cwd: "/owned/linked"}, {cwd: "/foreign"}, nil}, + } + for _, editor := range []int{-1, 2, 3, 4} { + if repo, err := run.repoFor(editProbeStep{Editor: editor}); err == nil { + t.Fatalf("unknown editor %d inherited route %+v", editor, repo) + } + } + for editor := range 2 { + repo, err := run.repoFor(editProbeStep{Editor: editor}) + if err != nil || repo.Path != run.editors[editor].cwd { + t.Fatalf("registered shared-prefix checkout lost: editor=%d repo=%+v err=%v", editor, repo, err) + } + } + state := &editProbeState{step: editProbeStep{Editor: 1}} + if _, _, err := run.perform(t.Context(), state, run.editors[0], request{}, "status", "probe", 0); err == nil { + t.Fatal("foreign primary session accepted for linked-checkout evidence") + } + for i, label := range []string{"primary", "linked"} { + run.p.Repos[i].Corpus = Corpus{Label: label, Commit: fmt.Sprintf("%040x", i+1), File: "main.go"} + c := run.p.Repos[i].Corpus + run.plan.Routes = append(run.plan.Routes, editProbeRoute{ID: label, Repo: label, Commit: c.Commit, File: c.File}) + } + for _, route := range []string{"", "unknown", "primary"} { + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: route}); err == nil { + t.Fatalf("invalid offered route %q inherited a checkout", route) + } + } + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: "linked"}); err != nil { + t.Fatal(err) + } + run.plan.Routes[1].Commit = strings.Repeat("f", 40) + if _, err := run.repoFor(editProbeStep{Editor: 1, Route: "linked"}); err == nil { + t.Fatal("foreign pin inherited registered checkout") + } +} + +func editProbeRoutingCorpora() []Corpus { + corpora := make([]Corpus, 6) + for i := range corpora { + c := editProbeCorpus() + c.Label = fmt.Sprintf("route-%d", i) + c.Source = fmt.Sprintf("/private/route-%d", i) + c.Commit = fmt.Sprintf("%040x", i+1) + callee := fmt.Sprintf("Example%d", i) + c.Symbol = c.File + "::" + callee + c.EditAnchor = "func Caller() int { return " + callee + "() }" + c.EditTemplate = "\nfunc {{marker}}() int { return " + callee + "() }\n" + corpora[i] = c + } + return corpora +} + +func TestEditProbeRoutedPlansRejectMissingOrForeignOwnership(t *testing.T) { + o := runOptions{Scenario: editProbeScenario, Seed: 42, EditCount: 100, Editors: 6, EditRepos: 6} + for _, name := range []string{"empty", "too_few", "extra", "duplicate_label", "duplicate_source", "too_few_editors", "readers", "worktrees", "native", "cli"} { + t.Run(name, func(t *testing.T) { + corpora, options := editProbeRoutingCorpora(), o + switch name { + case "empty": + corpora = nil + case "too_few": + corpora = corpora[:5] + case "extra": + corpora = append(corpora, corpora[0]) + case "duplicate_label": + corpora[1].Label = corpora[0].Label + case "duplicate_source": + corpora[1].Source = corpora[0].Source + case "too_few_editors": + options.Editors = 5 + case "readers": + options.Agents = 1 + case "worktrees": + options.Worktrees = 1 + case "native": + corpora[1].EditSender = editProbeShellAfterTool + case "cli": + corpora[1].EditOutput = "cli" + } + if _, err := planEditProbeRoutes(corpora, options); err == nil { + t.Fatal("invalid route selection accepted") + } + }) + } + for _, name := range []string{"missing", "unknown", "foreign_editor", "foreign_pin", "missing_metadata"} { + t.Run(name, func(t *testing.T) { + corpora := editProbeRoutingCorpora() + plan, err := planEditProbeRoutes(corpora, o) + if err != nil { + t.Fatal(err) + } + switch name { + case "missing": + plan.Steps[0].Route = "" + case "unknown": + plan.Steps[0].Route = "unknown" + case "foreign_editor": + plan.Steps[0].Route = plan.Routes[(plan.Steps[0].Editor+1)%len(plan.Routes)].ID + case "foreign_pin": + corpora[1].Commit = strings.Repeat("f", 40) + case "missing_metadata": + plan.Routes = nil + } + if _, err := editProbeRouteOperations(plan, corpora); err == nil { + t.Fatal("missing or foreign route inherited primary operations") + } + }) + } +} + +func TestEditProbeRoutedPlanDoesNotEnableAnUnvalidatedRuntime(t *testing.T) { + o := runOptions{Scenario: editProbeScenario, EditRepos: 6} + phase, _, artifacts, _, _, err := runPhase(t.Context(), o, editProbeScenario, editProbeRoutingCorpora(), "", t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "multi-repository edit-and-probe execution is unsupported") || artifacts["private_root"] != "" || phase.editProbeRuntime != nil { + t.Fatalf("planning metadata silently enabled a partial runtime: err=%v artifacts=%v", err, artifacts) + } +} diff --git a/bench/load/edit_probe_run.go b/bench/load/edit_probe_run.go index c6f9596f8..a9e1e9732 100644 --- a/bench/load/edit_probe_run.go +++ b/bench/load/edit_probe_run.go @@ -31,6 +31,7 @@ type editProbeReadback struct { type editProbeRecord struct { Ordinal int `json:"ordinal"` Editor int `json:"editor,omitempty"` + Route string `json:"route,omitempty"` Marker string `json:"marker"` MutationID string `json:"mutation_id"` PlannedDueNS int64 `json:"planned_due_ns"` @@ -123,6 +124,7 @@ type editProbeRuntime struct { modeCounts map[string]editProbeModeCounts states []*editProbeState original []byte + originals map[string][]byte // routed checkout path -> independently admitted original bytes moveOriginal []byte origin time.Time mu sync.Mutex // Phase.Attempts, reads and mode counts only; request lifetimes are never serialized @@ -133,7 +135,7 @@ type editProbeRuntime struct { } func newEditProbeRecord(c Corpus, step editProbeStep) editProbeRecord { - record := editProbeRecord{Ordinal: step.Ordinal, Editor: step.Editor, Marker: step.Marker, MutationID: step.MutationID, PlannedDueNS: step.DueNS, Outcome: "phase_censored", Ack: unavailable("ms", "edit not sent"), PhasesMS: editProbePhases(nil, "no receipt status observed"), Readback: unavailable("ms", "edit not acknowledged"), Derived: unavailable("ms", "no receipt status observed"), FirstCorrect: make(map[string]Metric), Accounting: editProbeAccounting{State: "not_observed"}, Errors: []string{"scenario never reached edit dispatch"}} + record := editProbeRecord{Ordinal: step.Ordinal, Editor: step.Editor, Route: step.Route, Marker: step.Marker, MutationID: step.MutationID, PlannedDueNS: step.DueNS, Outcome: "phase_censored", Ack: unavailable("ms", "edit not sent"), PhasesMS: editProbePhases(nil, "no receipt status observed"), Readback: unavailable("ms", "edit not acknowledged"), Derived: unavailable("ms", "no receipt status observed"), FirstCorrect: make(map[string]Metric), Accounting: editProbeAccounting{State: "not_observed"}, Errors: []string{"scenario never reached edit dispatch"}} for _, oracle := range editProbeOracles(c, step) { record.FirstCorrect[oracle.name] = unavailable("ms", "not observed") } @@ -185,14 +187,77 @@ func newEditProbeReport(plan editProbePlan, c Corpus, required bool) *editProbeR return out } +// The editor's successful handshake owns its checkout. Prefixes are shared by +// linked worktrees and cannot identify a view. A singleton has one unambiguous +// route; with several checkouts an unknown session never inherits the primary. +func (r *editProbeRuntime) repoFor(step editProbeStep) (privateRepo, error) { + if len(r.p.Repos) == 1 { + return r.p.Repos[0], nil // standalone receipt collectors need no editor session + } + if step.Editor < 0 || step.Editor >= len(r.editors) { + return privateRepo{}, fmt.Errorf("edit %d has unknown editor %d", step.Ordinal, step.Editor) + } + editor := r.editors[step.Editor] + if editor == nil { + return privateRepo{}, fmt.Errorf("edit %d has no editor session", step.Ordinal) + } + var offered *editProbeRoute + for i := range r.plan.Routes { + if r.plan.Routes[i].ID == step.Route { + offered = &r.plan.Routes[i] + break + } + } + if len(r.plan.Routes) > 0 && (offered == nil || step.Route == "" || r.plan.Routes[step.Editor%len(r.plan.Routes)].ID != step.Route) || len(r.plan.Routes) == 0 && step.Route != "" { + return privateRepo{}, errors.New("edit has an unknown or foreign offered route") + } + for _, repo := range r.p.Repos { + if editor.cwd != "" && filepath.Clean(editor.cwd) == filepath.Clean(repo.Path) { + if offered != nil && (offered.Repo != repo.Corpus.Label || offered.Commit != repo.Corpus.Commit || offered.File != repo.Corpus.File) { + return privateRepo{}, errors.New("editor checkout differs from its offered route pin") + } + return repo, nil + } + } + return privateRepo{}, fmt.Errorf("edit %d editor checkout is not a registered route", step.Ordinal) +} + +func (r *editProbeRuntime) calleeFor(repo privateRepo) string { + if len(r.p.Repos) == 1 { + return r.plan.Callee + } + selected := repo.Corpus.Symbol + if repo.Corpus.EditKind == "add_crossfile_call" { + selected = repo.Corpus.EditCallee + } + separator := strings.LastIndex(selected, "::") + if separator < 0 { + return "" + } + return selected[separator+2:] +} + func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, editors []*socketClient, phase *Phase) (*editProbeRuntime, []*socketClient, error) { - plan, err := planEditProbe(p.Repos[0].Corpus, o) + corpora := make([]Corpus, len(p.Repos)) + for i, repo := range p.Repos { + corpora[i] = repo.Corpus + } + plan, err := planEditProbeRoutes(corpora, o) if err != nil { return nil, nil, err } + if len(p.Repos) > 1 { + for _, repo := range p.Repos { + c := repo.Corpus + if plan.Kind != "add_intrafile_call" || plan.Tool != "edit.file" || plan.Mode != "json" || plan.Sender != "" || filepath.Ext(c.File) != ".go" || c.EditSender != "" || c.EditKind != "" && c.EditKind != "add_intrafile_call" || c.EditTool != "" && c.EditTool != "edit.file" || c.EditOutput != "" && c.EditOutput != "json" { + return nil, nil, errors.New("routed edit-probe runtime currently supports ordinary Go add_intrafile_call/edit.file JSON fixtures only") + } + } + } if len(editors) != max(1, plan.Editors) || len(readers) != plan.Readers { return nil, nil, fmt.Errorf("edit-and-probe plan has %d fixed editors and %d readers but %d editor and %d reader sessions", max(1, plan.Editors), plan.Readers, len(editors), len(readers)) } + r := &editProbeRuntime{options: o, p: p, phase: phase, plan: plan, editors: editors, readers: readers, requireAccounting: !o.editProbeBaseline} original, err := os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(plan.File))) if err != nil { return nil, nil, err @@ -220,7 +285,29 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, return nil, nil, errors.New("planned marker already exists before edit") } } - r := &editProbeRuntime{options: o, p: p, phase: phase, plan: plan, editors: editors, readers: readers, original: original, requireAccounting: !o.editProbeBaseline} + r.original = original + phase.EditProbe = newEditProbeReport(plan, p.Repos[0].Corpus, r.requireAccounting) + if len(p.Repos) > 1 { + r.originals = make(map[string][]byte, len(p.Repos)) + for _, repo := range p.Repos { + data, readErr := os.ReadFile(filepath.Join(repo.Path, filepath.FromSlash(repo.Corpus.File))) + if readErr != nil { + return r, nil, readErr + } + if err := editProbeWriteAdmission(repo.Corpus, data); err != nil { + return r, nil, err + } + if strings.Count(string(data), repo.Corpus.EditAnchor) != 1 { + return r, nil, errors.New("each routed edit-probe anchor must occur exactly once") + } + for _, step := range plan.Steps { + if strings.Contains(string(data), step.Marker) { + return r, nil, errors.New("planned marker already exists in a routed fixture") + } + } + r.originals[repo.Path] = data + } + } if plan.Tool == "refactor.move" { r.moveOriginal, err = os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(p.Repos[0].Corpus.EditDestination))) if err != nil { @@ -230,7 +317,6 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, return r, nil, errors.New("move destination bytes differ from offered fixture") } } - phase.EditProbe = newEditProbeReport(plan, p.Repos[0].Corpus, r.requireAccounting) if plan.Sender == editProbeRestart { r.restarter = editProbeDaemonRestarter{p: p} } @@ -247,7 +333,11 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, var clients []*socketClient setup := time.Now() for _, step := range plan.Steps { - record := newEditProbeRecord(p.Repos[0].Corpus, step) + repo, err := r.repoFor(step) + if err != nil { + return r, nil, err + } + record := newEditProbeRecord(repo.Corpus, step) record.Outcome = "not_started" record.Errors = nil state := &editProbeState{step: step, record: record} @@ -255,13 +345,17 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, state.record.Evidence = filepath.Join(dir, fmt.Sprintf("edit-%03d.jsonl", step.Ordinal)) } for _, state := range r.states { + repo, err := r.repoFor(state.step) + if err != nil { + return r, clients, err + } sink, err := os.OpenFile(state.record.Evidence, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return r, clients, err } state.sink = sink state.startRaw() - state.client, err = openMCP(ctx, p.Socket, p.Repos[0].Path) + state.client, err = openMCP(ctx, p.Socket, repo.Path) if err != nil { return r, clients, err } @@ -270,11 +364,11 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, // paired delete is sent; it cannot be admitted here. if state.step.BeforeSymbol != "" && state.step.File == "" { req := request{Tool: "read", Operation: "source", Arguments: map[string]any{"operation": "source", "target": map[string]any{"symbol": state.step.BeforeSymbol}, "options": map[string]any{"context_lines": 0}}} - raw, identity, readErr := state.client.callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, p.Repos[0])}) + raw, identity, readErr := state.client.callWithIdentity(ctx, "tools/call", map[string]any{"name": req.Tool, "arguments": dispatchArgs(req, repo)}) if readErr == nil { readErr = checkToolResult(raw) } - proven := readErr == nil && editProbePresenceProof(raw, p.Repos[0].Corpus, state.step.BeforeSymbol) + proven := readErr == nil && editProbePresenceProof(raw, repo.Corpus, state.step.BeforeSymbol) if err := state.emit(map[string]any{"event": "before_presence_admission", "symbol": state.step.BeforeSymbol, "wire_id": identity.ID, "write_started": identity.WriteStarted, "finished": identity.Finished, "payload": raw, "proven": proven}); err != nil { return r, clients, err } @@ -447,6 +541,13 @@ func (r *editProbeRuntime) perform(ctx context.Context, s *editProbeState, clien } func (r *editProbeRuntime) performObserved(ctx context.Context, s *editProbeState, client *socketClient, req request, purpose, role string, stream int) (json.RawMessage, wireCallIdentity, []Attempt, error) { + repo, routeErr := r.repoFor(s.step) + if routeErr != nil { + return nil, wireCallIdentity{}, nil, routeErr + } + if len(r.p.Repos) > 1 && (client == nil || filepath.Clean(client.cwd) != filepath.Clean(repo.Path)) { + return nil, wireCallIdentity{}, nil, errors.New("edit-probe request session belongs to another checkout") + } budget := r.options.Deadline if strings.HasPrefix(purpose, "oracle:") || purpose == "status" { budget = min(budget, 20*time.Second) @@ -455,7 +556,7 @@ func (r *editProbeRuntime) performObserved(ctx context.Context, s *editProbeStat budget = min(budget, time.Until(end)) } caller := &editProbeCaller{runtime: r, state: s, client: client, purpose: purpose, request: req, role: role} - data, attempts, err := callEditProbeTool(ctx, caller, req, dispatchArgs(req, r.p.Repos[0]), role, 0, s.step.Ordinal, stream, budget, editProbeRetrySleep(r.options.Seed, s.step.Ordinal, stream)) + data, attempts, err := callEditProbeTool(ctx, caller, req, dispatchArgs(req, repo), role, 0, s.step.Ordinal, stream, budget, editProbeRetrySleep(r.options.Seed, s.step.Ordinal, stream)) r.mu.Lock() r.phase.Attempts = append(r.phase.Attempts, attempts...) r.mu.Unlock() @@ -486,14 +587,18 @@ func (r *editProbeRuntime) activateProbeState(step editProbeStep, cancel context } func (r *editProbeRuntime) observeProbe(s *editProbeState, oracle editProbeOracle, sent time.Time, data json.RawMessage, identity wireCallIdentity, err error, pending bool) error { + repo, routeErr := r.repoFor(s.step) + if routeErr != nil { + return routeErr + } ended := identity.Finished if !identity.WriteStarted.IsZero() { sent = identity.WriteStarted } body, decodeErr := decodeEditProbeOutput(data, r.probeOutput(oracle), oracle.call.name()) absence, exactRefusal := err.(*editProbeExpectedAbsence) - expectedAbsence := oracle.call.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+oracle.call.ExpectedAbsentSymbol - correct := expectedAbsence || err == nil && decodeErr == nil && editProbeCorrect(oracle.name, body, r.p.Repos[0].Corpus, s.step, r.plan.Callee) + expectedAbsence := oracle.call.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == repo.Prefix+"/"+oracle.call.ExpectedAbsentSymbol + correct := expectedAbsence || err == nil && decodeErr == nil && editProbeCorrect(oracle.name, body, repo.Corpus, s.step, r.calleeFor(repo)) fresh := decodeErr == nil && editProbeClaimsFresh(body) stale, precommit := false, false s.mu.Lock() @@ -535,7 +640,15 @@ func (r *editProbeRuntime) activate(ctx context.Context, step editProbeStep) { } func (r *editProbeRuntime) probe(probeCtx context.Context, cancel context.CancelFunc, s *editProbeState) { - for index, oracle := range editProbeOracles(r.p.Repos[0].Corpus, s.step) { + repo, err := r.repoFor(s.step) + if err != nil { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + cancel() + return + } + for index, oracle := range editProbeOracles(repo.Corpus, s.step) { s.probes.Go(func() { ticker := time.NewTicker(editProbePeriod) defer ticker.Stop() @@ -843,8 +956,15 @@ func (r *editProbeRuntime) finishEditAfterACK(ctx, callCtx context.Context, s *e // Observation-only kinds are recorded but neither gate the readback time nor // fail the record. func (r *editProbeRuntime) immediateReadbacks(callCtx context.Context, s *editProbeState, step editProbeStep, readbackKinds []string, observeOnly map[string]bool) (time.Time, bool) { + repo, err := r.repoFor(step) + if err != nil { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, err.Error()) + s.mu.Unlock() + return time.Time{}, false + } readbackEnd, readbacksCorrect := time.Time{}, true - file := r.plan.File + file := repo.Corpus.File if step.File != "" { file = step.File } @@ -876,12 +996,12 @@ func (r *editProbeRuntime) immediateReadbacks(callCtx context.Context, s *editPr begin := time.Now() raw, identity, readErr := r.perform(callCtx, s, r.editorFor(step), req, "readback:"+kind, "readback", 100+index) payload, decodeErr := decodeEditProbeOutput(raw, editProbeRequestOutput(req), req.name()) - back := editProbeReadback{Kind: kind, Latency: unavailable("ms", "response endpoint unavailable"), Correct: readErr == nil && decodeErr == nil && editProbeCorrect(kind, payload, r.p.Repos[0].Corpus, step, r.plan.Callee)} + back := editProbeReadback{Kind: kind, Latency: unavailable("ms", "response endpoint unavailable"), Correct: readErr == nil && decodeErr == nil && editProbeCorrect(kind, payload, repo.Corpus, step, r.calleeFor(repo))} if !identity.Finished.IsZero() { back.Latency = measured(float64(identity.Finished.Sub(begin))/float64(time.Millisecond), "ms") } absence, exactRefusal := readErr.(*editProbeExpectedAbsence) - if req.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == r.p.Repos[0].Prefix+"/"+req.ExpectedAbsentSymbol { + if req.ExpectedAbsentSymbol != "" && exactRefusal && absence.symbol == repo.Prefix+"/"+req.ExpectedAbsentSymbol { back.Correct = true back.ExpectedOutcome = "symbol_absent" back.Refusal = absence.message @@ -1049,6 +1169,9 @@ func (r *editProbeRuntime) restore(ctx context.Context) error { return errors.New("restore batch unavailable while mutation outcome is unknown/pending") } } + if len(r.p.Repos) > 1 { + return r.restoreRoutes(ctx) + } current, err := os.ReadFile(filepath.Join(r.p.Repos[0].Path, filepath.FromSlash(r.plan.File))) if err != nil { return err diff --git a/bench/load/run.go b/bench/load/run.go index a843aeea7..ed7a677ea 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -39,6 +39,7 @@ type runOptions struct { SweepRepetitions int SweepBudget time.Duration EditCount int + EditRepos int EditProbeRole string editProbeBaseline bool Startup startupOptions @@ -58,6 +59,9 @@ func parseRun(args []string) (runOptions, error) { if f.NArg() != 0 { return o, errors.New("unexpected positional run arguments") } + if o.EditRepos < 1 { + return o, errors.New("--edit-repos must be positive") + } o.editProbeBaseline = o.EditProbeRole == "baseline" return o, errors.Join(validateRunOptions(o), validateMemoryOptions(o)) } @@ -85,11 +89,16 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") bindStartupFlags(f, &o.Startup) + f.IntVar(&o.EditRepos, "edit-repos", 1, "edit-and-probe: one repository (default), or an explicit six-pinned-repository plan") f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } func validateRunOptions(o runOptions) error { + if o.EditRepos < 0 || o.EditRepos > 1 && (o.EditRepos != 6 || o.Scenario != editProbeScenario) { + return errors.New("--edit-repos selects one repository or exactly six for edit-and-probe only") + } + if err := validatePRSmokeDrain(o); err != nil { return err } @@ -263,12 +272,15 @@ func newRunReport(o runOptions, corpora []Corpus, sha string, host Host, outputR report.Workload.Operations = append(report.Workload.Operations, ops...) } if o.Scenario == editProbeScenario { - plan, err := planEditProbe(corpora[0], o) + plan, err := planEditProbeRoutes(corpora, o) if err != nil { return report, err } report.Workload.EditProbe = &plan - report.Workload.Operations = editProbeOperations(plan, corpora[0]) + report.Workload.Operations, err = editProbeRouteOperations(plan, corpora) + if err != nil { + return report, err + } report.Workload.SessionModel = plan.SessionModel } report.Workload.Scenario, report.Workload.Agents, report.Workload.EditAgents, report.Workload.LinkedWorktrees = o.Scenario, o.Agents, o.Editors, o.Worktrees diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 079b1cf3b..d8e00a7f7 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -61,6 +61,9 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp resultErr = errors.New(strings.Join(phase.Errors, "; ")) } }() + if scenario == editProbeScenario && o.EditRepos > 1 { + return phase, nil, artifacts, crashes, "", errors.New("multi-repository edit-and-probe execution is unsupported until routed oracle and restore ownership are validated") + } selected := corpora[:1] if scenario == "many" || scenario == "many-edit" { if len(corpora) < 2 { diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index 56717f6fb..0746cfc62 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -1456,6 +1456,9 @@ } ] }, + "route": { + "type": "string" + }, "stale_claimed_fresh": { "type": "boolean" }, @@ -1807,6 +1810,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -1905,6 +1958,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, @@ -3592,6 +3648,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -3690,6 +3796,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index ac950dfa7..eea3971b7 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -1443,6 +1443,9 @@ } ] }, + "route": { + "type": "string" + }, "stale_claimed_fresh": { "type": "boolean" }, @@ -1794,6 +1797,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -1892,6 +1945,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, @@ -3579,6 +3635,56 @@ "repo": { "type": "string" }, + "routes": { + "items": { + "additionalProperties": false, + "properties": { + "callee": { + "type": "string" + }, + "commit": { + "type": "string" + }, + "file": { + "type": "string" + }, + "id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "language": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "tool": { + "type": "string" + } + }, + "required": [ + "id", + "repo", + "commit", + "file", + "callee", + "kind", + "tool", + "language", + "mode" + ], + "type": "object" + }, + "type": [ + "array", + "null" + ] + }, "seed": { "type": "integer" }, @@ -3677,6 +3783,9 @@ "request_sha256": { "type": "string" }, + "route": { + "type": "string" + }, "sender": { "type": "string" }, From eb0c10532d768df54aa2abc314d49f5893780323 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 15:50:38 +0100 Subject: [PATCH 33/60] feat(load): admit repository and linked-worktree edit cells --- bench/load/README.md | 32 +++- bench/load/edit_probe_plan.go | 116 ++++++++++---- bench/load/edit_probe_route_phase_test.go | 166 ++++++++++++++++++++ bench/load/edit_probe_routes.go | 34 ++++ bench/load/edit_probe_routes_test.go | 15 +- bench/load/edit_probe_run.go | 6 +- bench/load/edit_probe_worktree_plan_test.go | 101 ++++++++++++ bench/load/run.go | 11 +- bench/load/scenario.go | 29 ++-- bench/load/testdata/paired-v1.schema.json | 6 + bench/load/testdata/report-v1.schema.json | 6 + 11 files changed, 467 insertions(+), 55 deletions(-) create mode 100644 bench/load/edit_probe_route_phase_test.go create mode 100644 bench/load/edit_probe_worktree_plan_test.go diff --git a/bench/load/README.md b/bench/load/README.md index 976ad529a..d2c591305 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -242,7 +242,7 @@ probes and immediate same-editing-session file/source readbacks. Session setup is outside the edit window. Future arrivals do not wait for earlier ACKs or probe replies. Canceled/unstarted edits remain in the offered denominator. -`--edit-agents N` (1 to 32) runs N fixed editors against the same single +`--edit-agents N` (1 to 32) defaults to N fixed editors against one repository. Each editor offers its own open-loop Poisson stream with the same 15 second mean; editor 0 keeps the one-editor seeding, so `--edit-agents 1` offers exactly the original plan. The streams merge by due time (ties by editor @@ -250,16 +250,36 @@ index) and the merge order is the global ordinal, so markers and mutation IDs stay keyed by ordinal and each ordinal is owned by one editor for the whole run. Plan steps and edit records name that editor. An edit and its immediate file/source readbacks go through its own editor's session; probes and receipt -status stay on the edit's separate prober session, and run-level status, stats -and the restore batch use editor 0. A blocked editor ACK does not delay another -editor's arrival. Editors share one file. `edit.write` carries whole-file +status stay on the edit's separate prober session, and run-level status and +stats use editor 0. The singleton restore batch also uses editor 0. A blocked editor ACK does not delay another +editor's arrival. In a singleton cell, editors share one file. `edit.write` carries whole-file content precomputed in ordinal order, so an out-of-order write would silently drop another editor's marker until the next write restored it; it is refused with more than one editor. An out-of-order `edit.batch` fails its `old_string` precondition and stays a failed record, and `edit.file` anchor insertions do not depend on order. The 1-edit diagnostic and -the paired no-regression cell stay one editor. Routing across several -repositories or linked worktrees is not implemented. +the paired no-regression cell stay one editor. + +`--edit-repos 6 --edit-agents 6 --agents 0 --worktrees 0` selects exactly +six distinct pinned repositories. Source checkouts that share one Git common +directory count as one repository. For linked views of one pinned primary, +use `--edit-repos 1 --worktrees 3 --edit-agents 4 --agents 0`; linked counts +3, 7 and 15 give 4, 8 and 16 total checkouts. These routed cells require 100 +global offered edits, at least one fixed editor per checkout, and ordinary Go +`add_intrafile_call` / `edit.file` / JSON fixtures. Pure readers, native and +lifecycle senders, moves and other output/tool/language cells are explicitly +unsupported for routed runs; their existing singleton cells remain separate. + +The plan pins every route's logical repository, checkout identity, commit, +file and callee. A fixed editor, its immediate readbacks, and each edit's +independent prober and receipt query stay on that checkout's cwd. Linked +checkouts require their own exact public view admission even though their +repository prefix is shared. Each route restores its admitted original bytes +through its own editor and transaction after mutation outcomes settle. The +source repositories stay read-only. Route steps preserve the original merged +editor arrival stream and global marker/mutation IDs; a route with no offered +ordinal is not silently given an extra edit. Implementation and fixture tests +do not establish a completed live or reference campaign. `edit_sender` selects a native or lifecycle traffic cell; without it every plan is unchanged, and paired runs refuse it. A native write goes around MCP, so its diff --git a/bench/load/edit_probe_plan.go b/bench/load/edit_probe_plan.go index 84830db96..88950a2c9 100644 --- a/bench/load/edit_probe_plan.go +++ b/bench/load/edit_probe_plan.go @@ -60,28 +60,29 @@ type editProbeRoute struct { } type editProbePlan struct { - Version int `json:"version"` - Arrival string `json:"arrival"` - MeanNS int64 `json:"mean_interarrival_ns"` - ProbePeriodNS int64 `json:"probe_period_ns"` - Count int `json:"count"` - Editors int `json:"editors,omitempty"` - Readers int `json:"readers,omitempty"` - NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` - Seed int64 `json:"seed"` - Kind string `json:"kind"` - Tool string `json:"tool"` - Language string `json:"language"` - Mode string `json:"mode"` - Sender string `json:"sender,omitempty"` - Repo string `json:"repo"` - Commit string `json:"commit"` - File string `json:"file"` - Callee string `json:"callee"` - SessionModel string `json:"session_model"` - NotSelected []string `json:"not_selected_not_tested"` - Steps []editProbeStep `json:"steps"` - Routes []editProbeRoute `json:"routes,omitempty"` + Version int `json:"version"` + Arrival string `json:"arrival"` + MeanNS int64 `json:"mean_interarrival_ns"` + ProbePeriodNS int64 `json:"probe_period_ns"` + Count int `json:"count"` + Editors int `json:"editors,omitempty"` + Readers int `json:"readers,omitempty"` + NoEditWindow int64 `json:"no_edit_window_ns,omitempty"` + Seed int64 `json:"seed"` + Kind string `json:"kind"` + Tool string `json:"tool"` + Language string `json:"language"` + Mode string `json:"mode"` + Sender string `json:"sender,omitempty"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + Callee string `json:"callee"` + SessionModel string `json:"session_model"` + NotSelected []string `json:"not_selected_not_tested"` + Steps []editProbeStep `json:"steps"` + Routes []editProbeRoute `json:"routes,omitempty"` + LinkedWorktrees int `json:"linked_worktrees,omitempty"` } type editProbeArrival struct { @@ -141,6 +142,30 @@ func planEditProbe(c Corpus, o runOptions) (editProbePlan, error) { return p, nil } +// A linked plan has one logical repository and separately identified checkouts. +// The runtime supplies the expanded private corpora; reporting starts from the +// one pinned primary. Both must describe the same immutable source fixture. +func editProbeLinkedCorpora(corpora []Corpus, linked int) ([]Corpus, error) { + if linked < 1 || linked > 15 || len(corpora) != 1 && len(corpora) != linked+1 { + return nil, errors.New("linked edit-probe routes require one primary and at most 15 linked checkouts") + } + primary := corpora[0] + out := []Corpus{primary} + for i := 1; i <= linked; i++ { + label := fmt.Sprintf("linked-%d", i) + if primary.Label == label { + return nil, errors.New("primary label collides with a linked checkout identity") + } + c := primary + c.Label = label + if len(corpora) > 1 && corpora[i] != c { + return nil, errors.New("linked checkout differs from the pinned primary fixture") + } + out = append(out, c) + } + return out, nil +} + // Routed plans reuse each corpus's existing finite plan. Every corpus sees // the same global ordinals/arrivals; each fixed editor takes only its own // corpus's request at that ordinal, so no secondary stream is reseeded. @@ -151,20 +176,33 @@ func planEditProbeRoutes(corpora []Corpus, o runOptions) (editProbePlan, error) if o.EditRepos < 0 { return editProbePlan{}, errors.New("invalid edit repository count") } - if o.EditRepos <= 1 { + linked := o.Worktrees > 0 + if !linked && o.EditRepos <= 1 { return planEditProbe(corpora[0], o) } - if o.EditRepos != 6 || len(corpora) != 6 || o.Editors < 6 || o.EditCount != 100 || o.Agents != 0 || o.Worktrees != 0 { - return editProbePlan{}, errors.New("six-repository planning requires exactly six pinned corpora, at least six fixed editors, 100 edits, no pure readers and no linked worktrees") + if linked { + if o.EditRepos > 1 { + return editProbePlan{}, errors.New("six repositories and linked checkouts are separate edit-probe cells") + } + var err error + corpora, err = editProbeLinkedCorpora(corpora, o.Worktrees) + if err != nil { + return editProbePlan{}, err + } + } else if o.EditRepos != 6 || len(corpora) != 6 { + return editProbePlan{}, errors.New("six-repository planning requires exactly six distinct pinned corpora") + } + if o.Editors < len(corpora) || o.EditCount != 100 || o.Agents != 0 { + return editProbePlan{}, errors.New("routed planning requires at least one fixed editor per checkout, 100 edits and no pure readers") } plans := make([]editProbePlan, len(corpora)) routes := make([]editProbeRoute, len(corpora)) labels, sources := map[string]bool{}, map[string]bool{} for i, c := range corpora { - if c.Label == "" || labels[c.Label] || sources[c.Source] { + if c.Label == "" || labels[c.Label] || !linked && sources[filepath.Clean(c.Source)] { return editProbePlan{}, errors.New("six-repository routes require distinct corpus labels and source repositories") } - labels[c.Label], sources[c.Source] = true, true + labels[c.Label], sources[filepath.Clean(c.Source)] = true, true p, err := planEditProbe(c, o) if err != nil { return editProbePlan{}, fmt.Errorf("route %s: %w", c.Label, err) @@ -173,10 +211,17 @@ func planEditProbeRoutes(corpora []Corpus, o runOptions) (editProbePlan, error) return editProbePlan{}, errors.New("six-repository planning currently supports ordinary Go add_intrafile_call/edit.file JSON fixtures only; native, lifecycle, move and other output cells are unsupported") } plans[i] = p - routes[i] = editProbeRoute{ID: c.Label, Repo: c.Label, Commit: c.Commit, File: p.File, Callee: p.Callee, Kind: p.Kind, Tool: p.Tool, Language: p.Language, Mode: p.Mode} + logicalRepo := c.Label + if linked { + logicalRepo = corpora[0].Label + } + routes[i] = editProbeRoute{ID: c.Label, Repo: logicalRepo, Commit: c.Commit, File: p.File, Callee: p.Callee, Kind: p.Kind, Tool: p.Tool, Language: p.Language, Mode: p.Mode} } out := plans[0] out.Routes = routes + if linked { + out.LinkedWorktrees = o.Worktrees + } out.Steps = append([]editProbeStep(nil), out.Steps...) for i, step := range out.Steps { route := step.Editor % len(plans) @@ -189,7 +234,7 @@ func planEditProbeRoutes(corpora []Corpus, o runOptions) (editProbePlan, error) out.NotSelected = append(out.NotSelected, limitation) } } - out.NotSelected = append(out.NotSelected, "multi-repository execution remains refused until routed oracle and restore ownership are validated") + out.NotSelected = append(out.NotSelected, "routed cells support ordinary Go add_intrafile_call/edit.file JSON only; reference and live campaign acceptance remains separate") return out, nil } @@ -205,13 +250,24 @@ func editProbeRouteOperations(plan editProbePlan, corpora []Corpus) ([]request, } return editProbeOperations(plan, corpora[0]), nil } + if plan.LinkedWorktrees > 0 { + var err error + corpora, err = editProbeLinkedCorpora(corpora, plan.LinkedWorktrees) + if err != nil { + return nil, err + } + } if len(plan.Routes) != len(corpora) { return nil, errors.New("edit route corpus denominator differs from pinned metadata") } byRoute := make(map[string]Corpus, len(corpora)) for i, c := range corpora { route := plan.Routes[i] - if _, duplicate := byRoute[route.ID]; duplicate || route.ID != c.Label || route.Repo != c.Label || route.Commit != c.Commit || route.File != c.File { + logicalRepo := c.Label + if plan.LinkedWorktrees > 0 { + logicalRepo = corpora[0].Label + } + if _, duplicate := byRoute[route.ID]; duplicate || route.ID != c.Label || route.Repo != logicalRepo || route.Commit != c.Commit || route.File != c.File { return nil, errors.New("edit route metadata differs from its pinned corpus") } byRoute[route.ID] = c diff --git a/bench/load/edit_probe_route_phase_test.go b/bench/load/edit_probe_route_phase_test.go new file mode 100644 index 000000000..5f62ea986 --- /dev/null +++ b/bench/load/edit_probe_route_phase_test.go @@ -0,0 +1,166 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// This is startup/layout admission followed by an intentional pre-edit stop. +// Every offered edit stays censored; no live mutation/capacity claim is made. +func TestEditProbeMultiRootPhaseAdmitsOnlyItsPlannedOwnedCheckouts(t *testing.T) { + for _, cell := range []struct { + name string + repos, linked, editors int + }{ + {"six_repositories", 6, 0, 6}, {"four_total_worktrees", 1, 3, 4}, + } { + t.Run(cell.name, func(t *testing.T) { + t.Setenv("LOAD_DAEMON_FIXTURE", "serve") + var corpora []Corpus + var originals []string + for i := range cell.repos { + c := gitFixture(t) + c.Label = fmt.Sprintf("route-%d", i) + c.EditAnchor = "func Caller() int { return Example() }" + c.EditTemplate = "\nfunc {{marker}}() int { return Example() }\n" + content := fmt.Sprintf("package sample\n// owner-%d\nfunc Example() int { return 1 }\n%s\n", i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(c.Source, c.File), []byte(content), 0600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"add", c.File}, {"commit", "--quiet", "-m", "routing fixture"}, {"rev-parse", "HEAD"}} { + cmd := exec.Command("git", append([]string{"-C", c.Source, "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "-c", "core.hooksPath=" + os.DevNull}, args...)...) + data, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v %s", args, err, data) + } + if args[0] == "rev-parse" { + c.Commit = strings.TrimSpace(string(data)) + } + } + corpora = append(corpora, c) + originals = append(originals, content) + } + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + o := runOptions{Binary: binary, Profile: "agent", Host: "native", Editors: cell.editors, EditCount: 100, EditRepos: cell.repos, Worktrees: cell.linked, Seed: 42, Duration: time.Second, Deadline: time.Second, WarmTimeout: 10 * time.Second, StopTimeout: 5 * time.Second} + stop := errors.New("layout admitted; intentional diagnostic stop before edits") + called := false + o.pairedDriver = func(_ context.Context, p *privateRun, _, _ []*socketClient, phase *Phase) error { + called = true + run := phase.editProbeRuntime + total := cell.repos + cell.linked + if run == nil || len(p.Repos) != total || len(run.plan.Routes) != total || len(run.originals) != total { + return fmt.Errorf("planned checkout denominator lost: private=%d runtime=%v", len(p.Repos), run != nil) + } + data, err := os.ReadFile(filepath.Join(p.Root, "config", "gortex", "config.yaml")) + if err != nil { + return err + } + var config struct { + Repos []json.RawMessage `json:"repos"` + } + if err := json.Unmarshal(data, &config); err != nil { + return err + } + expectedBoot := cell.repos + if len(config.Repos) != expectedBoot { + return fmt.Errorf("boot configured %d roots, want%d", len(config.Repos), expectedBoot) + } + for _, step := range run.plan.Steps { + repo, err := run.repoFor(step) + if err != nil { + return err + } + expected := p.Repos[step.Editor%len(p.Repos)] + if repo.Path != expected.Path || run.editorFor(step).cwd != expected.Path || run.states[step.Ordinal].client.cwd != expected.Path { + return fmt.Errorf("ordinal%d fell back from its owned checkout", step.Ordinal) + } + } + for i, repo := range p.Repos { + current, err := os.ReadFile(repo.File) + if err != nil || string(run.originals[repo.Path]) != string(current) { + return fmt.Errorf("checkout%d original capture missing: %v", i, err) + } + if cell.linked > 0 && i > 0 { + var receipt struct { + View struct { + Kind string `json:"kind"` + Exact bool `json:"exact"` + Prefix string `json:"repo_prefix"` + } `json:"view"` + } + if err := json.Unmarshal(phase.Gate[repo.Corpus.Label+".view"], &receipt); err != nil || !receipt.View.Exact || receipt.View.Kind != "worktree" || receipt.View.Prefix != repo.Prefix { + return fmt.Errorf("checkout%d has no exact public worktree admission", i) + } + } + } + return stop + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + phase, _, artifacts, _, _, err := runPhase(ctx, o, editProbeScenario, corpora, "", t.TempDir()) + if !called || !errors.Is(err, stop) { + t.Fatalf("owned routed phase was not admitted: called=%v err=%v", called, err) + } + if phase.EditProbe == nil || len(phase.EditProbe.Records) != 100 { + t.Fatal("diagnostic stop lost offered denominator") + } + for _, record := range phase.EditProbe.Records { + if record.Outcome != "phase_censored" || record.Ack.Value != nil { + t.Fatal("pre-edit layout diagnostic fabricated work", record) + } + } + forced := phase.Metrics["force_killed"] + if forced.Value == nil || *forced.Value != 0 { + t.Fatal("layout fixture failed normal owned stop", phase.Metrics) + } + if _, err := os.Stat(filepath.Join(artifacts["private_root"], "repos")); !os.IsNotExist(err) { + t.Fatal("private checkout handles/state retained", err) + } + for i, c := range corpora { + actual, err := os.ReadFile(filepath.Join(c.Source, c.File)) + if err != nil || string(actual) != originals[i] { + t.Fatal("source fixture was mutated", i, err) + } + } + }) + } +} + +func TestEditProbeSourceRepositoryIdentityDoesNotCountLinkedViewsTwice(t *testing.T) { + primary := gitFixture(t) + before, err := os.ReadFile(filepath.Join(primary.Source, primary.File)) + if err != nil { + t.Fatal(err) + } + linkedPath := filepath.Join(t.TempDir(), "linked-source") + cmd := exec.CommandContext(t.Context(), "git", "-C", primary.Source, "-c", "core.hooksPath="+os.DevNull, "-c", "gc.auto=0", "-c", "maintenance.auto=false", "worktree", "add", "--detach", linkedPath, primary.Commit) + if data, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("own source worktree: %v %s", err, data) + } + linked := primary + linked.Source, linked.Label = linkedPath, "linked-source" + if err := validateEditProbeSourceRepos(t.Context(), []Corpus{primary, linked}, t.TempDir()); err == nil || !strings.Contains(err.Error(), "distinct source repositories") { + t.Fatalf("two source views masqueraded as independent repositories: %v", err) + } + independent := gitFixture(t) + if err := validateEditProbeSourceRepos(t.Context(), []Corpus{primary, independent}, t.TempDir()); err != nil { + t.Fatalf("independent source repositories refused: %v", err) + } + for _, c := range []Corpus{primary, linked} { + after, err := os.ReadFile(filepath.Join(c.Source, c.File)) + if err != nil || string(after) != string(before) { + t.Fatalf("read-only identity admission mutated source bytes: %v", err) + } + } +} diff --git a/bench/load/edit_probe_routes.go b/bench/load/edit_probe_routes.go index f6bbd9388..91e780830 100644 --- a/bench/load/edit_probe_routes.go +++ b/bench/load/edit_probe_routes.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" ) @@ -72,3 +73,36 @@ func (r *editProbeRuntime) restoreRoutes(ctx context.Context) (resultErr error) } return nil } + +// Distinct source repositories are a physical admission check, not a path in +// the workload hash. Read-only git metadata cannot turn linked source views +// into six independent repositories or touch any source working bytes. +func validateEditProbeSourceRepos(ctx context.Context, corpora []Corpus, outputRoot string) error { + reader := &privateRun{Root: outputRoot, Env: append(os.Environ(), "GIT_OPTIONAL_LOCKS=0")} + var seen []os.FileInfo + for _, c := range corpora { + data, err := reader.git(ctx, c.Source, "rev-parse", "--git-common-dir") + if err != nil { + return err + } + common := strings.TrimSpace(string(data)) + if !filepath.IsAbs(common) { + common = filepath.Join(c.Source, common) + } + common, err = canonicalPath(common) + if err != nil { + return err + } + identity, err := os.Stat(common) + if err != nil { + return err + } + for _, prior := range seen { + if os.SameFile(prior, identity) { + return errors.New("six-repository execution requires six distinct source repositories; linked checkouts share one repository") + } + } + seen = append(seen, identity) + } + return nil +} diff --git a/bench/load/edit_probe_routes_test.go b/bench/load/edit_probe_routes_test.go index 5351f56ff..fd0dbf016 100644 --- a/bench/load/edit_probe_routes_test.go +++ b/bench/load/edit_probe_routes_test.go @@ -109,6 +109,9 @@ func TestEditProbeSelectedCheckoutsKeepProberSessionsOnTheirEditorRoute(t *testi prefix := c.Label if sharedPrefix { prefix, c.Label = "fixture", "fixture" + if i > 0 { + c.Label = fmt.Sprintf("linked-%d", i) + } } p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: prefix, Corpus: c}) } @@ -288,10 +291,12 @@ func TestEditProbeRoutedPlansRejectMissingOrForeignOwnership(t *testing.T) { } } -func TestEditProbeRoutedPlanDoesNotEnableAnUnvalidatedRuntime(t *testing.T) { - o := runOptions{Scenario: editProbeScenario, EditRepos: 6} - phase, _, artifacts, _, _, err := runPhase(t.Context(), o, editProbeScenario, editProbeRoutingCorpora(), "", t.TempDir()) - if err == nil || !strings.Contains(err.Error(), "multi-repository edit-and-probe execution is unsupported") || artifacts["private_root"] != "" || phase.editProbeRuntime != nil { - t.Fatalf("planning metadata silently enabled a partial runtime: err=%v artifacts=%v", err, artifacts) +func TestEditProbeRoutedPhaseRefusesUnsupportedTrafficBeforeAllocatingOwner(t *testing.T) { + corpora := editProbeRoutingCorpora() + corpora[1].EditOutput = "cli" + o := runOptions{Scenario: editProbeScenario, EditRepos: 6, Editors: 6, EditCount: 100, Seed: 42} + phase, _, artifacts, _, _, err := runPhase(t.Context(), o, editProbeScenario, corpora, "", t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "unsupported") || artifacts["private_root"] != "" || phase.editProbeRuntime != nil { + t.Fatalf("unsupported routed traffic allocated an execution owner: err=%v artifacts=%v", err, artifacts) } } diff --git a/bench/load/edit_probe_run.go b/bench/load/edit_probe_run.go index a9e1e9732..b835b72b7 100644 --- a/bench/load/edit_probe_run.go +++ b/bench/load/edit_probe_run.go @@ -213,7 +213,11 @@ func (r *editProbeRuntime) repoFor(step editProbeStep) (privateRepo, error) { } for _, repo := range r.p.Repos { if editor.cwd != "" && filepath.Clean(editor.cwd) == filepath.Clean(repo.Path) { - if offered != nil && (offered.Repo != repo.Corpus.Label || offered.Commit != repo.Corpus.Commit || offered.File != repo.Corpus.File) { + logicalRepo := repo.Corpus.Label + if r.plan.LinkedWorktrees > 0 { + logicalRepo = r.plan.Repo + } + if offered != nil && (offered.ID != repo.Corpus.Label || offered.Repo != logicalRepo || offered.Commit != repo.Corpus.Commit || offered.File != repo.Corpus.File) { return privateRepo{}, errors.New("editor checkout differs from its offered route pin") } return repo, nil diff --git a/bench/load/edit_probe_worktree_plan_test.go b/bench/load/edit_probe_worktree_plan_test.go new file mode 100644 index 000000000..6942bf526 --- /dev/null +++ b/bench/load/edit_probe_worktree_plan_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "encoding/json" + "fmt" + "testing" +) + +func TestEditProbeLinkedCheckoutPlansKeepPrimaryPinAndBothCounts(t *testing.T) { + for _, linked := range []int{3, 7, 15} { + t.Run(fmt.Sprintf("total-%d", linked+1), func(t *testing.T) { + total := linked + 1 + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", editProbeScenario, "--agents", "0", "--edit-agents", fmt.Sprint(total), "--worktrees", fmt.Sprint(linked), "--edit-count", "100", "--seed", "42"} + options, err := parseRun(args) + if err != nil { + t.Fatalf("explicit linked-checkout plan refused: %v", err) + } + corpus := editProbeCorpus() + report, err := newRunReport(options, []Corpus{corpus}, corpus.Commit, Host{}, t.TempDir()) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(report.Workload.EditProbe) + if err != nil { + t.Fatal(err) + } + var plan struct { + Routes []struct { + ID string `json:"id"` + Repo string `json:"repo"` + Commit string `json:"commit"` + File string `json:"file"` + } `json:"routes"` + Steps []struct { + Editor int `json:"editor"` + Route string `json:"route"` + DueNS int64 `json:"due_ns"` + } `json:"steps"` + } + if err := json.Unmarshal(data, &plan); err != nil { + t.Fatal(err) + } + if report.Workload.LinkedWorktrees != linked || len(plan.Routes) != total || len(plan.Steps) != 100 { + t.Fatalf("primary/linked denominator lost: workload=%d routes=%d steps=%d", report.Workload.LinkedWorktrees, len(plan.Routes), len(plan.Steps)) + } + ids := make([]string, total) + for i, route := range plan.Routes { + expected := corpus.Label + if i > 0 { + expected = fmt.Sprintf("linked-%d", i) + } + if route.ID != expected || route.Repo != corpus.Label || route.Commit != corpus.Commit || route.File != corpus.File { + t.Fatalf("checkout%d lost logical primary pin: %+v", i, route) + } + ids[i] = route.ID + } + reference, err := makeEditProbeTrafficPlan(corpus, 42, 100, total) + if err != nil { + t.Fatal(err) + } + seen := make([]int, total) + expectedCounts := make([]int, total) + for i, step := range plan.Steps { + if step.Editor < 0 || step.Editor >= total || step.Route != ids[step.Editor] || step.DueNS != reference.Steps[i].DueNS { + t.Fatalf("checkout changed its fixed-editor offer: %+v", step) + } + seen[step.Editor]++ + expectedCounts[reference.Steps[i].Editor]++ + } + for editor := range total { + if seen[editor] != expectedCounts[editor] { + t.Fatalf("editor%d offered denominator changed: actual%d expected%d", editor, seen[editor], expectedCounts[editor]) + } + } + }) + } +} + +func TestEditProbeLinkedCheckoutPlansRefuseAmbiguousPrimaryOwnership(t *testing.T) { + options := runOptions{Scenario: editProbeScenario, Seed: 42, EditCount: 100, Editors: 4, EditRepos: 1, Worktrees: 3} + for _, name := range []string{"two_primaries", "too_few_editors", "readers", "mixed_six_repos", "linked_label_collision"} { + t.Run(name, func(t *testing.T) { + corpora, o := []Corpus{editProbeCorpus()}, options + switch name { + case "two_primaries": + corpora = append(corpora, corpora[0]) + case "too_few_editors": + o.Editors = 3 + case "readers": + o.Agents = 1 + case "mixed_six_repos": + o.EditRepos = 6 + case "linked_label_collision": + corpora[0].Label = "linked-1" + } + if _, err := planEditProbeRoutes(corpora, o); err == nil { + t.Fatal("ambiguous checkout ownership accepted") + } + }) + } +} diff --git a/bench/load/run.go b/bench/load/run.go index ed7a677ea..cd447d5ec 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -89,7 +89,7 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") bindStartupFlags(f, &o.Startup) - f.IntVar(&o.EditRepos, "edit-repos", 1, "edit-and-probe: one repository (default), or an explicit six-pinned-repository plan") + f.IntVar(&o.EditRepos, "edit-repos", 1, "edit-and-probe: one repository (default), or exactly six distinct pinned repositories") f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } @@ -117,8 +117,11 @@ func validateRunOptions(o runOptions) error { if _, err := parseProfile(o.Profile); err != nil { return err } - if o.Scenario == editProbeScenario && (o.Profile != "agent" || o.Agents > editProbeMaxReaders || o.Editors < 1 || o.Editors > editProbeMaxEditors || o.Worktrees != 0 || o.CPUProfile != 0 || o.EditCount != 1 && o.EditCount != 100 || o.EditCount == 1 && o.Editors != 1) { - return fmt.Errorf("edit-and-probe requires --profile agent --agents 0..%d --edit-agents 1..%d on one repository --worktrees 0, no CPU profile, and --edit-count 100 (or an explicit 1-edit, 1-editor diagnostic)", editProbeMaxReaders, editProbeMaxEditors) + if o.Scenario == editProbeScenario && (o.Profile != "agent" || o.Agents > editProbeMaxReaders || o.Editors < 1 || o.Editors > editProbeMaxEditors || o.Worktrees > 15 || o.CPUProfile != 0 || o.EditCount != 1 && o.EditCount != 100 || o.EditCount == 1 && o.Editors != 1) { + return fmt.Errorf("edit-and-probe requires --profile agent --agents 0..%d --edit-agents 1..%d on one repository or explicit routed checkouts, at most 15 linked worktrees, no CPU profile, and --edit-count 100 (or an explicit 1-edit, 1-editor diagnostic)", editProbeMaxReaders, editProbeMaxEditors) + } + if o.Scenario == editProbeScenario && (o.EditRepos > 1 || o.Worktrees > 0) && (o.EditRepos > 1 && o.Worktrees > 0 || o.Agents != 0 || o.EditCount != 100 || o.Editors < max(o.EditRepos, o.Worktrees+1)) { + return errors.New("routed edit-and-probe selects six repositories or linked checkouts, 100 edits, no pure readers and at least one fixed editor per checkout") } if o.EditProbeRole != "" && (o.Scenario != editProbeScenario || o.EditProbeRole != "candidate" && o.EditProbeRole != "baseline") { return errors.New("--edit-probe-role is candidate or baseline, for edit-and-probe only") @@ -137,7 +140,7 @@ func validateRunOptions(o runOptions) error { activeAgents += o.Editors } activeWorktrees := 1 - if o.Scenario == "worktrees" || o.Scenario == "all" { + if o.Scenario == "worktrees" || o.Scenario == "all" || o.Scenario == editProbeScenario && o.Worktrees > 0 { activeWorktrees += o.Worktrees } if o.Host == "floor" && (activeAgents > 4 || activeWorktrees > 4) { diff --git a/bench/load/scenario.go b/bench/load/scenario.go index d8e00a7f7..9378b86b4 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -61,11 +61,19 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp resultErr = errors.New(strings.Join(phase.Errors, "; ")) } }() - if scenario == editProbeScenario && o.EditRepos > 1 { - return phase, nil, artifacts, crashes, "", errors.New("multi-repository edit-and-probe execution is unsupported until routed oracle and restore ownership are validated") + worktreeViews := scenario == "worktrees" || scenario == editProbeScenario && o.Worktrees > 0 + if scenario == editProbeScenario && (o.EditRepos > 1 || o.Worktrees > 0) { + if _, err := planEditProbeRoutes(corpora, o); err != nil { + return phase, nil, artifacts, crashes, "", err + } + if o.EditRepos > 1 { + if err := validateEditProbeSourceRepos(ctx, corpora, outputRoot); err != nil { + return phase, nil, artifacts, crashes, "", err + } + } } selected := corpora[:1] - if scenario == "many" || scenario == "many-edit" { + if scenario == "many" || scenario == "many-edit" || scenario == editProbeScenario && o.EditRepos > 1 { if len(corpora) < 2 { return phase, nil, artifacts, crashes, "", errors.New("many-repository scenarios require at least two pinned repositories") } @@ -90,11 +98,14 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } } if scenario == editProbeScenario { - plan, err := planEditProbe(selected[0], o) + plan, err := planEditProbeRoutes(selected, o) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + phase.Operations, err = editProbeRouteOperations(plan, selected) if err != nil { return phase, nil, artifacts, crashes, "", err } - phase.Operations = editProbeOperations(plan, selected[0]) phase.EditProbe = newEditProbeReport(plan, selected[0], !o.editProbeBaseline) phase.SessionModel = plan.SessionModel } @@ -154,7 +165,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp resultErr = errors.Join(resultErr, p.removeState()) } }() - if scenario == "worktrees" { + if worktreeViews { if err := p.addWorktrees(ctx, bootWorktrees(o)); err != nil { return phase, nil, artifacts, crashes, "", err } @@ -167,7 +178,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } warmCtx, cancelWarm := context.WithTimeout(ctx, o.WarmTimeout) defer cancelWarm() - configured, err := p.writeBootRepos(scenario == "worktrees") + configured, err := p.writeBootRepos(worktreeViews) if err != nil { return phase, nil, artifacts, crashes, "", err } @@ -233,7 +244,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp p.Repos[i].Prefix = tracked.Prefix } } - if i > 0 && scenario == "worktrees" { + if i > 0 && worktreeViews { p.Repos[i].Prefix = p.Repos[0].Prefix } if p.Repos[i].Prefix == "" { @@ -263,7 +274,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp version = client.version if i < len(p.Repos) { subscribeInvalidations(warmCtx, client, repo.Corpus.Label, o.Deadline, &phase) - if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0, o.Deadline, &phase); err != nil { + if err := preflightRoot(warmCtx, p, client, repo, worktreeViews && i > 0, o.Deadline, &phase); err != nil { return phase, nil, artifacts, crashes, version, err } gate, rows, err := warmAnalysis(warmCtx, client, o.Deadline) diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index 0746cfc62..64ef1524d 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -1783,6 +1783,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -3621,6 +3624,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index eea3971b7..39516d7ed 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -1770,6 +1770,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, @@ -3608,6 +3611,9 @@ "language": { "type": "string" }, + "linked_worktrees": { + "type": "integer" + }, "mean_interarrival_ns": { "type": "integer" }, From 55ea4e3ef12bdca05a9adf4cb50c3b8ff482870e Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 16:05:30 +0100 Subject: [PATCH 34/60] fix(load): bind linked mutation status to admitted checkout --- bench/load/README.md | 4 +- .../load/edit_probe_checkout_receipt_test.go | 155 ++++++++++++++++++ bench/load/edit_probe_routes_test.go | 9 +- bench/load/edit_probe_run.go | 35 ++++ bench/load/selftest_test.go | 4 +- bench/load/testdata/paired-v1.schema.json | 3 + bench/load/testdata/report-v1.schema.json | 3 + 7 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 bench/load/edit_probe_checkout_receipt_test.go diff --git a/bench/load/README.md b/bench/load/README.md index d2c591305..40f6f9ba3 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -274,7 +274,9 @@ The plan pins every route's logical repository, checkout identity, commit, file and callee. A fixed editor, its immediate readbacks, and each edit's independent prober and receipt query stay on that checkout's cwd. Linked checkouts require their own exact public view admission even though their -repository prefix is shared. Each route restores its admitted original bytes +repository prefix is shared. Linked edit records retain that admitted checkout +ID; a missing or different receipt checkout ID cannot supply terminal graph +or accounting evidence. Each route restores its admitted original bytes through its own editor and transaction after mutation outcomes settle. The source repositories stay read-only. Route steps preserve the original merged editor arrival stream and global marker/mutation IDs; a route with no offered diff --git a/bench/load/edit_probe_checkout_receipt_test.go b/bench/load/edit_probe_checkout_receipt_test.go new file mode 100644 index 000000000..b7e2569d4 --- /dev/null +++ b/bench/load/edit_probe_checkout_receipt_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// A same-prefix view must not borrow another checkout's terminal status even +// when the mutation ID, source bytes and all graph answers otherwise match. +func TestEditProbeLinkedReceiptCannotBorrowAnotherCheckoutIdentity(t *testing.T) { + for _, cell := range []string{"matching", "foreign", "missing", "anonymous_admission", "duplicate_admission", "missing_checkout"} { + t.Run(cell, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + c := editProbeCorpus() + c.Label = "fixture" + p := &privateRun{Root: t.TempDir()} + labels := []string{"fixture", "linked-1"} + if cell == "missing_checkout" { + labels = labels[:1] + } + if cell == "duplicate_admission" { + labels = append(labels, "linked-2") + } + for i, label := range labels { + root := filepath.Join(p.Root, label) + if err := os.Mkdir(root, 0700); err != nil { + t.Fatal(err) + } + content := fmt.Sprintf("package sample\n// checkout%d\nfunc Example() int { return 1 }\n%s\n", i, c.EditAnchor) + if err := os.WriteFile(filepath.Join(root, c.File), []byte(content), 0600); err != nil { + t.Fatal(err) + } + corpus := c + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Path: root, File: filepath.Join(root, c.File), Prefix: "fixture", Corpus: corpus}) + } + editors, linkedCount := len(labels), len(labels)-1 + if cell == "missing_checkout" { + editors, linkedCount = 2, 1 + } + full, err := makeEditProbeTrafficPlan(c, 42, 100, editors) + if err != nil { + t.Fatal(err) + } + var selected editProbeStep + for _, step := range full.Steps { + if step.Editor == 1 { + selected = step + break + } + } + disk := editProbeDiskStub(t, c, p.Repos[len(p.Repos)-1].Path) + viewID := "expected-linked-checkout" + if cell == "anonymous_admission" { + viewID = "" + } + statusID := "expected-linked-checkout" + if cell == "foreign" { + statusID = "foreign-checkout" + } + if cell == "missing" { + statusID = "" + } + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Kind == "search_symbols" { + return map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": viewID}}, nil + } + if frame.Method != "tools/call" { + return stubContent(map[string]any{}), nil + } + args, _ := frame.Params["arguments"].(map[string]any) + if frame.Params["name"] == "edit" && args["operation"] == "file" { + data, err := os.ReadFile(p.Repos[1].File) + if err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "read failed"} + } + match, _ := args["match"].(string) + replacement, _ := args["replacement"].(string) + if match == "" || strings.Count(string(data), match) != 1 { + return nil, &rpcError{Code: -32000, Message: "exact match missing"} + } + if err := os.WriteFile(p.Repos[1].File, []byte(strings.Replace(string(data), match, replacement, 1)), 0600); err != nil { + t.Error(err) + return nil, &rpcError{Code: -32000, Message: "write failed"} + } + return stubContent(map[string]any{"mutation_receipt": "own-receipt"}), nil + } + if frame.Params["name"] == "change" && args["operation"] == "receipt" { + status := map[string]any{"found": true, "receipt": "own-receipt", "mutation_id": selected.MutationID, "graph_status_terminal": true, "graph_status": "fresh"} + if statusID != "" { + status["checkout_id"] = statusID + } + return stubContent(status), nil + } + return disk(frame) + }) + gate, err := json.Marshal(map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": viewID}}) + if err != nil { + t.Fatal(err) + } + phase := &Phase{Metrics: map[string]Metric{}, Gate: map[string]json.RawMessage{}} + for _, label := range labels[1:] { + phase.Gate[label+".view"] = gate + } + var clients []*socketClient + t.Cleanup(func() { closeClients(clients) }) + for i := range editors { + repo := p.Repos[i%len(p.Repos)] + client, err := openMCP(ctx, p.Socket, repo.Path) + if err != nil { + t.Fatal(err) + } + clients = append(clients, client) + } + run, probers, err := prepareEditProbe(ctx, runOptions{Seed: 42, EditCount: 100, Editors: editors, Worktrees: linkedCount, Deadline: 2 * time.Second, editProbeBaseline: true}, p, nil, clients, phase) + clients = append(clients, probers...) + if run != nil { + t.Cleanup(func() { _ = run.close() }) + } + if cell == "anonymous_admission" || cell == "duplicate_admission" || cell == "missing_checkout" { + if err == nil { + t.Fatal("anonymous exact/shared-prefix view admitted without a checkout identity") + } + return + } + if err != nil { + t.Fatal(err) + } + run.origin = time.Now() + step := run.plan.Steps[selected.Ordinal] + state := run.states[step.Ordinal] + run.activate(ctx, step) + awaitEditProbeFirstProbe(t, state) + run.doEdit(ctx, step) + if state.record.CheckoutID != "expected-linked-checkout" { + t.Fatalf("record lost its admitted checkout identity: %+v", state.record) + } + if cell == "matching" { + if state.record.Outcome != "complete" || !state.record.GraphTerminal { + t.Fatalf("own checkout receipt refused: %+v", state.record) + } + } else if state.record.Outcome == "complete" || state.record.GraphTerminal || !strings.Contains(strings.Join(state.record.Errors, ";"), "checkout") { + t.Fatalf("%s checkout identity borrowed a terminal mutation: %+v", cell, state.record) + } + }) + } +} diff --git a/bench/load/edit_probe_routes_test.go b/bench/load/edit_probe_routes_test.go index fd0dbf016..022aef95f 100644 --- a/bench/load/edit_probe_routes_test.go +++ b/bench/load/edit_probe_routes_test.go @@ -129,7 +129,14 @@ func TestEditProbeSelectedCheckoutsKeepProberSessionsOnTheirEditorRoute(t *testi editors = append(editors, client) } t.Cleanup(func() { closeClients(editors) }) - phase := &Phase{Metrics: map[string]Metric{}} + phase := &Phase{Metrics: map[string]Metric{}, Gate: map[string]json.RawMessage{}} + if sharedPrefix { + view, err := json.Marshal(map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": "fixture", "checkout_id": "socket-linked-checkout"}}) + if err != nil { + t.Fatal(err) + } + phase.Gate["linked-1.view"] = view + } options := runOptions{Seed: 42, EditCount: 100, Editors: 2, Deadline: time.Second} if sharedPrefix { options.Worktrees = 1 // one linked checkout in addition to primary diff --git a/bench/load/edit_probe_run.go b/bench/load/edit_probe_run.go index b835b72b7..eaa1b3232 100644 --- a/bench/load/edit_probe_run.go +++ b/bench/load/edit_probe_run.go @@ -32,6 +32,7 @@ type editProbeRecord struct { Ordinal int `json:"ordinal"` Editor int `json:"editor,omitempty"` Route string `json:"route,omitempty"` + CheckoutID string `json:"checkout_id,omitempty"` Marker string `json:"marker"` MutationID string `json:"mutation_id"` PlannedDueNS int64 `json:"planned_due_ns"` @@ -125,6 +126,7 @@ type editProbeRuntime struct { states []*editProbeState original []byte originals map[string][]byte // routed checkout path -> independently admitted original bytes + checkoutIDs map[string]string // linked checkout path -> exact admitted public view identity moveOriginal []byte origin time.Time mu sync.Mutex // Phase.Attempts, reads and mode counts only; request lifetimes are never serialized @@ -250,6 +252,9 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, if err != nil { return nil, nil, err } + if len(plan.Routes) > 0 && len(plan.Routes) != len(p.Repos) { + return nil, nil, errors.New("offered edit routes differ from registered private checkout count") + } if len(p.Repos) > 1 { for _, repo := range p.Repos { c := repo.Corpus @@ -312,6 +317,29 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, r.originals[repo.Path] = data } } + if plan.LinkedWorktrees > 0 { + r.checkoutIDs = make(map[string]string, plan.LinkedWorktrees) + seen := map[string]bool{} + for _, repo := range p.Repos[1:] { + var admitted struct { + View *struct { + Kind string `json:"kind"` + Exact bool `json:"exact"` + Prefix string `json:"repo_prefix"` + ID string `json:"checkout_id"` + } `json:"view"` + } + if err := json.Unmarshal(phase.Gate[repo.Corpus.Label+".view"], &admitted); err != nil || admitted.View == nil || !admitted.View.Exact || admitted.View.Kind != "worktree" || admitted.View.Prefix != repo.Prefix || strings.TrimSpace(admitted.View.ID) == "" { + return r, nil, errors.New("linked checkout has no exact admitted public checkout identity") + } + id := admitted.View.ID + if seen[id] { + return r, nil, errors.New("linked checkouts share an admitted checkout identity") + } + seen[id] = true + r.checkoutIDs[repo.Path] = id + } + } if plan.Tool == "refactor.move" { r.moveOriginal, err = os.ReadFile(filepath.Join(p.Repos[0].Path, filepath.FromSlash(p.Repos[0].Corpus.EditDestination))) if err != nil { @@ -342,6 +370,7 @@ func prepareEditProbe(ctx context.Context, o runOptions, p *privateRun, readers, return r, nil, err } record := newEditProbeRecord(repo.Corpus, step) + record.CheckoutID = r.checkoutIDs[repo.Path] record.Outcome = "not_started" record.Errors = nil state := &editProbeState{step: step, record: record} @@ -884,6 +913,12 @@ func (r *editProbeRuntime) finishEditAfterACK(ctx, callCtx context.Context, s *e } if statusErr == nil { if status, decodeErr := decodeEditProbeResult(raw); decodeErr == nil { + if expected := s.record.CheckoutID; expected != "" && status["checkout_id"] != expected { + s.mu.Lock() + s.record.Errors = append(s.record.Errors, "status checkout identity does not match the admitted linked view") + s.mu.Unlock() + break // Foreign terminal/accounting fields cannot settle this edit. + } accounting := checkEditProbeAccountingForTool(status, receipt, r.plan.Tool) s.mu.Lock() s.record.Accounting = accounting diff --git a/bench/load/selftest_test.go b/bench/load/selftest_test.go index 208a28bde..6f8277c00 100644 --- a/bench/load/selftest_test.go +++ b/bench/load/selftest_test.go @@ -143,12 +143,14 @@ func fixtureDaemon(mode string) int { case "status": result = map[string]any{"ready": true, "enriched": true, "views": map[string]any{"coordinators": coordinators()}, "storage": map[string]any{"wal_pending_frames": 0}, "pprof_addr": pprof.Addr().String()} case "search_symbols": + checkoutID := "" if path, _ := frame.Params["path"].(string); path != "" { viewsMu.Lock() views[filepath.Clean(path)] = true viewsMu.Unlock() + checkoutID = "fixture-checkout-" + filepath.Base(path) } - result = map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": config.Repos[0].Name}} + result = map[string]any{"view": map[string]any{"kind": "worktree", "exact": true, "repo_prefix": config.Repos[0].Name, "checkout_id": checkoutID}} case "shutdown": result = map[string]any{"stopping": true} default: diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index 64ef1524d..828857679 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -697,6 +697,9 @@ } ] }, + "checkout_id": { + "type": "string" + }, "editor": { "type": "integer" }, diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index 39516d7ed..7a70edad2 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -684,6 +684,9 @@ } ] }, + "checkout_id": { + "type": "string" + }, "editor": { "type": "integer" }, From 8d8d6b4ad70a27a855a54f5e00d1eff03072a310 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 16:38:48 +0100 Subject: [PATCH 35/60] fix(mcp): explain untracked session recovery --- cmd/gortex/daemon_mcp.go | 7 +++- cmd/gortex/daemon_mcp_recovery_test.go | 56 ++++++++++++++++++++++++++ docs/multi-repo.md | 2 +- 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 cmd/gortex/daemon_mcp_recovery_test.go diff --git a/cmd/gortex/daemon_mcp.go b/cmd/gortex/daemon_mcp.go index 44912b21e..444c29f99 100644 --- a/cmd/gortex/daemon_mcp.go +++ b/cmd/gortex/daemon_mcp.go @@ -845,16 +845,19 @@ func (d *mcpDispatcher) notTrackedError(sess *daemon.Session, inbound []byte) [] } _ = json.Unmarshal(inbound, &peek) + // Tool arguments cannot change session admission. Point callers to an + // explicit listing instead of enumerating unrelated repositories per error. + const recovery = "Use `gortex repos` to list tracked repositories, then reconnect MCP with its working directory inside the intended repository." resp := map[string]any{ "jsonrpc": "2.0", "id": peek.ID, "error": map[string]any{ "code": -32000, - "message": fmt.Sprintf("repository not tracked: %s", sess.CWD), + "message": fmt.Sprintf("repository not tracked: %s. %s", sess.CWD, recovery), "data": map[string]any{ "error_code": "repo_not_tracked", "path": sess.CWD, - "suggestion": fmt.Sprintf("Run `gortex track %s` to include this repo in the shared graph.", sess.CWD), + "suggestion": fmt.Sprintf("%s Alternatively, run `gortex track %s` to include this repo in the shared graph.", recovery, sess.CWD), }, }, } diff --git a/cmd/gortex/daemon_mcp_recovery_test.go b/cmd/gortex/daemon_mcp_recovery_test.go new file mode 100644 index 000000000..8514dbd02 --- /dev/null +++ b/cmd/gortex/daemon_mcp_recovery_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/daemon" +) + +func TestDispatcherUntrackedRecoveryNamesWorkingDirectoryRemedy(t *testing.T) { + tracked, untracked := t.TempDir(), t.TempDir() + d, _ := trackedPathMCPSetup(t, tracked) + sess := &daemon.Session{ID: "untracked_recovery", CWD: untracked} + + for _, tool := range []string{"explore", "search_symbols"} { + t.Run(tool, func(t *testing.T) { + // Repository arguments cannot bypass session admission. The recovery + // must tell the client to change its MCP working directory instead. + frame := fmt.Appendf(nil, `{"jsonrpc":"2.0","id":19,"method":"tools/call","params":{"name":%q,"arguments":{"path":%q,"repo":"intended"}}}`, tool, tracked) + reply, err := d.Dispatch(context.Background(), sess, frame) + require.NoError(t, err) + var response struct { + ID int `json:"id"` + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + ErrorCode string `json:"error_code"` + Path string `json:"path"` + Suggestion string `json:"suggestion"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.Unmarshal(reply, &response)) + assert.Equal(t, 19, response.ID) + assert.Equal(t, -32000, response.Error.Code) + assert.Equal(t, "repo_not_tracked", response.Error.Data.ErrorCode) + assert.Equal(t, untracked, response.Error.Data.Path, "retain the existing machine-readable attempted path") + for _, text := range []string{response.Error.Message, response.Error.Data.Suggestion} { + assert.Contains(t, text, "gortex repos") + assert.Contains(t, text, "working directory") + assert.NotContains(t, text, tracked) + assert.Contains(t, text, untracked, "retain the attempted-path diagnostic") + assert.NotContains(t, text, "path=") + assert.NotContains(t, text, "repo=") + } + assert.Contains(t, response.Error.Data.Suggestion, "reconnect") + assert.Contains(t, response.Error.Data.Suggestion, "gortex track "+untracked) + }) + } +} diff --git a/docs/multi-repo.md b/docs/multi-repo.md index ed8722ddf..0173b1c47 100644 --- a/docs/multi-repo.md +++ b/docs/multi-repo.md @@ -22,7 +22,7 @@ A session's boundary comes from its working directory. Inside a tracked repo it Such a session has no single workspace slug, so `_meta.scope_applied` reports `repos:N` rather than `workspace`. `repo:`, `project:`, `workspace:` and `scope:` narrow *within* the contained set; naming a repo or workspace outside it is refused with an error rather than answered empty. `repo:"*"` widens only back to the session's own repos. -A directory that neither lies inside nor contains a tracked repo still fails closed with the structured `repo_not_tracked` error — tracking the parent of your repos is not required, and doing so would index every child a second time. A configured repo that warmup has not registered yet after a daemon start is not "untracked": its sessions get `repo_loading` (retryable) until it is ready, then work in place without reconnecting — see [Startup states](mcp.md#startup-states). Over the HTTP surface (`gortex daemon start --http-addr ...`) the `/v1/graph` route accepts `?project=` and `?repo=` to narrow the dump, so a typo'd value returns an empty result for that request instead of bringing the whole index up empty. +A directory that neither lies inside nor contains a tracked repo still fails closed with the structured `repo_not_tracked` error — tracking the parent of your repos is not required, and doing so would index every child a second time. The error preserves the attempted path and points to `gortex repos` to list tracked repositories. Reconnect MCP with its working directory inside the intended repository, or track the current directory if it is a new repository. Tool-level `path` or `repo` arguments do not change this session admission check. A configured repo that warmup has not registered yet after a daemon start is not "untracked": its sessions get `repo_loading` (retryable) until it is ready, then work in place without reconnecting — see [Startup states](mcp.md#startup-states). Over the HTTP surface (`gortex daemon start --http-addr ...`) the `/v1/graph` route accepts `?project=` and `?repo=` to narrow the dump, so a typo'd value returns an empty result for that request instead of bringing the whole index up empty. ## Repository prefixes From 3d8d02841eafe1181f356144034a9128844d19ba Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 17:25:35 +0100 Subject: [PATCH 36/60] test: reproduce permanent caching of cold LSP references --- internal/mcp/refs_retry_test.go | 133 ++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 internal/mcp/refs_retry_test.go diff --git a/internal/mcp/refs_retry_test.go b/internal/mcp/refs_retry_test.go new file mode 100644 index 000000000..fd79e1870 --- /dev/null +++ b/internal/mcp/refs_retry_test.go @@ -0,0 +1,133 @@ +package mcp + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/semantic" + "github.com/zzet/gortex/internal/semantic/lsp" + "go.uber.org/zap" +) + +// Exercise the real provider and output authority without an installed language +// server. The first hierarchy response models an indexing server's empty answer. +func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int32, *atomic.Int32) { + t.Helper() + stack := newViewStack(t) + source := "fn target() {}\nfn caller() { target(); }\n" + path := filepath.Join(stack.repoRoot, "retry.rs") + if err := os.WriteFile(path, []byte(source), 0o600); err != nil { + t.Fatal(err) + } + target := viewRepoNode("repo/retry.rs::target", "target", graph.KindFunction, "repo/retry.rs", 1) + caller := viewRepoNode("repo/retry.rs::caller", "caller", graph.KindFunction, "repo/retry.rs", 2) + stack.store.AddBatch([]*graph.Node{target, caller}, nil) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + calls, connections := &atomic.Int32{}, &atomic.Int32{} + done := make(chan struct{}) + var active net.Conn + accepted := make(chan struct{}) + go func() { + defer close(done) + conn, err := ln.Accept() + if err != nil { + close(accepted) + return + } + active = conn + connections.Add(1) + close(accepted) + defer conn.Close() + reader := bufio.NewReader(conn) + for { + size := 0 + for { + line, err := reader.ReadString('\n') + if err != nil { + return + } + if line == "\r\n" { + break + } + if strings.HasPrefix(line, "Content-Length:") { + size, _ = strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(line, "Content-Length:"))) + } + } + body := make([]byte, size) + if _, err := io.ReadFull(reader, body); err != nil { + return + } + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + } + if json.Unmarshal(body, &req) != nil { + return + } + if len(req.ID) == 0 { + continue + } + var result any + switch req.Method { + case "initialize": + result = map[string]any{"capabilities": map[string]any{"callHierarchyProvider": true}} + case "textDocument/prepareCallHierarchy": + if calls.Add(1) > 1 { + result = []lsp.CallHierarchyItem{{Name: "target", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 0}, End: lsp.Position{Line: 0, Character: 14}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 0, Character: 3}, End: lsp.Position{Line: 0, Character: 9}}}} + } else { + result = []any{} + } + case "callHierarchy/incomingCalls": + result = []lsp.CallHierarchyIncomingCall{{From: lsp.CallHierarchyItem{Name: "caller", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 1}, End: lsp.Position{Line: 1, Character: 25}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 1, Character: 3}, End: lsp.Position{Line: 1, Character: 9}}}, FromRanges: []lsp.Range{{Start: lsp.Position{Line: 1, Character: 14}, End: lsp.Position{Line: 1, Character: 20}}}}} + } + response, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result}) + if _, err := fmt.Fprintf(conn, "Content-Length: %d\r\n\r\n%s", len(response), response); err != nil { + return + } + } + }() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + provider := lsp.NewProviderFromSpec(&lsp.ServerSpec{Name: "reference-fixture", Command: executable, Languages: []string{"rust"}, Daemon: true, Connect: &lsp.ConnectSpec{Network: "tcp", Address: ln.Addr().String()}}, zap.NewNop()) + stack.srv.semanticMgr = semantic.NewManager(semantic.Config{}, zap.NewNop()) + stack.srv.semanticMgr.RegisterProvider(provider) + t.Cleanup(func() { + _ = provider.Close() + _ = ln.Close() + <-accepted + if active != nil { + _ = active.Close() + } + <-done + }) + return stack, target, calls, connections +} + +func TestColdReferenceConfirmationIsNotPermanent(t *testing.T) { + stack, node, calls, _ := newReferenceRetryStack(t) + stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) + if calls.Load() != 1 { + t.Fatalf("hierarchy requests = %d, want 1", calls.Load()) + } + if entry, ok := stack.srv.refsConfirmed.Load(node.ID); ok { + if _, permanent := entry.(struct{}); permanent { + t.Fatal("cold empty response permanently confirmed the symbol") + } + } +} From 02b3e59fdf72035ba7483e77831ad5cb2b80eafc Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 17:48:55 +0100 Subject: [PATCH 37/60] fix: retry incomplete on-demand LSP references --- internal/mcp/refs_retry_test.go | 206 ++++++++++++++++++++++++++++++-- internal/mcp/server.go | 28 ++--- internal/mcp/tools_lsp.go | 79 +++++++++--- 3 files changed, 269 insertions(+), 44 deletions(-) diff --git a/internal/mcp/refs_retry_test.go b/internal/mcp/refs_retry_test.go index fd79e1870..8c72328ba 100644 --- a/internal/mcp/refs_retry_test.go +++ b/internal/mcp/refs_retry_test.go @@ -11,18 +11,21 @@ import ( "path/filepath" "strconv" "strings" + "sync" "sync/atomic" "testing" + "time" + + "go.uber.org/zap" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/semantic" "github.com/zzet/gortex/internal/semantic/lsp" - "go.uber.org/zap" ) // Exercise the real provider and output authority without an installed language // server. The first hierarchy response models an indexing server's empty answer. -func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int32, *atomic.Int32) { +func newReferenceRetryStack(t *testing.T, first string, beforeIncoming func()) (*viewStack, *graph.Node, *atomic.Int32, *atomic.Int32) { t.Helper() stack := newViewStack(t) source := "fn target() {}\nfn caller() { target(); }\n" @@ -32,7 +35,11 @@ func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int3 } target := viewRepoNode("repo/retry.rs::target", "target", graph.KindFunction, "repo/retry.rs", 1) caller := viewRepoNode("repo/retry.rs::caller", "caller", graph.KindFunction, "repo/retry.rs", 2) - stack.store.AddBatch([]*graph.Node{target, caller}, nil) + target.Language, caller.Language = "rust", "rust" + stack.store.AddBatch([]*graph.Node{target, caller}, []*graph.Edge{{ + From: caller.ID, To: target.ID, Kind: graph.EdgeCalls, + FilePath: caller.FilePath, Line: 2, Origin: graph.OriginASTResolved, + }}) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -82,19 +89,35 @@ func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int3 continue } var result any + var rpcError any switch req.Method { case "initialize": result = map[string]any{"capabilities": map[string]any{"callHierarchyProvider": true}} case "textDocument/prepareCallHierarchy": - if calls.Add(1) > 1 { + attempt := calls.Add(1) + if first == "error" && attempt == 1 { + rpcError = map[string]any{"code": -32603, "message": "workspace loading"} + } + if first != "zero" && (attempt > 1 || first == "positive" || first == "incoming-error") { result = []lsp.CallHierarchyItem{{Name: "target", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 0}, End: lsp.Position{Line: 0, Character: 14}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 0, Character: 3}, End: lsp.Position{Line: 0, Character: 9}}}} } else { result = []any{} } case "callHierarchy/incomingCalls": + if first == "incoming-error" && calls.Load() == 1 { + rpcError = map[string]any{"code": -32603, "message": "workspace loading"} + } + if beforeIncoming != nil { + beforeIncoming() + } result = []lsp.CallHierarchyIncomingCall{{From: lsp.CallHierarchyItem{Name: "caller", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 1}, End: lsp.Position{Line: 1, Character: 25}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 1, Character: 3}, End: lsp.Position{Line: 1, Character: 9}}}, FromRanges: []lsp.Range{{Start: lsp.Position{Line: 1, Character: 14}, End: lsp.Position{Line: 1, Character: 20}}}}} } - response, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result}) + reply := map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result} + if rpcError != nil { + delete(reply, "result") + reply["error"] = rpcError + } + response, _ := json.Marshal(reply) if _, err := fmt.Fprintf(conn, "Content-Length: %d\r\n\r\n%s", len(response), response); err != nil { return } @@ -104,7 +127,11 @@ func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int3 if err != nil { t.Fatal(err) } - provider := lsp.NewProviderFromSpec(&lsp.ServerSpec{Name: "reference-fixture", Command: executable, Languages: []string{"rust"}, Daemon: true, Connect: &lsp.ConnectSpec{Network: "tcp", Address: ln.Addr().String()}}, zap.NewNop()) + provider := lsp.NewProviderFromSpec(&lsp.ServerSpec{ + Name: "reference-fixture", Command: executable, Languages: []string{"rust"}, + Daemon: true, NoHeavyRequests: first == "optout", + Connect: &lsp.ConnectSpec{Network: "tcp", Address: ln.Addr().String()}, + }, zap.NewNop()) stack.srv.semanticMgr = semantic.NewManager(semantic.Config{}, zap.NewNop()) stack.srv.semanticMgr.RegisterProvider(provider) t.Cleanup(func() { @@ -120,14 +147,177 @@ func newReferenceRetryStack(t *testing.T) (*viewStack, *graph.Node, *atomic.Int3 } func TestColdReferenceConfirmationIsNotPermanent(t *testing.T) { - stack, node, calls, _ := newReferenceRetryStack(t) + stack, node, calls, _ := newReferenceRetryStack(t, "empty", nil) stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) if calls.Load() != 1 { t.Fatalf("hierarchy requests = %d, want 1", calls.Load()) } if entry, ok := stack.srv.refsConfirmed.Load(node.ID); ok { - if _, permanent := entry.(struct{}); permanent { + if entry.(*referenceConfirmation).confirmed { t.Fatal("cold empty response permanently confirmed the symbol") } } } + +func TestReferenceConfirmationRecoversAfterCooldown(t *testing.T) { + for _, first := range []string{"empty", "error", "incoming-error", "positive", "zero"} { + t.Run(first, func(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, first, nil) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), node, now) + for range 3 { + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown-time.Nanosecond)) + } + if calls.Load() != 1 { + t.Fatalf("cooldown requests = %d, want 1", calls.Load()) + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + want := int32(2) + if first == "positive" { + want = 1 + } + if calls.Load() != want { + t.Fatalf("post-cooldown requests = %d, want %d", calls.Load(), want) + } + if first != "zero" { + edges := stack.store.GetInEdges(node.ID) + found := false + for _, edge := range edges { + if edge.From == "repo/retry.rs::caller" && edge.Kind == graph.EdgeCalls && edge.Origin == graph.OriginLSPResolved { + found = true + } + } + if !found { + t.Fatalf("retry did not promote caller edge: %+v", edges) + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(2*referenceRetryCooldown)) + if calls.Load() != want { + t.Fatal("positive confirmation was not cached") + } + } + if first == "zero" { + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(2*referenceRetryCooldown)) + if calls.Load() != 3 { + t.Fatal("valid zero callers lost eventual retry eligibility") + } + } + if connections.Load() != 1 { + t.Fatalf("provider connections = %d, want 1", connections.Load()) + } + }) + } +} + +func TestReferenceConfirmationClaimBoundsConcurrentQueries(t *testing.T) { + stack, node, calls, _ := newReferenceRetryStack(t, "empty", nil) + now := time.Now() + for _, stamp := range []time.Time{now, now.Add(referenceRetryCooldown)} { + var queries sync.WaitGroup + for range 16 { + queries.Go(func() { stack.srv.confirmSymbolRefsAt(context.Background(), node, stamp) }) + } + queries.Wait() + } + if calls.Load() != 2 { + t.Fatalf("concurrent queries made %d hierarchy requests, want 2", calls.Load()) + } +} + +func TestReferenceConfirmationRefusedCompletionRetries(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + stack, node, calls, _ := newReferenceRetryStack(t, "positive", func() { once.Do(func() { close(started); <-release }) }) + now := time.Now() + done := make(chan struct{}) + go func() { defer close(done); stack.srv.confirmSymbolRefsAt(context.Background(), node, now) }() + <-started + out, err := BeginBaseEnrichment(context.Background(), stack.srv.outputGenerationAuthority(), stack.srv.graph, EnrichProducerSymbolRefs, node.RepoPrefix, stack.repoRoot) + close(release) + <-done + if err != nil { + t.Fatal(err) + } + out.Abandon() + entry, _ := stack.srv.refsConfirmed.Load(node.ID) + if entry.(*referenceConfirmation).confirmed { + t.Fatal("superseded output was permanently confirmed") + } + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + if calls.Load() != 2 { + t.Fatalf("refused completion made %d requests, want 2", calls.Load()) + } +} + +func TestReferenceConfirmationResetCannotReviveInflightEntry(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + stack, node, _, _ := newReferenceRetryStack(t, "positive", func() { close(started); <-release }) + done := make(chan struct{}) + go func() { defer close(done); stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) }() + <-started + stack.srv.resetConfirmedRefs() + // A new epoch's entry must survive the previous attempt finishing. + replacement := stack.srv.claimReferenceConfirmation(node.ID, time.Now()) + close(release) + <-done + if entry, ok := stack.srv.refsConfirmed.Load(node.ID); !ok || entry != replacement { + t.Fatal("old completion replaced the new epoch's claim") + } +} + +func TestReferenceConfirmationClaimsAreIndependent(t *testing.T) { + srv := &Server{} + now := time.Now() + first := srv.claimReferenceConfirmation("first", now) + if first == nil || srv.claimReferenceConfirmation("second", now) == nil { + t.Fatal("different symbol could not claim") + } + if srv.claimReferenceConfirmation("first", now) != nil { + t.Fatal("active symbol claimed twice") + } + srv.resetConfirmedRefs() + if srv.refsConfirmed.CompareAndSwap("first", first, &referenceConfirmation{confirmed: true}) { + t.Fatal("old claim revived after reset") + } + if srv.claimReferenceConfirmation("first", now) == nil { + t.Fatal("reset did not allow a new attempt") + } +} + +func TestReferenceConfirmationRoutedRefusalLeavesPrimaryEligible(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, "empty", nil) + stack.srv.confirmSymbolRefsOnDemand(routedEnrichmentCtx(t, stack), node) + if calls.Load() != 0 || connections.Load() != 0 { + t.Fatal("routed refusal reached the language server") + } + if _, ok := stack.srv.refsConfirmed.Load(node.ID); ok { + t.Fatal("routed refusal retained a primary claim") + } + stack.srv.confirmSymbolRefsOnDemand(context.Background(), node) + if calls.Load() != 1 { + t.Fatalf("primary request made %d calls, want 1", calls.Load()) + } +} + +func TestReferenceConfirmationHonorsProviderOptOut(t *testing.T) { + stack, node, calls, connections := newReferenceRetryStack(t, "optout", nil) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), node, now) + stack.srv.confirmSymbolRefsAt(context.Background(), node, now.Add(referenceRetryCooldown)) + if calls.Load() != 0 { + t.Fatalf("opted-out provider made %d hierarchy requests", calls.Load()) + } + if connections.Load() != 1 { + t.Fatalf("opted-out provider connections = %d, want 1", connections.Load()) + } +} + +func TestReferenceConfirmationProviderFailureBacksOff(t *testing.T) { + stack := newLSPEnrichmentStack(t) + now := time.Now() + stack.srv.confirmSymbolRefsAt(context.Background(), stack.node, now) + standing := stack.standingReceipt(t, EnrichProducerSymbolRefs) + stack.srv.confirmSymbolRefsAt(context.Background(), stack.node, now.Add(referenceRetryCooldown-time.Nanosecond)) + if err := standing.Complete(); err != nil { + t.Fatalf("provider failure retried inside cooldown: %v", err) + } +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index d0e52bdc7..79ca8fb10 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -374,12 +374,10 @@ type Server struct { equivalence *search.EquivalenceTable contractRegistry *contracts.Registry semanticMgr *semantic.Manager - // refsConfirmed is the lazy-enrichment ledger: symbol IDs whose incoming - // references have been confirmed on demand (via ConfirmSymbolRefs) this - // daemon session, so a repeat usages query serves from the confirmed graph - // instead of re-spawning the language server. An entry is recorded after - // the first attempt (success or empty) to bound per-query LSP work. - refsConfirmed sync.Map // symbolID → struct{} + // refsConfirmed bounds on-demand incoming-reference work per symbol. Entries + // hold an active claim, a retry deadline, or an accepted positive result for + // this analysis epoch. Failed/empty attempts never permanently suppress retry. + refsConfirmed sync.Map // symbolID → *referenceConfirmation feedback *feedbackManager notes *notesManager memories *memoryManager @@ -3229,22 +3227,16 @@ func ScheduleMemoryReleaseAfterBurst(logger *zap.Logger, reason string) { go runScheduledMCPMemoryRelease(logger, reason) } -// resetConfirmedRefs clears the lazy-enrichment ledger (see the -// refsConfirmed field). sync.Map has no clear-all, so this ranges and -// deletes; it is safe against concurrent confirmSymbolRefsOnDemand -// readers because a racing miss just re-confirms the symbol, which is -// idempotent — it re-lands the same lsp_resolved edges. +// resetConfirmedRefs starts a new reference-confirmation epoch. Finishers use +// pointer CAS, so an attempt admitted before Clear cannot revive its old entry. func (s *Server) resetConfirmedRefs() { - s.refsConfirmed.Range(func(k, _ any) bool { - s.refsConfirmed.Delete(k) - return true - }) + s.refsConfirmed.Clear() } // resetTestIndexProbe clears the per-repo-prefix test-symbol probe (see the -// testIndexProbe field) so the next review re-scans the rebuilt graph. Like -// resetConfirmedRefs it ranges-and-deletes because sync.Map has no clear-all, -// and it is safe against a concurrent testLangsIndexed reader: a racing miss +// testIndexProbe field) so the next review re-scans the rebuilt graph. +// Ranging and deleting is safe against a concurrent testLangsIndexed reader: +// a racing miss // just re-scans, which is idempotent. func (s *Server) resetTestIndexProbe() { s.testIndexProbe.Range(func(k, _ any) bool { diff --git a/internal/mcp/tools_lsp.go b/internal/mcp/tools_lsp.go index f525fb29b..b340d8752 100644 --- a/internal/mcp/tools_lsp.go +++ b/internal/mcp/tools_lsp.go @@ -64,7 +64,8 @@ func (s *Server) enrichNodeOnDemand(ctx context.Context, node *graph.Node) { // point: a request reading a checkout of its own has no writable output, so it // must not spawn a server or hover any bytes at all. Returns a nil output for // that case, for a path that cannot be placed on disk, and when no server -// serves the language. +// serves the language. A nonempty path with nil output means admission succeeded +// but provider resolution failed; callers can back off that failure separately. func (s *Server) lspEnrichmentTarget( ctx context.Context, producer string, node *graph.Node, ) (string, *EnrichmentOutput, *lsp.Provider) { @@ -95,7 +96,7 @@ func (s *Server) lspEnrichmentTarget( provider, _, err := s.lspProviderForPath(absPath) if err != nil || provider == nil { out.Abandon() - return "", nil, nil + return absPath, nil, nil } return absPath, out, provider } @@ -110,37 +111,79 @@ func lspEnrichmentCapability(producer string) graphview.CapabilityID { return graphview.CapLSPHover } -// confirmSymbolRefsOnDemand faults in a callable symbol's INCOMING references -// on a usages-shaped query (find_usages / get_callers) when the synchronous -// LSP sweep is off. It lazy-spawns the language server, confirms this one -// symbol's callers via call hierarchy, and lands the lsp_resolved edges into -// the graph so the query answer — and every repeat — reflects compiler-grade -// usages accuracy. Idempotent per session via the refsConfirmed ledger; a -// no-op for non-callable nodes, when no call-hierarchy server serves the -// language, or when already confirmed. +// referenceConfirmation is immutable once published. An entry with neither a +// deadline nor confirmed set is an active claim; its pointer is the finish token. +// Replacing that exact token prevents an old attempt from restoring a reset entry. +type referenceConfirmation struct { + retryAfter time.Time + confirmed bool +} + +const referenceRetryCooldown = time.Minute + +func (s *Server) claimReferenceConfirmation(id string, now time.Time) *referenceConfirmation { + for { + previous, loaded := s.refsConfirmed.Load(id) + if loaded { + entry := previous.(*referenceConfirmation) + if entry.confirmed || entry.retryAfter.IsZero() || now.Before(entry.retryAfter) { + return nil + } + claim := &referenceConfirmation{} + if s.refsConfirmed.CompareAndSwap(id, entry, claim) { + return claim + } + } else { + claim := &referenceConfirmation{} + if _, loaded := s.refsConfirmed.LoadOrStore(id, claim); !loaded { + return claim + } + } + } +} + +// confirmSymbolRefsOnDemand faults in compiler-confirmed incoming calls for a +// callable symbol. Positive, accepted results last for the analysis epoch. Empty +// results (including a still-indexing server), errors, and refused completions +// retry after a short cooldown, reusing the provider's existing client. func (s *Server) confirmSymbolRefsOnDemand(ctx context.Context, node *graph.Node) { + s.confirmSymbolRefsAt(ctx, node, time.Now()) +} + +func (s *Server) confirmSymbolRefsAt(ctx context.Context, node *graph.Node, now time.Time) { if node == nil || s.semanticMgr == nil || s.graph == nil { return } if node.Kind != graph.KindFunction && node.Kind != graph.KindMethod { return } - if _, done := s.refsConfirmed.Load(node.ID); done { + claim := s.claimReferenceConfirmation(node.ID, now) + if claim == nil { return } - _, out, provider := s.lspEnrichmentTarget(ctx, EnrichProducerSymbolRefs, node) + path, out, provider := s.lspEnrichmentTarget(ctx, EnrichProducerSymbolRefs, node) if out == nil { + // Admission refusal, particularly a routed view, must leave the primary + // symbol eligible. Only remove our own claim, never a later epoch's entry. + if path == "" { + s.refsConfirmed.CompareAndDelete(node.ID, claim) + } else { + // Output was admitted, but its provider failed to start or resolve. + s.refsConfirmed.CompareAndSwap(node.ID, claim, &referenceConfirmation{retryAfter: now.Add(referenceRetryCooldown)}) + } return } - if _, err := provider.ConfirmSymbolRefs(out.Store, out.Root, node); err != nil { + count, err := provider.ConfirmSymbolRefs(out.Store, out.Root, node) + accepted := false + if err != nil { out.Abandon() } else { - _ = out.Complete() + accepted = out.Complete() == nil } - // Record after the attempt (even on 0/err) so a query doesn't re-spawn the - // server every call; a file re-index clears staleness through the normal - // restub path. Durable-ledger + retry semantics are the fuller version. - s.refsConfirmed.Store(node.ID, struct{}{}) + s.refsConfirmed.CompareAndSwap(node.ID, claim, &referenceConfirmation{ + confirmed: count > 0 && err == nil && accepted, + retryAfter: now.Add(referenceRetryCooldown), + }) } // registerLSPTools wires the LSP-action MCP surface: From db5580728ed6ca5486ac99337c68d1aca10c5d81 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:00:27 +0100 Subject: [PATCH 38/60] test(bench): preserve each source checkout byte snapshot --- bench/load/edit_probe_route_phase_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/bench/load/edit_probe_route_phase_test.go b/bench/load/edit_probe_route_phase_test.go index 5f62ea986..320303e0d 100644 --- a/bench/load/edit_probe_route_phase_test.go +++ b/bench/load/edit_probe_route_phase_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -144,10 +145,21 @@ func TestEditProbeSourceRepositoryIdentityDoesNotCountLinkedViewsTwice(t *testin t.Fatal(err) } linkedPath := filepath.Join(t.TempDir(), "linked-source") - cmd := exec.CommandContext(t.Context(), "git", "-C", primary.Source, "-c", "core.hooksPath="+os.DevNull, "-c", "gc.auto=0", "-c", "maintenance.auto=false", "worktree", "add", "--detach", linkedPath, primary.Commit) + cmd := exec.CommandContext(t.Context(), "git", "-C", primary.Source, "-c", "core.hooksPath="+os.DevNull, "-c", "gc.auto=0", "-c", "maintenance.auto=false", "-c", "core.autocrlf=true", "-c", "core.eol=crlf", "worktree", "add", "--detach", linkedPath, primary.Commit) if data, err := cmd.CombinedOutput(); err != nil { t.Fatalf("own source worktree: %v %s", err, data) } + // Git can materialize the same tree with different line endings. Capture + // each checkout before admission; equality to the primary's bytes would + // falsely blame a read-only query for Git's checkout normalization. + linkedBefore, err := os.ReadFile(filepath.Join(linkedPath, primary.File)) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(linkedBefore, []byte("\r\n")) || bytes.Equal(linkedBefore, before) { + t.Fatal("fixture must distinguish the linked CRLF checkout from the primary LF source") + } + originals := map[string][]byte{primary.Source: before, linkedPath: linkedBefore} linked := primary linked.Source, linked.Label = linkedPath, "linked-source" if err := validateEditProbeSourceRepos(t.Context(), []Corpus{primary, linked}, t.TempDir()); err == nil || !strings.Contains(err.Error(), "distinct source repositories") { @@ -159,8 +171,8 @@ func TestEditProbeSourceRepositoryIdentityDoesNotCountLinkedViewsTwice(t *testin } for _, c := range []Corpus{primary, linked} { after, err := os.ReadFile(filepath.Join(c.Source, c.File)) - if err != nil || string(after) != string(before) { - t.Fatalf("read-only identity admission mutated source bytes: %v", err) + if err != nil || !bytes.Equal(after, originals[c.Source]) { + t.Fatalf("read-only identity admission mutated %s source bytes: %v", c.Label, err) } } } From f0712300e68d850ce854d0f1a449e6c24fc59322 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:12:44 +0100 Subject: [PATCH 39/60] feat(sqlite): report every cold bulk admission outcome --- .../bulk_begin_observation_test.go | 94 +++++++++++++++++++ internal/graph/store_sqlite/bulk_load.go | 61 +++++++----- 2 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 internal/graph/store_sqlite/bulk_begin_observation_test.go diff --git a/internal/graph/store_sqlite/bulk_begin_observation_test.go b/internal/graph/store_sqlite/bulk_begin_observation_test.go new file mode 100644 index 000000000..39a0cd577 --- /dev/null +++ b/internal/graph/store_sqlite/bulk_begin_observation_test.go @@ -0,0 +1,94 @@ +package store_sqlite + +import ( + "bytes" + "log" + "strings" + "sync" + "testing" + + "github.com/zzet/gortex/internal/graph" +) + +type bulkAdmissionLogBuffer struct { + mu sync.Mutex + bytes.Buffer +} + +func (b *bulkAdmissionLogBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.Buffer.Write(data) +} +func (b *bulkAdmissionLogBuffer) reset() { b.mu.Lock(); defer b.mu.Unlock(); b.Reset() } +func (b *bulkAdmissionLogBuffer) text() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.String() +} + +func TestBulkBeginReportsEveryAdmissionOutcome(t *testing.T) { + s, _ := openTempStore(t) + var output bulkAdmissionLogBuffer + prior := log.Writer() + log.SetOutput(&output) + t.Cleanup(func() { log.SetOutput(prior) }) + expect := func(window, outcome, reason string, begin func()) { + t.Helper() + output.reset() + begin() + lines := strings.Split(strings.TrimSpace(output.text()), "\n") + var admissions []string + for _, line := range lines { + if strings.Contains(line, "bulk begin ") { + admissions = append(admissions, line) + } + } + want := "window=" + window + " outcome=" + outcome + " reason=" + reason + if len(admissions) != 1 || !strings.Contains(admissions[0], want) { + t.Fatalf("admission logs=%q, want exactly one %q", admissions, want) + } + } + expect("standalone", "engaged", "cold_store", s.BeginBulkLoad) + expect("standalone", "nested_noop", "active_window", s.BeginBulkLoad) + expect("coordinated", "nested_noop", "active_window", func() { s.BeginCoordinatedBulkLoad() }) + if err := s.FlushBulk(); err != nil { + t.Fatal(err) + } + s.AddBatch([]*graph.Node{{ID: "repo/a.go::A", Name: "A", Kind: graph.KindFunction}}, nil) + expect("standalone", "refused_not_cold", "nodes", s.BeginBulkLoad) + expect("coordinated", "refused_not_cold", "nodes", func() { s.BeginCoordinatedBulkLoad() }) + memory, err := Open(":memory:") + if err != nil { + t.Fatal(err) + } + defer memory.Close() + expect("standalone", "memory_store", "memory_store", memory.BeginBulkLoad) + expect("coordinated", "memory_store", "memory_store", func() { memory.BeginCoordinatedBulkLoad() }) + for _, predicate := range []string{"edges", "file_mtimes", "repo_index_state"} { + warm, _ := openTempStore(t) + switch predicate { + case "edges": + nodes, edges := bulkFixture(2, 1) + warm.AddBatch(nodes, edges) + if _, err := warm.writerDB.Exec("DELETE FROM nodes"); err != nil { + t.Fatal(err) + } + case "file_mtimes": + if err := warm.SetFileMtime("repo", "a.go", 1); err != nil { + t.Fatal(err) + } + case "repo_index_state": + if err := warm.SetRepoIndexState(graph.RepoIndexState{RepoPrefix: "repo"}); err != nil { + t.Fatal(err) + } + } + expect("coordinated", "refused_not_cold", predicate, func() { warm.BeginCoordinatedBulkLoad() }) + } + generation, _ := openTempStore(t) + if active, err := generation.BeginGenerationBulkLoad(42); err != nil || !active { + t.Fatalf("generation active=%v err=%v", active, err) + } + expect("standalone", "refused_generation_window", "generation_window", generation.BeginBulkLoad) + expect("coordinated", "refused_generation_window", "generation_window", func() { generation.BeginCoordinatedBulkLoad() }) +} diff --git a/internal/graph/store_sqlite/bulk_load.go b/internal/graph/store_sqlite/bulk_load.go index c9fd33b04..3ffbbd75f 100644 --- a/internal/graph/store_sqlite/bulk_load.go +++ b/internal/graph/store_sqlite/bulk_load.go @@ -383,10 +383,8 @@ func (s *Store) activeWriteConnLocked(ctx context.Context) (*sql.Conn, func(), e func (s *Store) BeginBulkLoad() { s.writeMu.Lock() defer s.writeMu.Unlock() - if s.coordinatedBulkLoad { - return - } - s.beginBulkLoadLocked() + outcome, reason := s.beginBulkLoadLocked() + log.Printf("store_sqlite: bulk begin window=standalone outcome=%s reason=%s", outcome, reason) } // BeginCoordinatedBulkLoad opens one outer cold-load window around a set of @@ -398,11 +396,9 @@ func (s *Store) BeginBulkLoad() { func (s *Store) BeginCoordinatedBulkLoad() bool { s.writeMu.Lock() defer s.writeMu.Unlock() - if s.coordinatedBulkLoad || s.bulkConn != nil { - return false - } - s.beginBulkLoadLocked() - if s.bulkConn == nil { + outcome, reason := s.beginBulkLoadLocked() + log.Printf("store_sqlite: bulk begin window=coordinated outcome=%s reason=%s", outcome, reason) + if outcome != "engaged" { return false } s.coordinatedBulkLoad = true @@ -410,14 +406,17 @@ func (s *Store) BeginCoordinatedBulkLoad() bool { return true } -func (s *Store) beginBulkLoadLocked() { - - // Re-entrancy / non-disk guard: a second BeginBulkLoad without an - // intervening FlushBulk, or an in-memory store, stays a no-op. - if s.bulkConn != nil || isMemoryPath(s.dbPath) { - return +func (s *Store) beginBulkLoadLocked() (outcome, reason string) { + if s.generationBulkLoad != 0 { + return "refused_generation_window", "generation_window" } - + if s.coordinatedBulkLoad || s.bulkConn != nil { + return "nested_noop", "active_window" + } + if isMemoryPath(s.dbPath) { + return "memory_store", "memory_store" + } + outcome, reason = "refused_setup", "writer_connection" ctx := context.Background() conn, err := s.writerDB.Conn(ctx) if err != nil { @@ -427,10 +426,11 @@ func (s *Store) beginBulkLoadLocked() { // Nodes alone are not a cold-store proof. A degraded warm store can have // zero nodes while retaining millions of edges, and sidecar receipts prove // an incremental lifecycle even when both graph tables happen to be empty. - if !coldGraphStoreEmpty(ctx, conn) { + if predicate := coldGraphStoreRefusal(ctx, conn); predicate != "" { _ = conn.Close() - return + return "refused_not_cold", predicate } + reason = "pragma_setup" // Capture prior pragma values so FlushBulk (and every early-return / // error path) can restore them. If they can't be read, don't engage — @@ -500,6 +500,7 @@ func (s *Store) beginBulkLoadLocked() { // The bulk path changes durability and secondary-index maintenance outside // the ordinary row mutation protocol. Active receipts therefore fail closed. s.markMutationReceiptsIncompleteLocked() + return "engaged", "cold_store" } // ErrGenerationBulkLoadPopulated refuses a generation-scoped bulk window over @@ -1560,14 +1561,24 @@ func shouldBackoffBulkRowCheckpoint(err error) bool { // rows has been indexed before, whichever view wrote them, so it is not the // cold store this fast path is for. func coldGraphStoreEmpty(ctx context.Context, conn *sql.Conn) bool { - var empty int + return coldGraphStoreRefusal(ctx, conn) == "" +} + +// One bounded probe retains the cold-store proof and identifies its first +// failed predicate without counting or hydrating any graph rows. +func coldGraphStoreRefusal(ctx context.Context, conn *sql.Conn) string { + var reason string err := conn.QueryRowContext(ctx, ` -SELECT NOT EXISTS(SELECT 1 FROM nodes WHERE view_gen = ?) - AND NOT EXISTS(SELECT 1 FROM edges WHERE view_gen = ?) - AND NOT EXISTS(SELECT 1 FROM file_mtimes) - AND NOT EXISTS(SELECT 1 FROM repo_index_state)`, - baseViewGeneration, baseViewGeneration).Scan(&empty) - return err == nil && empty == 1 +SELECT CASE + WHEN EXISTS(SELECT 1 FROM nodes WHERE view_gen = ?) THEN 'nodes' + WHEN EXISTS(SELECT 1 FROM edges WHERE view_gen = ?) THEN 'edges' + WHEN EXISTS(SELECT 1 FROM file_mtimes) THEN 'file_mtimes' + WHEN EXISTS(SELECT 1 FROM repo_index_state) THEN 'repo_index_state' + ELSE '' END`, baseViewGeneration, baseViewGeneration).Scan(&reason) + if err != nil { + return "cold_probe_error" + } + return reason } // WriterPragmaForTest reads an integer PRAGMA off the store's writer From a358445990b65aa97ffb2652c33f3a9536a86fff Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:12:44 +0100 Subject: [PATCH 40/60] feat(indexer): expose the last full drain and owed index repair --- cmd/gortex/daemon_controller.go | 8 +++ cmd/gortex/daemon_warmup_bulk_test.go | 49 +++++++++++++++++++ docs/mcp.md | 12 +++++ internal/daemon/proto.go | 7 ++- internal/graph/first_index_drain.go | 21 ++++++++ .../graph/store_sqlite/first_index_drain.go | 48 ++++++++++++++++++ .../store_sqlite/first_index_drain_test.go | 41 ++++++++++++++++ internal/graph/store_sqlite/store.go | 3 ++ internal/graph/store_sqlite/store_purge.go | 11 +++++ .../indexer/edit_delta_capability_test.go | 5 ++ internal/indexer/first_index_bulk_test.go | 16 ++++++ internal/indexer/indexer.go | 40 +++++++++++++++ .../mcp/index_health_planner_stats_test.go | 23 +++++++++ internal/mcp/tools_enhancements.go | 7 +++ 14 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 internal/graph/first_index_drain.go create mode 100644 internal/graph/store_sqlite/first_index_drain.go create mode 100644 internal/graph/store_sqlite/first_index_drain_test.go diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index df8ec5892..14d526e61 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -1782,6 +1782,14 @@ func storageStatusFor(g graph.Store) *daemon.StorageStatus { CloseCheckpointEstimateMS: estimate.Milliseconds(), WALReclaim: walReclaimStatus(reporter.WALReclaimStats()), } + if reporter, ok := g.(interface{ DenseIndexesMissing() []string }); ok { + out.DenseIndexesMissing = reporter.DenseIndexesMissing() + } + if reporter, ok := g.(interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + }); ok { + out.LastFirstIndexDrain = reporter.LastFirstIndexDrains() + } if policy, ok := g.(interface{ CloseCheckpointPolicy() string }); ok { out.CloseCheckpointPolicy = policy.CloseCheckpointPolicy() } diff --git a/cmd/gortex/daemon_warmup_bulk_test.go b/cmd/gortex/daemon_warmup_bulk_test.go index 412d9bf9b..61b3980ab 100644 --- a/cmd/gortex/daemon_warmup_bulk_test.go +++ b/cmd/gortex/daemon_warmup_bulk_test.go @@ -2,9 +2,13 @@ package main import ( "context" + "encoding/json" "errors" + "io" + "log" "os" "path/filepath" + "strings" "testing" "time" @@ -14,12 +18,28 @@ import ( "go.uber.org/zap/zaptest/observer" "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" ) +// The existing missing-index log is emitted after the atomic record is installed +// and before repair is scheduled. Observe status at that causal boundary instead +// of racing a successful background repair after warmup returns. +type warmupSealStatusProbe struct { + output io.Writer + missing func() +} + +func (p warmupSealStatusProbe) Write(data []byte) (int, error) { + if strings.Contains(string(data), "store_sqlite: dense indexes missing after bulk window close") { + p.missing() + } + return p.output.Write(data) +} + func TestDrainGoModAndFinalizeWarmupBulkOrdersDrainBeforeFinalize(t *testing.T) { var order []string wantErr := errors.New("finalize failed") @@ -95,7 +115,22 @@ func TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow(t *testing.T) { return nil }) + missingStatus := make(chan []string, 1) + priorOutput := log.Writer() + log.SetOutput(warmupSealStatusProbe{output: priorOutput, missing: func() { + select { + case missingStatus <- storageStatusFor(store).DenseIndexesMissing: + default: + } + }}) + defer log.SetOutput(priorOutput) _, _ = warmupDaemonState(state, logger, func() {}) + select { + case missing := <-missingStatus: + assert.Contains(t, missing, "nodes_by_kind", "status reads the existing missing-index record before repair is scheduled") + default: + t.Error("the failed seal never published its missing-index boundary") + } restore() require.NotEmpty(t, observed.FilterMessage("daemon: coordinated cold bulk-load finalize failed").All(), "the fixture must fail the coordinated window's final seal") @@ -113,3 +148,17 @@ func TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow(t *testing.T) { } assert.NotEqual(t, "dense_indexes_missing", health.Reason, "the repair must rebuild the index the seal left out") } + +func TestStorageStatusReportsFirstIndexDrainWithoutInventingEmptyResult(t *testing.T) { + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "status.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + empty, err := json.Marshal(storageStatusFor(store)) + require.NoError(t, err) + assert.NotContains(t, string(empty), "last_first_index_drain") + store.RecordFirstIndexDrain("repo", graph.FirstIndexDrain{Window: "coordinated", Nodes: 7, Edges: 9}) + response := storageStatusFor(store) + require.NotNil(t, response) + assert.Equal(t, 7, response.LastFirstIndexDrain["repo"].Nodes) + assert.Equal(t, "coordinated", response.LastFirstIndexDrain["repo"].Window) +} diff --git a/docs/mcp.md b/docs/mcp.md index 167a6157f..61cc4028f 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -774,3 +774,15 @@ Read-only, URI-addressable, no args. Clients that speak resources can `resources | `pre_commit` | Review uncommitted changes — shows changed symbols, blast radius, risk level, affected tests | | `orientation` | Orient in an unfamiliar codebase — graph stats, communities, execution flows, key symbols | | `safe_to_change` | Analyze whether it's safe to change specific symbols — blast radius, edit plan, affected tests | + +`index_health` includes `last_first_index_drain` after a full shadow index has +persisted, keyed by repository prefix. Daemon status exposes the same snapshot +under `storage`, together with `dense_indexes_missing` when a failed seal owes +index repair. A drain records its actual window kind, node/edge counts, drain +and seal-call durations, completion time and any seal error. A coordinated +window seals at its outer boundary, so its per-repository seal call can be a +no-op. `wal_peak_scope=shared_store_drain_chunk_samples` identifies physical WAL +observations at the start, bounded graph/FTS chunks and seal boundaries. This +is a sampled peak on the shared store; concurrent repository work can contribute +and a peak between samples can be missed. Missing observations +are omitted, and these process-local snapshots reset when the store reopens. diff --git a/internal/daemon/proto.go b/internal/daemon/proto.go index 1e02423bc..143c8e491 100644 --- a/internal/daemon/proto.go +++ b/internal/daemon/proto.go @@ -5,6 +5,7 @@ package daemon import ( "encoding/json" "fmt" + "github.com/zzet/gortex/internal/graph" "io" "time" ) @@ -525,8 +526,10 @@ type PublicationPhaseOffset struct { // StorageStatus is the store's write-ahead-log state for daemon status. type StorageStatus struct { - DBBytes int64 `json:"db_bytes"` - WALBytes int64 `json:"wal_bytes"` + DenseIndexesMissing []string `json:"dense_indexes_missing,omitempty"` + LastFirstIndexDrain map[string]graph.FirstIndexDrain `json:"last_first_index_drain,omitempty"` + DBBytes int64 `json:"db_bytes"` + WALBytes int64 `json:"wal_bytes"` // WALPendingFrames is the backlog not yet copied into the database // (mxFrame - nBackfill), -1 when the wal-index could not be read. WALPendingFrames int64 `json:"wal_pending_frames"` diff --git a/internal/graph/first_index_drain.go b/internal/graph/first_index_drain.go new file mode 100644 index 000000000..7f1fc1e24 --- /dev/null +++ b/internal/graph/first_index_drain.go @@ -0,0 +1,21 @@ +package graph + +import "time" + +// FirstIndexDrain is the last full shadow persisted for one repository during +// this store's lifetime. It is an observation, not a readiness or durability +// verdict. A coordinated window defers its final seal to the outer owner. +type FirstIndexDrain struct { + Window string `json:"window"` + Nodes int `json:"nodes"` + Edges int `json:"edges"` + DrainMS float64 `json:"drain_ms"` + SealMS float64 `json:"seal_ms"` + CompletedAt time.Time `json:"completed_at"` + Error string `json:"error,omitempty"` + WALPeakBytes int64 `json:"wal_peak_bytes,omitempty"` + WALPeakObserved bool `json:"wal_peak_observed"` + // WAL peak is sampled at bounded drain/FTS chunks and seal boundaries. + // The WAL is shared: overlapping work may contribute to these observations. + WALPeakScope string `json:"wal_peak_scope"` +} diff --git a/internal/graph/store_sqlite/first_index_drain.go b/internal/graph/store_sqlite/first_index_drain.go new file mode 100644 index 000000000..8bc764897 --- /dev/null +++ b/internal/graph/store_sqlite/first_index_drain.go @@ -0,0 +1,48 @@ +package store_sqlite + +import "github.com/zzet/gortex/internal/graph" + +// BulkLoadWindow snapshots the owner while the writer is between drain chunks. +func (s *Store) BulkLoadWindow() string { + s.writeMu.Lock() + defer s.writeMu.Unlock() + switch { + case s.generationBulkLoad != 0: + return "generation_noop" + case s.coordinatedBulkLoad: + return "coordinated_noop" + case s.bulkConn != nil: + return "cold" + default: + return "none" + } +} + +// RecordFirstIndexDrain retains one value per repository, shared by every +// generation handle and temporary indexer over this open store. +func (s *Store) RecordFirstIndexDrain(repo string, observation graph.FirstIndexDrain) { + s.firstIndexDrainsMu.Lock() + defer s.firstIndexDrainsMu.Unlock() + if s.firstIndexDrains == nil { + s.firstIndexDrains = make(map[string]graph.FirstIndexDrain) + } + s.firstIndexDrains[repo] = observation +} + +// LastFirstIndexDrains does no SQL or writer admission. Return a detached map +// so status and cached health payloads cannot mutate or race the live record. +func (s *Store) LastFirstIndexDrains() map[string]graph.FirstIndexDrain { + s.firstIndexDrainsMu.Lock() + defer s.firstIndexDrainsMu.Unlock() + if len(s.firstIndexDrains) == 0 { + return nil + } + out := make(map[string]graph.FirstIndexDrain, len(s.firstIndexDrains)) + for repo, observation := range s.firstIndexDrains { + out[repo] = observation + } + return out +} + +// DenseIndexesMissing reports the existing atomic repair record without SQL. +func (s *Store) DenseIndexesMissing() []string { return s.denseIndexesMissingNames() } diff --git a/internal/graph/store_sqlite/first_index_drain_test.go b/internal/graph/store_sqlite/first_index_drain_test.go new file mode 100644 index 000000000..76bfe4580 --- /dev/null +++ b/internal/graph/store_sqlite/first_index_drain_test.go @@ -0,0 +1,41 @@ +package store_sqlite + +import ( + "testing" + "time" + + "github.com/zzet/gortex/internal/graph" +) + +func TestFirstIndexDrainSnapshotAvoidsWriterAndCallerMutation(t *testing.T) { + s, _ := openTempStore(t) + s.RecordFirstIndexDrain("repo", graph.FirstIndexDrain{Window: "standalone", Nodes: 3}) + s.writeMu.Lock() + result := make(chan map[string]graph.FirstIndexDrain, 1) + go func() { result <- s.LastFirstIndexDrains() }() + var snapshot map[string]graph.FirstIndexDrain + select { + case snapshot = <-result: + s.writeMu.Unlock() + case <-time.After(5 * time.Second): + s.writeMu.Unlock() + t.Fatal("status snapshot queued behind writer") + } + delete(snapshot, "repo") + actual := s.LastFirstIndexDrains() + if actual["repo"].Nodes != 3 { + t.Fatalf("caller changed retained observation: %v", actual) + } + if err := s.RekeyRepoPrefix("repo", "renamed"); err != nil { + t.Fatal(err) + } + if got := s.LastFirstIndexDrains(); len(got) != 1 || got["renamed"].Nodes != 3 { + t.Fatalf("rekey lost or duplicated observation: %v", got) + } + if err := s.PurgeRepo("renamed"); err != nil { + t.Fatal(err) + } + if got := s.LastFirstIndexDrains(); len(got) != 0 { + t.Fatalf("purged repository retained observation: %v", got) + } +} diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index a31c15d63..a8d1ec5a9 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -66,6 +66,9 @@ type backgroundCheckpointCoordination struct { } type storeCore struct { + firstIndexDrainsMu sync.Mutex + firstIndexDrains map[string]graph.FirstIndexDrain + // baseFTSPages retains only exact ranked rows keyed by same-transaction // global FTS state and mutable base ownership revision. baseFTSPages symbolFTSPageCache diff --git a/internal/graph/store_sqlite/store_purge.go b/internal/graph/store_sqlite/store_purge.go index e3805ef9a..72ff63704 100644 --- a/internal/graph/store_sqlite/store_purge.go +++ b/internal/graph/store_sqlite/store_purge.go @@ -141,6 +141,9 @@ func (s *Store) PurgeRepo(prefix string) error { if err := tx.Commit(); err != nil { return err } + s.firstIndexDrainsMu.Lock() + delete(s.firstIndexDrains, prefix) + s.firstIndexDrainsMu.Unlock() s.finishAnalysisMutationLocked(len(ids) > 0) if changed { s.payloadInputAdminRevision.Add(1) @@ -351,6 +354,14 @@ func (s *Store) RekeyRepoPrefix(oldPrefix, newPrefix string) error { if err := tx.Commit(); err != nil { return err } + s.firstIndexDrainsMu.Lock() + if observation, ok := s.firstIndexDrains[oldPrefix]; ok { + if current, exists := s.firstIndexDrains[newPrefix]; !exists || current.CompletedAt.Before(observation.CompletedAt) { + s.firstIndexDrains[newPrefix] = observation + } + delete(s.firstIndexDrains, oldPrefix) + } + s.firstIndexDrainsMu.Unlock() if changed { s.payloadInputAdminRevision.Add(1) s.markMutationReceiptsIncompleteLocked() diff --git a/internal/indexer/edit_delta_capability_test.go b/internal/indexer/edit_delta_capability_test.go index 294dd3d83..ae8b7c639 100644 --- a/internal/indexer/edit_delta_capability_test.go +++ b/internal/indexer/edit_delta_capability_test.go @@ -34,6 +34,11 @@ var _ graph.FileIdentityBatchEvicter = (*graph.DeltaWriter)(nil) // deliberately does not implement, with the reason. The reason names what // the engine does instead. var editDeltaAbsentCapabilities = map[string]string{ + "anonymous{DBStats}": "physical durable-store file-size telemetry; full-shadow drain samples a real store, whereas DeltaWriter stages per-file changes without owning a database or WAL file", + + "anonymous{BulkLoadWindow}": "physical durable-store full-shadow window telemetry; DeltaWriter stages per-file changes and does not own a cold or generation bulk window", + "anonymous{RecordFirstIndexDrain}": "physical durable-store full-shadow completion telemetry; a per-file DeltaWriter has no full-shadow drain to record and must not forward writes to its base", + "graph.BFSAdmissionReader": "optional selected-reader traversal snapshot; the query engine retains its composed point-walk admission when a delta has no snapshot accelerator", "graph.FilteredBFSAdmissionReader": "physical indexed scoped traversal; forwarding it would discard delta ownership, so the engine keeps composed Go admission instead", "graph.KindAdjacencyReader": "optional typed adjacency accelerator; ReadKindAdjacency filters the delta's composed point reads instead of forwarding physical rows past delta ownership", diff --git a/internal/indexer/first_index_bulk_test.go b/internal/indexer/first_index_bulk_test.go index 61b4f6fd6..30855ebfc 100644 --- a/internal/indexer/first_index_bulk_test.go +++ b/internal/indexer/first_index_bulk_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" @@ -577,3 +578,18 @@ func TestFirstIndexFailureRestoresSQLiteReplacement(t *testing.T) { }) } } + +func TestFirstIndexDrainReportsWindowAndSeal(t *testing.T) { + store, _ := newFirstIndexStore(t) + core, observed := observer.New(zap.InfoLevel) + idx := newFirstIndexBulkIndexer(t, store, "repo") + idx.logger = zap.New(core) + result, err := idx.IndexCtx(t.Context(), firstIndexFixtureRepo(t)) + require.NoError(t, err) + entries := observed.FilterMessage("indexer: FlushBulk complete").All() + require.Len(t, entries, 1) + fields := entries[0].ContextMap() + assert.Equal(t, "cold", fields["window"]) + assert.Contains(t, fields, "seal_elapsed") + assert.EqualValues(t, result.NodeCount, fields["nodes"]) +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index a2e4bc488..d4ce2cb8d 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -3408,6 +3408,19 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index return } drainStart := time.Now() + var drainWALPeak int64 + var drainWALObserved bool + walReporter, hasWALReporter := diskTarget.(interface{ DBStats() (int64, int64) }) + sampleDrainWAL := func() { + if hasWALReporter { + _, walBytes := walReporter.DBStats() + if walBytes > 0 { + drainWALObserved = true + } + drainWALPeak = max(drainWALPeak, walBytes) + } + } + sampleDrainWAL() shadowNodeCount := inMemShadow.NodeCount() shadowEdgeCount := inMemShadow.EdgeCount() if !shadowEstimateReady { @@ -3466,6 +3479,10 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index return } bl.BeginBulkLoad() + bulkWindow := "unknown" + if reporter, ok := diskTarget.(interface{ BulkLoadWindow() string }); ok { + bulkWindow = reporter.BulkLoadWindow() + } // FlushBulk is a no-op under an outer coordinated window. Otherwise // it must also restore the standalone window if a drain write panics. bulkFlushPending := true @@ -3542,6 +3559,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index nodeDrainStopped = true break } + sampleDrainWAL() if retErr != nil && drainChecksCancellation(ctx) && ctx.Err() != nil { break } @@ -3562,6 +3580,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index ftsReady = false return false } + sampleDrainWAL() ftsItemCount += len(ftsItems) ftsItems = make([]graph.SymbolFTSItem, 0, min(len(nodes), persistFTSChunkRows)) ftsBytes = 0 @@ -3609,6 +3628,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index } break } + sampleDrainWAL() edgeRows := len(edges) drainPressure.afterEdgeBatch(edgeRows) } @@ -3644,6 +3664,7 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index } } + sampleDrainWAL() flushStart := time.Now() idx.logger.Info("indexer: FlushBulk start", zap.String("repo", idx.RepoPrefix()), @@ -3656,7 +3677,26 @@ func (idx *Indexer) indexCtxRaw(ctx context.Context, root string) (result *Index if ferr != nil && retErr == nil { retErr = fmt.Errorf("indexer: persist bulk graph: %w", ferr) } + sealElapsed := time.Since(flushStart) + sampleDrainWAL() + drainElapsed := time.Since(drainStart) + if recorder, ok := diskTarget.(interface { + RecordFirstIndexDrain(string, graph.FirstIndexDrain) + }); ok { + observation := graph.FirstIndexDrain{ + Window: bulkWindow, Nodes: shadowNodeCount, Edges: shadowEdgeCount, + DrainMS: float64(drainElapsed) / float64(time.Millisecond), + SealMS: float64(sealElapsed) / float64(time.Millisecond), CompletedAt: time.Now(), + WALPeakBytes: drainWALPeak, WALPeakObserved: drainWALObserved, WALPeakScope: "shared_store_drain_chunk_samples", + } + if retErr != nil { + observation.Error = retErr.Error() + } + recorder.RecordFirstIndexDrain(idx.RepoPrefix(), observation) + } idx.logger.Info("indexer: FlushBulk complete", + zap.String("window", bulkWindow), + zap.Duration("seal_elapsed", sealElapsed), zap.String("repo", idx.RepoPrefix()), zap.Duration("flush_elapsed", time.Since(flushStart)), zap.Duration("total_drain", time.Since(drainStart)), diff --git a/internal/mcp/index_health_planner_stats_test.go b/internal/mcp/index_health_planner_stats_test.go index 0245957bc..7ce1553bf 100644 --- a/internal/mcp/index_health_planner_stats_test.go +++ b/internal/mcp/index_health_planner_stats_test.go @@ -348,3 +348,26 @@ func TestIndexHealth_PlannerStatsAbsentOnBackendWithoutCapability(t *testing.T) assert.NotContains(t, payload, "planner_stats", "the in-memory graph has no planner statistics; reporting zeros would invent a defect") } + +func TestIndexHealthReportsLastFirstIndexDrain(t *testing.T) { + t.Setenv("GORTEX_STREAMING_FLUSH", "0") + t.Setenv("GORTEX_SHADOW_MAX_FILES", "1000000") + t.Setenv("GORTEX_SHADOW_MAX_BYTES", "1073741824") + srv, _ := newSQLiteBackedServer(t) + payload, err := srv.buildIndexHealthPayloadCtx(t.Context()) + require.NoError(t, err) + observations, ok := payload["last_first_index_drain"].(map[string]graph.FirstIndexDrain) + require.True(t, ok, "a completed first index must publish its actual drain observation") + require.Len(t, observations, 1) + for _, observation := range observations { + assert.Equal(t, "cold", observation.Window) + assert.Greater(t, observation.Nodes, 0) + assert.GreaterOrEqual(t, observation.DrainMS, observation.SealMS) + assert.False(t, observation.CompletedAt.IsZero()) + assert.Equal(t, "shared_store_drain_chunk_samples", observation.WALPeakScope) + } + empty, _ := newEmptySQLiteBackedServer(t) + noDrain, err := empty.buildIndexHealthPayloadCtx(t.Context()) + require.NoError(t, err) + assert.NotContains(t, noDrain, "last_first_index_drain", "absence is not a fabricated completed zero drain") +} diff --git a/internal/mcp/tools_enhancements.go b/internal/mcp/tools_enhancements.go index c3b65a58a..5132d62ee 100644 --- a/internal/mcp/tools_enhancements.go +++ b/internal/mcp/tools_enhancements.go @@ -3723,6 +3723,13 @@ func (s *Server) buildIndexHealthBasePayloadCtx(ctx context.Context) (map[string } } } + if reporter, ok := s.graph.(interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + }); ok { + if observations := reporter.LastFirstIndexDrains(); len(observations) > 0 { + result["last_first_index_drain"] = observations + } + } if prefixAuditOK { ownership := map[string]any{ "owned_code_nodes": prefixAudit.OwnedCodeNodes, From 359d21cd4fb32f176f599406ec9c3832e800687b Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:12:44 +0100 Subject: [PATCH 41/60] test(bench): add an owned standalone first-index measurement harness --- bench/load/first_index_measurement_test.go | 173 +++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 bench/load/first_index_measurement_test.go diff --git a/bench/load/first_index_measurement_test.go b/bench/load/first_index_measurement_test.go new file mode 100644 index 000000000..086c2f87a --- /dev/null +++ b/bench/load/first_index_measurement_test.go @@ -0,0 +1,173 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/graph/store_sqlite" + "github.com/zzet/gortex/internal/indexer" + "github.com/zzet/gortex/internal/parser" + "github.com/zzet/gortex/internal/parser/languages" +) + +// TestMeasureStandaloneFirstIndex is an opt-in, single-build harness. Run the +// identical harness on the control and candidate with an owned clean corpus; +// compare reports separately. It never grants a ratio from a single run. +func TestMeasureStandaloneFirstIndex(t *testing.T) { + corpus := os.Getenv("GORTEX_FIRST_INDEX_CORPUS") + if corpus == "" { + t.Skip("set GORTEX_FIRST_INDEX_CORPUS and GORTEX_FIRST_INDEX_REPORT for an owned-fixture measurement") + } + reportPath := os.Getenv("GORTEX_FIRST_INDEX_REPORT") + if reportPath == "" { + t.Fatal("GORTEX_FIRST_INDEX_REPORT is required") + } + corpus, err := filepath.Abs(corpus) + if err != nil { + t.Fatal(err) + } + git := func(args ...string) string { + t.Helper() + result, err := exec.Command("git", append([]string{"-C", corpus}, args...)...).CombinedOutput() + if err != nil { + t.Fatalf("fixture git %v: %v: %s", args, err, result) + } + return string(result) + } + if status := git("status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("fixture must be clean: %s", status) + } + sourceSHA := strings.TrimSpace(git("rev-parse", "HEAD")) + manifest := sha256.Sum256([]byte(git("ls-files", "-s", "-z"))) + output, err := os.OpenFile(reportPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + t.Fatal(err) + } + defer output.Close() + store, err := store_sqlite.Open(filepath.Join(t.TempDir(), "fresh.sqlite")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := store.Close(); err != nil { + t.Error(err) + } + }) + sampler := newProcessSampler() + type sample struct { + At time.Time `json:"at"` + Metrics map[string]Metric `json:"metrics"` + } + var mu sync.Mutex + var samples []sample + var before, after map[string]Metric + var startAt, endAt time.Time + started, finished, joined := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var startOnce, endOnce sync.Once + core, logs := observer.New(zap.InfoLevel) + logger := zap.New(core, zap.Hooks(func(entry zapcore.Entry) error { + switch entry.Message { + case "indexer: drain start (shadow → disk)": + startOnce.Do(func() { + mu.Lock() + startAt = entry.Time + before = sampler.sample(os.Getpid()) + samples = append(samples, sample{time.Now(), before}) + mu.Unlock() + close(started) + }) + case "indexer: FlushBulk complete": + endOnce.Do(func() { + mu.Lock() + endAt = entry.Time + after = sampler.sample(os.Getpid()) + samples = append(samples, sample{time.Now(), after}) + mu.Unlock() + close(finished) + }) + } + return nil + })) + go func() { + defer close(joined) + select { + case <-started: + case <-finished: + return + } + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-finished: + return + case <-ticker.C: + mu.Lock() + if endAt.IsZero() { + samples = append(samples, sample{time.Now(), sampler.sample(os.Getpid())}) + } + mu.Unlock() + } + } + }() + registry := parser.NewRegistry() + languages.RegisterAll(registry) + idx := indexer.New(store, registry, config.Default().Index, logger) + t.Cleanup(idx.Close) + result, indexErr := idx.IndexCtx(t.Context(), corpus) + endOnce.Do(func() { close(finished) }) + <-joined + complete := indexErr == nil && !startAt.IsZero() && !endAt.IsZero() + deltas := make(map[string]Metric) + for _, name := range []string{"diskio_written_bytes", "logical_written_bytes", "syscall_written_bytes"} { + if complete { + deltas[name] = counterDelta(before[name], after[name]) + } else { + deltas[name] = unavailable("bytes", "complete drain endpoints unavailable") + } + } + entries := logs.FilterMessage("indexer: FlushBulk complete").All() + var drainFields map[string]any + if len(entries) == 1 { + drainFields = entries[0].ContextMap() + } + report := map[string]any{ + "go_version": runtime.Version(), "goos": runtime.GOOS, "goarch": runtime.GOARCH, "gomaxprocs": runtime.GOMAXPROCS(0), + "complete": complete, "corpus_sha": sourceSHA, "tracked_manifest_sha256": hex.EncodeToString(manifest[:]), + "result": result, "drain_start": startAt, "drain_end": endAt, "drain_fields": drainFields, + "process_write_deltas": deltas, "process_samples": samples, + "qualification": "process-wide counters over drain log boundaries; concurrent process activity can contribute; macOS logical writes exclude external-volume ledger; diskio is the physical meter; unavailable observations remain null; five-second samples plus endpoints; compare identical frozen corpus/harness/host/storage separately; no acceptance inferred", + } + if complete { + report["drain_and_seal_seconds"] = endAt.Sub(startAt).Seconds() + } + if indexErr != nil { + report["error"] = indexErr.Error() + } + if err := json.NewEncoder(output).Encode(report); err != nil { + t.Fatal(err) + } + if status := git("status", "--porcelain", "--untracked-files=all"); status != "" { + t.Fatalf("fixture changed: %s", status) + } + if strings.TrimSpace(git("rev-parse", "HEAD")) != sourceSHA || sha256.Sum256([]byte(git("ls-files", "-s", "-z"))) != manifest { + t.Fatal("fixture identity changed") + } + if !complete { + t.Fatalf("first index did not complete one measured drain: %v", indexErr) + } +} From 800ebcfc1b97f7b1898304380aa7cd7d663a1a58 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:18:41 +0100 Subject: [PATCH 42/60] fix(indexer): release identity permits before reporting completion --- internal/indexer/go_module_extern_test.go | 11 ++- .../prefix_observation_publication_test.go | 71 +++++++++++++++++++ internal/indexer/prefix_ownership.go | 11 ++- internal/indexer/track_naming.go | 8 ++- 4 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 internal/indexer/prefix_observation_publication_test.go diff --git a/internal/indexer/go_module_extern_test.go b/internal/indexer/go_module_extern_test.go index e8546be5f..147ee509e 100644 --- a/internal/indexer/go_module_extern_test.go +++ b/internal/indexer/go_module_extern_test.go @@ -153,8 +153,17 @@ func Caller() { library.Hello(); child.Nested() } } case "parallel": errors := make(chan error, len(repos)) + // This is a module-binding test, not an identity-admission + // saturation test. The decoy case has three roots; retain + // parallel tracking within the two native observation slots. + admitted := make(chan struct{}, 2) for _, entry := range repos { - go func() { _, e := mi.TrackRepoCtx(context.Background(), entry); errors <- e }() + go func() { + admitted <- struct{}{} + defer func() { <-admitted }() + _, e := mi.TrackRepoCtx(context.Background(), entry) + errors <- e + }() } for range repos { require.NoError(t, <-errors) diff --git a/internal/indexer/prefix_observation_publication_test.go b/internal/indexer/prefix_observation_publication_test.go new file mode 100644 index 000000000..26df8a0ec --- /dev/null +++ b/internal/indexer/prefix_observation_publication_test.go @@ -0,0 +1,71 @@ +package indexer + +import ( + "context" + "os" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" +) + +// Publication must mean the native worker has returned its allowance. A caller +// often starts another observation immediately; announcing completion before +// permit return can spuriously exhaust both slots under parallel tracking. +func TestIdentityObservationPublishesOnlyAfterPermitReturn(t *testing.T) { + root := t.TempDir() + for _, operation := range []string{"stat", "naming"} { + t.Run(operation, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var allowance prefixStatAllowance + allowance.once.Do(func() { allowance.slots = make(chan struct{}, 2) }) + nativeDone := make(chan struct{}) + // Temporarily intercept this worker's token so its release must + // wait for the test. This makes publication ordering observable + // without scheduler timing, polling or an extra production hook. + finishNative := func() { + <-allowance.slots + close(nativeDone) + } + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + result := make(chan error, 1) + if operation == "stat" { + allowance.stat = func(path string) (os.FileInfo, error) { + info, err := os.Stat(path) + finishNative() + return info, err + } + go func() { result <- allowance.observe(ctx, root).err }() + } else { + go func() { + _, err := allowance.observeNaming(ctx, root, func(context.Context, string) trackNamingFacts { + finishNative() + return trackNamingFacts{canonical: root} + }) + result <- err + }() + } + <-nativeDone + synctest.Wait() + var publishedEarly bool + var err error + select { + case err = <-result: + publishedEarly = true + default: + } + // Always let the worker finish before assertions, including RED. + allowance.slots <- struct{}{} + synctest.Wait() + if !publishedEarly { + err = <-result + } + require.NoError(t, err) + require.Zero(t, len(allowance.slots)) + require.False(t, publishedEarly, "completed identity result escaped before its permit was returned") + }) + }) + } +} diff --git a/internal/indexer/prefix_ownership.go b/internal/indexer/prefix_ownership.go index a8da85371..75beaf395 100644 --- a/internal/indexer/prefix_ownership.go +++ b/internal/indexer/prefix_ownership.go @@ -60,9 +60,14 @@ func (a *prefixStatAllowance) observe(ctx context.Context, path string) prefixSt slots := a.slots done := make(chan prefixStatResult, 1) go func() { - defer func() { <-slots }() - info, err := stat(path) - done <- prefixStatResult{info: info, err: err} + result := func() prefixStatResult { + defer func() { <-slots }() + info, err := stat(path) + return prefixStatResult{info: info, err: err} + }() + // A completed result authorizes the caller's next observation. + // Return this permit first, even when the receiver runs immediately. + done <- result }() select { case <-ctx.Done(): diff --git a/internal/indexer/track_naming.go b/internal/indexer/track_naming.go index 3505de97f..72b9dfb9b 100644 --- a/internal/indexer/track_naming.go +++ b/internal/indexer/track_naming.go @@ -86,8 +86,12 @@ func (a *prefixStatAllowance) observeNaming(ctx context.Context, root string, re workerCtx, cancel := context.WithDeadline(context.Background(), deadline) defer cancel() go func() { - defer func() { <-slots }() - done <- read(workerCtx, root) + facts := func() trackNamingFacts { + defer func() { <-slots }() + return read(workerCtx, root) + }() + // Completion includes permit return, as for root Stat observations. + done <- facts }() select { case <-ctx.Done(): From a924bb12c2c0e49f85453ada756267a0c2f22801 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:34:25 +0100 Subject: [PATCH 43/60] test(bench): retain checkout discovery refusal scope --- bench/load/discovery_scope_test.go | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 bench/load/discovery_scope_test.go diff --git a/bench/load/discovery_scope_test.go b/bench/load/discovery_scope_test.go new file mode 100644 index 000000000..a8293fa6a --- /dev/null +++ b/bench/load/discovery_scope_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "encoding/json" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDiscoveryAttemptsRetainSessionCWDAndRefusalTarget(t *testing.T) { + cwd := filepath.Join(t.TempDir(), "pending", "subdir") + root := filepath.Dir(cwd) + details := map[string]any{"error_code": "view_building", "path": root, + "family": "family-a", "reason": "family_capacity", "state": "discovery_pending", "retryable": true} + socket := startStub(t, func(_ stubFrame) (any, *rpcError) { + return nil, &rpcError{Code: -32000, Message: "checkout discovery unavailable", Data: details} + }) + client, err := openMCP(t.Context(), socket, cwd) + require.NoError(t, err) + defer client.Close() + _, attempts, err := callTool(t.Context(), client, request{Tool: "search", Operation: "symbols"}, + map[string]any{"operation": "symbols", "query": "Target"}, "workload", 1, 0, 0, time.Second, + func(context.Context, time.Duration) error { return nil }) + require.Error(t, err) + require.Len(t, attempts, 1) + encoded, err := json.Marshal(attempts[0]) + require.NoError(t, err) + var observation map[string]any + require.NoError(t, json.Unmarshal(encoded, &observation)) + require.Equal(t, cwd, observation["cwd"], "request cwd must remain distinct from its pending checkout root") + require.Equal(t, details, observation["error_data"], "refusal scope must survive report serialization") +} From ad725a6aa2fc79a56fd20f4b615dae860ad72669 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:34:04 +0100 Subject: [PATCH 44/60] bench: retain independent snapshots of structured RPC errors --- bench/load/rpc_error_observation.go | 29 ++++++++++++++++++++++++ bench/load/rpc_error_observation_test.go | 19 ++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 bench/load/rpc_error_observation.go create mode 100644 bench/load/rpc_error_observation_test.go diff --git a/bench/load/rpc_error_observation.go b/bench/load/rpc_error_observation.go new file mode 100644 index 000000000..52bc08e22 --- /dev/null +++ b/bench/load/rpc_error_observation.go @@ -0,0 +1,29 @@ +package main + +// Error data is decoded JSON. Keep a private snapshot when an observation +// outlives its RPC, including nested objects and lists of pending paths. +func copyRPCErrorData(data map[string]any) map[string]any { + if data == nil { + return nil + } + out := make(map[string]any, len(data)) + for key, value := range data { + out[key] = copyRPCErrorValue(value) + } + return out +} + +func copyRPCErrorValue(value any) any { + switch value := value.(type) { + case map[string]any: + return copyRPCErrorData(value) + case []any: + out := make([]any, len(value)) + for i, item := range value { + out[i] = copyRPCErrorValue(item) + } + return out + default: + return value + } +} diff --git a/bench/load/rpc_error_observation_test.go b/bench/load/rpc_error_observation_test.go new file mode 100644 index 000000000..8a40e8c11 --- /dev/null +++ b/bench/load/rpc_error_observation_test.go @@ -0,0 +1,19 @@ +package main + +import "testing" + +func TestRPCErrorDataSnapshotOwnsNestedJSON(t *testing.T) { + original := map[string]any{"code": "view_pending", "details": map[string]any{"paths": []any{"one.go", map[string]any{"scope": "checkout"}}}} + snapshot := copyRPCErrorData(original) + original["code"] = "changed" + details := original["details"].(map[string]any) + details["paths"].([]any)[0] = "two.go" + details["paths"].([]any)[1].(map[string]any)["scope"] = "other" + kept := snapshot["details"].(map[string]any)["paths"].([]any) + if snapshot["code"] != "view_pending" || kept[0] != "one.go" || kept[1].(map[string]any)["scope"] != "checkout" { + t.Fatalf("snapshot borrowed decoded RPC data: %+v", snapshot) + } + if copyRPCErrorData(nil) != nil { + t.Fatal("nil data became a fabricated object") + } +} From e972f6cd1cfcadd7325c40dbdca5951897f13aeb Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:42:17 +0100 Subject: [PATCH 45/60] bench: measure live checkout discovery by session cwd --- bench/load/README.md | 26 +++ bench/load/call.go | 23 ++- bench/load/client.go | 3 +- bench/load/discovery_scope.go | 214 ++++++++++++++++++++++ bench/load/discovery_scope_test.go | 83 +++++++++ bench/load/plan.go | 4 + bench/load/report.go | 30 +-- bench/load/run.go | 8 +- bench/load/scenario.go | 9 +- bench/load/testdata/report-v1.schema.json | 10 + 10 files changed, 388 insertions(+), 22 deletions(-) create mode 100644 bench/load/discovery_scope.go diff --git a/bench/load/README.md b/bench/load/README.md index 976ad529a..941c7ee4a 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -749,3 +749,29 @@ Inspect a profile with `go tool pprof -sample_index=inuse_space -top worktrees.step3.heap-load-peak.pprof`, and what one more worktree retains with `go tool pprof -sample_index=inuse_space -top -base worktrees.step2.heap-idle.pprof worktrees.step3.heap-idle.pprof`. + +### Live checkout discovery containment + +`--scenario worktree-discovery` starts with the first two pinned repositories +in the manifest. It keeps their reading/editing sessions open while adding +`--worktrees` detached checkouts to the first repository during the offered +window. Each added checkout gets separate root and subdirectory query sessions. +Use `--agents 0 --edit-agents 8` for eight combined edit/readback agents, or +`--agents 8 --edit-agents 0` for a read-only control. Editing requires the same +validated edit fixtures as the ordinary edit scenarios. + +`phase.gate.checkout_discovery` records every added root, first request, first +observed exact worktree view, and the resulting visibility lag. That lag is an +upper bound on catalog identity allocation; it does not claim an unobserved +earlier allocation time. Each cwd has its class, call/error/refusal counts, +complete-call read latency distribution and separately counted censored reads. +Attempts retain the actual session cwd and structured RPC `error_data`, including +the refused root, family, reason and retryability when supplied by the daemon. +A refusal or completed error outside a currently pending checkout fails the run. + +Run an otherwise identical `--worktrees 0` control to compare primary read p95. +Keep both reports and all refused/censored attempts. Zero refusals without any +`pending_refusals` is not evidence of fault containment under a stuck discovery. +A hung-discovery fixture, upgraded-store fixture and the reference-host runs +remain separate acceptance cells; this scenario does not manufacture them. +All measurements require an isolated host window and fresh owned clones/stores. diff --git a/bench/load/call.go b/bench/load/call.go index a9be86946..dc5a6bda3 100644 --- a/bench/load/call.go +++ b/bench/load/call.go @@ -112,18 +112,33 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[ } } } - if err == nil && callCtx.Err() != nil { - err = callCtx.Err() + // A socket deadline can fire before the context timer goroutine runs. + // Classify against the actual call deadline as well, so a clipped + // offered window does not invent a product/checkout failure. + expired := callCtx.Err() != nil + if deadline, ok := callCtx.Deadline(); ok && !returned.Before(deadline) { + expired = true + } + expiry := callCtx.Err() + if expired && expiry == nil { + expiry = context.DeadlineExceeded + } + if err == nil && expired { + err = expiry } if err == nil { err = checkToolResult(data) } a := Attempt{Tool: r.name(), Role: role, Request: hash, Agent: agent, Step: step, Slot: slot, Retry: retry, Started: start, DurationMS: float64(returned.Sub(start)) / float64(time.Millisecond), LogicalMS: float64(returned.Sub(logicalStart)) / float64(time.Millisecond), Outcome: "success"} + if session, ok := client.(*socketClient); ok { + a.CWD = session.cwd + } var remote *rpcError if err != nil { a.Outcome, a.Error = "error", err.Error() if errors.As(err, &remote) { a.Code = remote.Code + a.ErrorData = copyRPCErrorData(remote.Data) } if remote != nil && remote.Code == -32002 && (!retryTyped || remote.retryEditProbe()) { a.Outcome = "busy" @@ -135,9 +150,9 @@ func callToolPolicy(ctx context.Context, client toolCaller, r request, args map[ a.Code = -32000 a.ExpectedOutcome = "symbol_absent" } - if callCtx.Err() != nil { + if expired { a.Outcome = terminalOutcome() - err = errors.Join(err, callCtx.Err()) + err = errors.Join(err, expiry) } if retryTyped && errors.Is(err, errEditProbeEvidenceIncomplete) { a.Outcome = "harness_incomplete" diff --git a/bench/load/client.go b/bench/load/client.go index 66f5371a5..0e5b45046 100644 --- a/bench/load/client.go +++ b/bench/load/client.go @@ -64,6 +64,7 @@ type socketClient struct { notifies []notificationObservation notifyDropped int version string + cwd string // immutable checkout supplied by the successful handshake } // Per-call values returned to its caller, never a shared last-ID cache. Times @@ -142,7 +143,7 @@ func openMCP(ctx context.Context, socket, cwd string) (*socketClient, error) { if err != nil { return nil, err } - c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, writeGate: make(chan struct{}, 1)} + c := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: cwd, writeGate: make(chan struct{}, 1)} go c.readLoop() if _, err := c.call(ctx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "gortex-load", "version": "1"}}); err != nil { _ = c.Close() diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go new file mode 100644 index 000000000..d8223bc71 --- /dev/null +++ b/bench/load/discovery_scope.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +const checkoutDiscoveryScenario = "worktree-discovery" + +type checkoutDiscoveryObservation struct { + Root string `json:"root"` + Added time.Time `json:"added"` + FirstRequest time.Time `json:"first_request"` + ExactAt time.Time `json:"exact_at"` + ExactViewLag Metric `json:"exact_view_lag_ms"` + View json.RawMessage `json:"view,omitempty"` + Error string `json:"error,omitempty"` +} + +type checkoutCWDCounts struct { + Class string `json:"class"` + ReadLatency Distribution `json:"read_latency_ms"` + ReadCensored int `json:"read_censored"` + Calls int `json:"calls"` + Refusals int `json:"refusals"` + Errors int `json:"errors"` + Latency Distribution `json:"latency_ms"` +} + +type checkoutDiscoveryReport struct { + PendingRefusals int `json:"pending_refusals"` + Checkouts []checkoutDiscoveryObservation `json:"checkouts"` + ByCWD map[string]checkoutCWDCounts `json:"by_cwd"` + OutsidePendingRefusals int `json:"outside_pending_refusals"` + OutsidePendingErrors int `json:"outside_pending_errors"` +} + +// A refusal belongs to a pending checkout only while its discovery is +// unresolved. Component boundaries keep a sibling path out of that scope. +func pendingCheckoutForAttempt(a Attempt, checkouts []checkoutDiscoveryObservation) bool { + for _, checkout := range checkouts { + if inside(checkout.Root, a.CWD) && !a.Started.Before(checkout.Added) && + (checkout.ExactAt.IsZero() || a.Started.Before(checkout.ExactAt)) { + return true + } + } + return false +} + +func summarizeCheckoutDiscovery(attempts []Attempt, checkouts []checkoutDiscoveryObservation, primary, other string) checkoutDiscoveryReport { + out := checkoutDiscoveryReport{Checkouts: checkouts, ByCWD: make(map[string]checkoutCWDCounts)} + values := make(map[string][]float64) + reads := make(map[string][]float64) + for _, a := range attempts { + row := out.ByCWD[a.CWD] + row.Class = "outside_known_checkouts" + if inside(primary, a.CWD) { + row.Class = "primary" + } + if inside(other, a.CWD) { + row.Class = "other_repository" + } + for _, checkout := range checkouts { + if inside(checkout.Root, a.CWD) { + row.Class = "new_worktree" + } + } + if a.Role == "workload" && !strings.HasPrefix(a.Tool, "edit.") { + if a.Outcome == "phase_censored" { + row.ReadCensored++ + } else { + reads[a.CWD] = append(reads[a.CWD], a.DurationMS) + } + } + row.Calls++ + values[a.CWD] = append(values[a.CWD], a.DurationMS) + refused := a.Code != 0 + failed := a.Outcome != "success" && a.Outcome != "phase_censored" + if refused { + row.Refusals++ + } + if failed { + row.Errors++ + } + if pendingCheckoutForAttempt(a, checkouts) && refused { + out.PendingRefusals++ + } + if !pendingCheckoutForAttempt(a, checkouts) { + if refused { + out.OutsidePendingRefusals++ + } + if failed { + out.OutsidePendingErrors++ + } + } + out.ByCWD[a.CWD] = row + } + for cwd, row := range out.ByCWD { + row.Latency = distribution(values[cwd], "ms") + row.ReadLatency = distribution(reads[cwd], "ms") + out.ByCWD[cwd] = row + } + return out +} + +// The existing readers keep their original sessions and repository assignment +// throughout the additions. Discovery probes own separate root/subdirectory +// sessions; every refusal remains an attempt instead of being retried away. +func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRun, clients []*socketClient, phase *Phase) (resultErr error) { + ctx, cancel := context.WithTimeout(parent, o.Duration) + defer cancel() + frozen := *p + frozen.Repos = append([]privateRepo(nil), p.Repos...) + load := Phase{} + done := make(chan error, 1) + go func() { done <- runAgents(ctx, o, &frozen, clients, o.Editors > 0, &load) }() + first := len(phase.Attempts) + var checkouts []checkoutDiscoveryObservation + var probes []*socketClient + defer func() { + cancel() + loadErr := <-done + closeClients(probes) + phase.Attempts = append(phase.Attempts, load.Attempts...) + phase.Edits = append(phase.Edits, load.Edits...) + finishEditMetrics(phase) + if len(checkouts) != o.Worktrees { + resultErr = errors.Join(resultErr, errors.New("offered window ended before all planned worktree additions")) + } + report := summarizeCheckoutDiscovery(phase.Attempts[first:], checkouts, frozen.Repos[0].Path, frozen.Repos[1].Path) + phase.Gate["checkout_discovery"], _ = json.Marshal(report) + if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 { + resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors", report.OutsidePendingRefusals, report.OutsidePendingErrors)) + } + if parent.Err() != nil { + resultErr = errors.Join(resultErr, parent.Err(), loadErr) + } + }() + // Additions are spread over the offered window, including a baseline with + // --worktrees=0. Timing is recorded as observed, never inferred from count. + interval := o.Duration / time.Duration(o.Worktrees+1) + nextAdd := time.Now().Add(interval) + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return resultErr + case <-ticker.C: + } + if len(checkouts) < o.Worktrees && !time.Now().Before(nextAdd) { + if err := p.addWorktree(ctx); err != nil { + return err + } + repo := p.Repos[len(p.Repos)-1] + repo.Prefix = p.Repos[0].Prefix + p.Repos[len(p.Repos)-1].Prefix = repo.Prefix + entry := checkoutDiscoveryObservation{Root: repo.Path, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} + checkouts = append(checkouts, entry) + nextAdd = nextAdd.Add(interval) + subdir := filepath.Join(repo.Path, ".discovery-probe") + if err := os.Mkdir(subdir, 0o700); err != nil { + return err + } + for _, cwd := range []string{repo.Path, subdir} { + client, err := openMCP(ctx, p.Socket, cwd) + if err != nil { + return fmt.Errorf("open discovery session in %s: %w", cwd, err) + } + probes = append(probes, client) + } + } + for i, client := range probes { + entry := &checkouts[i/2] + if entry.FirstRequest.IsZero() { + entry.FirstRequest = time.Now() + } + r := request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": p.Repos[0].Corpus.Query}} + _, rows, _ := callTool(ctx, client, r, r.Arguments, "probe", i, len(checkouts), 0, o.Deadline, waitContext) + phase.Attempts = append(phase.Attempts, rows...) + if i%2 != 0 || !entry.ExactAt.IsZero() || ctx.Err() != nil { + continue + } + raw, err := control(ctx, p.Socket, "search_symbols", map[string]any{"path": entry.Root, "query": p.Repos[0].Corpus.Query, "limit": 1}) + entry.View = append(json.RawMessage(nil), raw...) + if err != nil { + entry.Error = err.Error() + continue + } + var answer struct { + View struct { + Exact bool `json:"exact"` + Kind string `json:"kind"` + Prefix string `json:"repo_prefix"` + } `json:"view"` + } + if err := json.Unmarshal(raw, &answer); err != nil { + entry.Error = err.Error() + continue + } + if answer.View.Exact && answer.View.Kind == "worktree" && answer.View.Prefix == p.Repos[0].Prefix { + entry.ExactAt = time.Now() + entry.ExactViewLag = measured(float64(entry.ExactAt.Sub(entry.FirstRequest))/float64(time.Millisecond), "ms") + entry.Error = "" + } + } + } +} diff --git a/bench/load/discovery_scope_test.go b/bench/load/discovery_scope_test.go index a8293fa6a..28c9e4ed5 100644 --- a/bench/load/discovery_scope_test.go +++ b/bench/load/discovery_scope_test.go @@ -3,7 +3,9 @@ package main import ( "context" "encoding/json" + "os" "path/filepath" + "sync" "testing" "time" @@ -33,3 +35,84 @@ func TestDiscoveryAttemptsRetainSessionCWDAndRefusalTarget(t *testing.T) { require.Equal(t, cwd, observation["cwd"], "request cwd must remain distinct from its pending checkout root") require.Equal(t, details, observation["error_data"], "refusal scope must survive report serialization") } + +func TestDiscoveryRefusalAccountingUsesCWDAndPendingWindow(t *testing.T) { + start := time.Unix(100, 0) + root := filepath.Join(t.TempDir(), "pending") + entries := []checkoutDiscoveryObservation{{Root: root, Added: start, ExactAt: start.Add(time.Second)}} + rows := []Attempt{ + {CWD: filepath.Join(root, "subdir"), Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", DurationMS: 2}, + {CWD: root + "-sibling", Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", DurationMS: 3}, + {CWD: root, Started: start.Add(2 * time.Second), Code: -32000, Outcome: "error", DurationMS: 4}, + {CWD: filepath.Dir(root), Started: start.Add(time.Millisecond), Outcome: "success", DurationMS: 1}, + } + report := summarizeCheckoutDiscovery(rows, entries, filepath.Dir(root), root+"-sibling") + require.Equal(t, 2, report.OutsidePendingRefusals, "a sibling and an already-published checkout are outside the pending scope") + require.Equal(t, 2, report.OutsidePendingErrors) + require.Equal(t, 1, report.ByCWD[root].Refusals) + require.Equal(t, 1, report.ByCWD[filepath.Dir(root)].Latency.Count) +} + +func TestDiscoveryScenarioOptionsHaveAnExplicitBaseline(t *testing.T) { + args := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--scenario", checkoutDiscoveryScenario, "--agents", "2", "--edit-agents", "0", "--worktrees", "0"} + options, err := parseRun(args) + require.NoError(t, err) + require.Equal(t, 0, options.Worktrees) + require.Equal(t, checkoutDiscoveryScenario, options.Scenario) + _, err = parseRun(append(args, "--agents", "1")) + require.Error(t, err, "both initial repositories must retain a reading session") +} + +func TestDiscoveryAddsWorktreesWhileOriginalSessionsKeepReading(t *testing.T) { + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + p.Repos[0].Prefix, p.Repos[1].Prefix = "sample", "other" + var mu sync.Mutex + starts, ends := make(map[int]time.Time), make(map[int]time.Time) + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + if frame.Kind != "" { + return map[string]any{"view": map[string]any{"exact": true, "kind": "worktree", "repo_prefix": "sample"}}, nil + } + mu.Lock() + if starts[frame.Connection].IsZero() { + starts[frame.Connection] = time.Now() + } + ends[frame.Connection] = time.Now() + mu.Unlock() + return stubContent(map[string]any{"results": []any{map[string]any{"id": "sample/sample.go::Example", "name": "Example"}}}), nil + }) + var clients []*socketClient + for _, repo := range p.Repos { + c, err := openMCP(t.Context(), p.Socket, repo.Path) + require.NoError(t, err) + clients = append(clients, c) + } + defer closeClients(clients) + phase := Phase{Gate: make(map[string]json.RawMessage)} + err = measureCheckoutDiscovery(t.Context(), runOptions{Agents: 2, Worktrees: 1, Duration: 1200 * time.Millisecond, Deadline: time.Second, Profile: "saturation", Seed: 1}, p, clients, &phase) + failures := make(map[string]int) + for _, a := range phase.Attempts { + if a.Outcome != "success" && a.Outcome != "phase_censored" { + failures[a.Error]++ + } + } + require.NoError(t, err, "attempt failures: %v", failures) + var report checkoutDiscoveryReport + require.NoError(t, json.Unmarshal(phase.Gate["checkout_discovery"], &report)) + require.Len(t, report.Checkouts, 1) + added := report.Checkouts[0].Added + require.NotNil(t, report.Checkouts[0].ExactViewLag.Value) + mu.Lock() + defer mu.Unlock() + for _, conn := range []int{1, 2} { + require.True(t, starts[conn].Before(added), "original session must run before worktree add") + require.True(t, ends[conn].After(added), "original session must remain active during discovery") + } + root := report.Checkouts[0].Root + require.Positive(t, report.ByCWD[root].Calls) + require.Positive(t, report.ByCWD[filepath.Join(root, ".discovery-probe")].Calls) + require.Zero(t, report.OutsidePendingRefusals) +} diff --git a/bench/load/plan.go b/bench/load/plan.go index d17afa478..c80add08d 100644 --- a/bench/load/plan.go +++ b/bench/load/plan.go @@ -200,6 +200,10 @@ func validateAgentCounts(scenario string, readers, editors int) error { if readers < 1 || editors < 1 { return errors.New("all scenarios require at least one pure reader and one combined edit/readback agent") } + case checkoutDiscoveryScenario: + if readers+editors < 2 { + return errors.New("worktree discovery needs sessions in both initial repositories") + } case "read", "many", "worktrees", "sweep", "cold", "restart": if readers < 1 { return errors.New("zero pure readers are supported only for edit/many-edit") diff --git a/bench/load/report.go b/bench/load/report.go index a02c5f307..3203592a7 100644 --- a/bench/load/report.go +++ b/bench/load/report.go @@ -83,20 +83,22 @@ type Host struct { // Every attempt is retained, including busy refusals and phase-censored calls. // Workload and readiness/visibility probes have distinct Role denominators. type Attempt struct { - Tool string `json:"tool"` - Role string `json:"role"` - Request string `json:"request_sha256"` - Agent int `json:"agent"` - Step int `json:"step"` - Slot int `json:"slot"` - Retry int `json:"retry"` - Started time.Time `json:"started"` - DurationMS float64 `json:"duration_ms"` - LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time - Code int `json:"code"` - Error string `json:"error"` - ExpectedOutcome string `json:"expected_outcome,omitempty"` - Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored + CWD string `json:"cwd,omitempty"` + ErrorData map[string]any `json:"error_data,omitempty"` + Tool string `json:"tool"` + Role string `json:"role"` + Request string `json:"request_sha256"` + Agent int `json:"agent"` + Step int `json:"step"` + Slot int `json:"slot"` + Retry int `json:"retry"` + Started time.Time `json:"started"` + DurationMS float64 `json:"duration_ms"` + LogicalMS float64 `json:"logical_ms"` // includes earlier refusals and backoff, excludes think time + Code int `json:"code"` + Error string `json:"error"` + ExpectedOutcome string `json:"expected_outcome,omitempty"` + Outcome string `json:"outcome"` // success, error, busy, product_deadline, phase_censored } type ToolMetrics struct { diff --git a/bench/load/run.go b/bench/load/run.go index a843aeea7..2aaadf31f 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -70,7 +70,7 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.StringVar(&o.GitSHA, "git-sha", "", "full binary revision if absent from Go build metadata") f.StringVar(&o.Profile, "profile", "agent", "agent (default) or explicit saturation") f.StringVar(&o.Host, "host", "native", "native, reference, or floor (validated, never provisioned)") - f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, cold, restart, sweep, idle-stop, edit-and-probe, all") + f.StringVar(&o.Scenario, "scenario", "read", "read, edit, many, many-edit, worktrees, worktree-discovery, cold, restart, sweep, idle-stop, edit-and-probe, all") f.IntVar(&o.Agents, "agents", 4, "pure reading agents (may be 0 for edit, many-edit, or edit-and-probe; idle-stop requires 0)") f.IntVar(&o.Editors, "edit-agents", 4, "combined editing/readback/reading agents in edit scenarios, in addition to pure readers") f.IntVar(&o.Worktrees, "worktrees", 4, "linked worktrees in addition to primary") @@ -141,6 +141,12 @@ func validateRunOptions(o runOptions) error { } func selectedScenarios(o runOptions) ([]string, error) { + if o.Scenario == checkoutDiscoveryScenario { + if o.Agents+o.Editors < 2 || o.Host == "floor" { + return nil, errors.New("worktree-discovery requires at least two reading/editing sessions and a native or reference host") + } + return []string{checkoutDiscoveryScenario}, nil + } if o.Scenario == "idle-stop" { return []string{"idle-stop"}, nil } diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 079b1cf3b..6be861960 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -62,13 +62,16 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } }() selected := corpora[:1] - if scenario == "many" || scenario == "many-edit" { + if scenario == "many" || scenario == "many-edit" || scenario == checkoutDiscoveryScenario { if len(corpora) < 2 { return phase, nil, artifacts, crashes, "", errors.New("many-repository scenarios require at least two pinned repositories") } selected = corpora + if scenario == checkoutDiscoveryScenario { + selected = corpora[:2] + } } - editing := scenario == "edit" || scenario == "many-edit" || scenario == editProbeScenario + editing := scenario == "edit" || scenario == "many-edit" || scenario == editProbeScenario || scenario == checkoutDiscoveryScenario && o.Editors > 0 for _, corpus := range selected { operations, planErr := readingPlan(corpus) if planErr != nil { @@ -429,6 +432,8 @@ func measureStandalone(ctx context.Context, o runOptions, p *privateRun, clients sweepCtx, cancelSweep := context.WithTimeout(ctx, o.SweepBudget) defer cancelSweep() err = runSweep(sweepCtx, o, sweepClients, p.Repos[0], sweepRequests, sweep, phase, waitContext) + } else if phase.Name == checkoutDiscoveryScenario { + err = measureCheckoutDiscovery(ctx, o, p, clients, phase) } else if phase.Name != "cold" && phase.Name != "restart" { err = runAgents(ctx, o, p, clients, phase.Name == "edit" || phase.Name == "many-edit", phase) } diff --git a/bench/load/testdata/report-v1.schema.json b/bench/load/testdata/report-v1.schema.json index ac950dfa7..095b97c34 100644 --- a/bench/load/testdata/report-v1.schema.json +++ b/bench/load/testdata/report-v1.schema.json @@ -245,12 +245,22 @@ "code": { "type": "integer" }, + "cwd": { + "type": "string" + }, "duration_ms": { "type": "number" }, "error": { "type": "string" }, + "error_data": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, "expected_outcome": { "type": "string" }, From 997860f4041f2873390cc521227be43f419aabdb Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 18:45:22 +0100 Subject: [PATCH 46/60] bench: describe discovery probe sessions in report contracts --- bench/load/discovery_scope.go | 1 + bench/load/testdata/paired-v1.schema.json | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go index d8223bc71..b541a6993 100644 --- a/bench/load/discovery_scope.go +++ b/bench/load/discovery_scope.go @@ -113,6 +113,7 @@ func summarizeCheckoutDiscovery(attempts []Attempt, checkouts []checkoutDiscover // throughout the additions. Discovery probes own separate root/subdirectory // sessions; every refusal remains an attempt instead of being retried away. func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRun, clients []*socketClient, phase *Phase) (resultErr error) { + phase.SessionModel = "per_agent_plus_new_worktree_root_and_subdirectory_probes" ctx, cancel := context.WithTimeout(parent, o.Duration) defer cancel() frozen := *p diff --git a/bench/load/testdata/paired-v1.schema.json b/bench/load/testdata/paired-v1.schema.json index 56717f6fb..27672c501 100644 --- a/bench/load/testdata/paired-v1.schema.json +++ b/bench/load/testdata/paired-v1.schema.json @@ -258,12 +258,22 @@ "code": { "type": "integer" }, + "cwd": { + "type": "string" + }, "duration_ms": { "type": "number" }, "error": { "type": "string" }, + "error_data": { + "additionalProperties": {}, + "type": [ + "object", + "null" + ] + }, "expected_outcome": { "type": "string" }, From 6e78b5ff89f5a9e3300b8a095f5b2631c063b569 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 19:28:41 +0100 Subject: [PATCH 47/60] test(mcp): classify shared-store drain observations --- internal/mcp/contract_core_capability_table_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/mcp/contract_core_capability_table_test.go b/internal/mcp/contract_core_capability_table_test.go index cdd3d5f34..798bfd7ee 100644 --- a/internal/mcp/contract_core_capability_table_test.go +++ b/internal/mcp/contract_core_capability_table_test.go @@ -118,6 +118,9 @@ type ( capBFSContext interface { BFSContext(context.Context, []string, graph.Direction, []graph.EdgeKind, int, int) ([]graph.BFSHop, error) } + capLastFirstIndexDrains interface { + LastFirstIndexDrains() map[string]graph.FirstIndexDrain + } capRepoPrefixes interface{ RepoPrefixes() []string } ) @@ -218,6 +221,7 @@ func contractCoreCapabilityTable() []contractCoreCapRow { {"NodeLightSequencer", capOf[graph.NodeLightSequencer], capDropped, "node-only streaming; graph.NodesLightSeq falls back to the AllNodesLight forwarder, the same rows materialised"}, {"Store", capOf[graph.Store], capDropped, "the whole backend contract; a request reader is never the store"}, {"local:WALReclaimStats", capOf[walReclaimSnapshotReader], capDropped, capReasonBackend + " (physical store counters, never selected graph state)"}, + {"local:LastFirstIndexDrains", capOf[capLastFirstIndexDrains], capDropped, capReasonBackend + " (shared-store drain observations, never selected graph state)"}, {"AnalysisGenerationStore", capOf[graph.AnalysisGenerationStore], capDropped, capReasonBackend}, {"AnalysisQueryStore", capOf[graph.AnalysisQueryStore], capDropped, capReasonBackend}, {"TryAnalysisQueryStore", capOf[graph.TryAnalysisQueryStore], capDropped, capReasonBackend}, From 80636ede9f2c2a462821a3a98f352b6a05a61e59 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 19:35:53 +0100 Subject: [PATCH 48/60] test: encode reference retry fixture URIs portably --- internal/mcp/refs_retry_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/mcp/refs_retry_test.go b/internal/mcp/refs_retry_test.go index 8c72328ba..0dac0d8f1 100644 --- a/internal/mcp/refs_retry_test.go +++ b/internal/mcp/refs_retry_test.go @@ -19,6 +19,7 @@ import ( "go.uber.org/zap" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/lspuri" "github.com/zzet/gortex/internal/semantic" "github.com/zzet/gortex/internal/semantic/lsp" ) @@ -33,6 +34,11 @@ func newReferenceRetryStack(t *testing.T, first string, beforeIncoming func()) ( if err := os.WriteFile(path, []byte(source), 0o600); err != nil { t.Fatal(err) } + // A Windows drive must be in the URI path, never its authority. + uri := lspuri.PathToURI(path) + if got := lspuri.URIToAbsPath(uri); got != path { + t.Fatalf("fixture URI %q resolves to %q, want %q", uri, got, path) + } target := viewRepoNode("repo/retry.rs::target", "target", graph.KindFunction, "repo/retry.rs", 1) caller := viewRepoNode("repo/retry.rs::caller", "caller", graph.KindFunction, "repo/retry.rs", 2) target.Language, caller.Language = "rust", "rust" @@ -99,7 +105,7 @@ func newReferenceRetryStack(t *testing.T, first string, beforeIncoming func()) ( rpcError = map[string]any{"code": -32603, "message": "workspace loading"} } if first != "zero" && (attempt > 1 || first == "positive" || first == "incoming-error") { - result = []lsp.CallHierarchyItem{{Name: "target", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 0}, End: lsp.Position{Line: 0, Character: 14}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 0, Character: 3}, End: lsp.Position{Line: 0, Character: 9}}}} + result = []lsp.CallHierarchyItem{{Name: "target", Kind: 12, URI: uri, Range: lsp.Range{Start: lsp.Position{Line: 0}, End: lsp.Position{Line: 0, Character: 14}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 0, Character: 3}, End: lsp.Position{Line: 0, Character: 9}}}} } else { result = []any{} } @@ -110,7 +116,7 @@ func newReferenceRetryStack(t *testing.T, first string, beforeIncoming func()) ( if beforeIncoming != nil { beforeIncoming() } - result = []lsp.CallHierarchyIncomingCall{{From: lsp.CallHierarchyItem{Name: "caller", Kind: 12, URI: "file://" + path, Range: lsp.Range{Start: lsp.Position{Line: 1}, End: lsp.Position{Line: 1, Character: 25}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 1, Character: 3}, End: lsp.Position{Line: 1, Character: 9}}}, FromRanges: []lsp.Range{{Start: lsp.Position{Line: 1, Character: 14}, End: lsp.Position{Line: 1, Character: 20}}}}} + result = []lsp.CallHierarchyIncomingCall{{From: lsp.CallHierarchyItem{Name: "caller", Kind: 12, URI: uri, Range: lsp.Range{Start: lsp.Position{Line: 1}, End: lsp.Position{Line: 1, Character: 25}}, SelectionRange: lsp.Range{Start: lsp.Position{Line: 1, Character: 3}, End: lsp.Position{Line: 1, Character: 9}}}, FromRanges: []lsp.Range{{Start: lsp.Position{Line: 1, Character: 14}, End: lsp.Position{Line: 1, Character: 20}}}}} } reply := map[string]any{"jsonrpc": "2.0", "id": req.ID, "result": result} if rpcError != nil { From 7578e2758b29be9d4acba9bf82ec780d46645695 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 19:39:50 +0100 Subject: [PATCH 49/60] test: align relation recovery and native file URI controls --- internal/mcp/mutation_freshness_hints_test.go | 14 +++++--------- internal/mcp/relation_freshness_test.go | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/internal/mcp/mutation_freshness_hints_test.go b/internal/mcp/mutation_freshness_hints_test.go index eb5f96d73..9e1cd9ffb 100644 --- a/internal/mcp/mutation_freshness_hints_test.go +++ b/internal/mcp/mutation_freshness_hints_test.go @@ -105,7 +105,7 @@ func TestFreshnessRefusalAnnotationBoundsAndScope(t *testing.T) { // Exercise the public call shapes against real handlers and an owned working // tree. The existing watcher seam supplies a failed publication result after // the edit acknowledges; this does not inject a real store patch failure. -func TestFreshnessRefusalRecoveryFlow(t *testing.T) { +func TestRelationFreshnessRecoveryFlow(t *testing.T) { s, root := setupTestServer(t) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -144,14 +144,10 @@ func TestFreshnessRefusalRecoveryFlow(t *testing.T) { waitCheckoutReceipt(t, s, id) detectArgs := map[string]any{"operation": "detect", "options": map[string]any{"scope": "unstaged"}} - refused := call("change", detectArgs) - require.True(t, refused.IsError) - for _, want := range []string{ - "graph freshness unavailable", "change.receipt options.receipt=" + id, - "workspace_admin operation=reindex", "paths=[P]", `mutation_id="recovery-flow"`, - } { - require.Contains(t, resultText(refused), want) - } + partial := resultJSON(t, call("change", detectArgs)) + require.Equal(t, false, partial["complete"]) + require.Equal(t, "failed", partial["graph_freshness"]) + require.NotEmpty(t, partial["recovery_hint"]) status := resultJSON(t, call("change", map[string]any{ "operation": "receipt", "options": map[string]any{"mutation_id": id}, })) diff --git a/internal/mcp/relation_freshness_test.go b/internal/mcp/relation_freshness_test.go index 1414642c2..368e8b28a 100644 --- a/internal/mcp/relation_freshness_test.go +++ b/internal/mcp/relation_freshness_test.go @@ -292,7 +292,7 @@ func TestRelationToolsLabelAndNeverRefuseWithUnresolvedReceipt(t *testing.T) { writeUnindexed(t, root, "namer.go", "package p\n\nfunc Namer() {\n\tTarget()\n}\n"), 1) id := targetID(t, s) rename := `{"changes":{"target.go":[{"range":{"start":{"line":2,"character":5},"end":{"line":2,"character":11}},"newText":"Renamed"}]}}` - comment := `{"changes":{"file://` + filepath.ToSlash(filepath.Join(root, "namer.go")) + + comment := `{"changes":{"` + nativeFileURI(filepath.Join(root, "namer.go")) + `":[{"range":{"start":{"line":0,"character":0},"end":{"line":0,"character":0}},"newText":"// Namer names Target.\n"}]}}` for tool, call := range map[string]func() *mcpgo.CallToolResult{ From 329013353630c62bebdd37dbddfcc516ef1f306f Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:02:04 +0100 Subject: [PATCH 50/60] feat(load): probe foreign discovery and owned Git stalls Source checkpoint; validation is pending shared lane availability. --- bench/load/README.md | 12 +++++- bench/load/discovery_fault.go | 64 ++++++++++++++++++++++++++++++ bench/load/discovery_fault_test.go | 57 ++++++++++++++++++++++++++ bench/load/discovery_scope.go | 27 ++++++++----- bench/load/discovery_scope_test.go | 31 +++++++++++++++ bench/load/discovery_worktree.go | 31 +++++++++++++++ bench/load/lifecycle.go | 5 ++- bench/load/run.go | 7 +++- bench/load/scenario.go | 20 ++++++++-- 9 files changed, 236 insertions(+), 18 deletions(-) create mode 100644 bench/load/discovery_fault.go create mode 100644 bench/load/discovery_fault_test.go create mode 100644 bench/load/discovery_worktree.go diff --git a/bench/load/README.md b/bench/load/README.md index 941c7ee4a..41d18d8a8 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -754,7 +754,7 @@ and what one more worktree retains with `--scenario worktree-discovery` starts with the first two pinned repositories in the manifest. It keeps their reading/editing sessions open while adding -`--worktrees` detached checkouts to the first repository during the offered +`--worktrees` detached checkouts alternating the two repositories during the offered window. Each added checkout gets separate root and subdirectory query sessions. Use `--agents 0 --edit-agents 8` for eight combined edit/readback agents, or `--agents 8 --edit-agents 0` for a read-only control. Editing requires the same @@ -775,3 +775,13 @@ Keep both reports and all refused/censored attempts. Zero refusals without any A hung-discovery fixture, upgraded-store fixture and the reference-host runs remain separate acceptance cells; this scenario does not manufacture them. All measurements require an isolated host window and fresh owned clones/stores. + + +Discovery additions alternate the primary and unrelated control families. Each +new root keeps its own family query and prefix for the exact-view witness; +foreign-family capacity refusals fail the containment gate even while that +new root is pending. `--discovery-stall-first` installs a daemon-only Git shim +for the first new primary-family checkout. Its child ignores TERM and retains +stdout until explicit release or a 60-second bound. The harness releases it +before daemon shutdown and retains child/root/operation events. A missing +stall or pending-refusal observation fails this cell. diff --git a/bench/load/discovery_fault.go b/bench/load/discovery_fault.go new file mode 100644 index 000000000..f2f376f52 --- /dev/null +++ b/bench/load/discovery_fault.go @@ -0,0 +1,64 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +type discoveryOptions struct { StallFirst bool } +func bindDiscoveryFlags(f *flag.FlagSet,o *discoveryOptions){f.BoolVar(&o.StallFirst,"discovery-stall-first",false,"worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound")} +func(o discoveryOptions)validate(scenario string,count int)error{ + if !o.StallFirst{return nil} + if scenario!=checkoutDiscoveryScenario||count<1{return errors.New("--discovery-stall-first requires worktree-discovery and at least one new worktree")} + if runtime.GOOS=="windows" {return errors.New("discovery Git stall requires Unix process ownership")} + return nil +} + +func discoveryShellLiteral(s string)string{return "'"+strings.ReplaceAll(s,"'","'\"'\"'")+"'"} + +// No shell runs while preparing this adapter. Only the owned daemon receives +// its PATH; worktree creation and all harness Git retain the real executable. +func(p *privateRun)installDiscoveryFault()(func()error,map[string]string,error){ + if runtime.GOOS=="windows"{return nil,nil,errors.New("discovery Git stall requires Unix")} + root,err:=canonicalPath(p.Root);if err!=nil||root!=p.Root{return nil,nil,errors.New("discovery fault requires its canonical private root")} + realGit,err:=exec.LookPath("git");if err!=nil{return nil,nil,err};realGit,err=filepath.Abs(realGit);if err!=nil{return nil,nil,err} + dir:=filepath.Join(p.Root,"discovery-git-fault");if err:=os.Mkdir(dir,0o700);err!=nil{return nil,nil,err} + target:=filepath.Join(p.Root,"repos","discovery-linked-0-0") + release,events:=filepath.Join(dir,"release"),filepath.Join(dir,"events.tsv") + script:=fmt.Sprintf(`#!/bin/sh +cwd=$(pwd -P) +want_cwd=0 +operation= +for arg in "$@"; do + if [ "$want_cwd" = 1 ]; then cwd=$arg; want_cwd=0; continue; fi + case "$arg" in -C) want_cwd=1;; rev-parse|worktree) operation=$arg;; esac +done +case "$cwd" in %s|%s/*) + if [ -n "$operation" ]; then + ready=%s.ready.$$ + (trap '' TERM HUP INT + : > "$ready" + count=0 + while [ ! -f %s ] && [ "$count" -lt 60 ]; do sleep 1; count=$((count+1)); done + ) & + child=$! + while [ ! -f "$ready" ] && kill -0 "$child" 2>/dev/null; do sleep 0.01; done + printf '%%s\t%%s\t%%s\n' "$child" "$cwd" "$operation" >> %s + wait "$child" + exit 124 + fi;; +esac +exec %s "$@" +`,discoveryShellLiteral(target),discoveryShellLiteral(target),discoveryShellLiteral(events),discoveryShellLiteral(release),discoveryShellLiteral(events),discoveryShellLiteral(realGit)) + shim:=filepath.Join(dir,"git");if err:=os.WriteFile(shim,[]byte(script),0o700);err!=nil{return nil,nil,err} + env:=append([]string(nil),p.Env...);found:=false + for i,entry:=range env{if strings.HasPrefix(entry,"PATH="){env[i]="PATH="+dir+string(os.PathListSeparator)+strings.TrimPrefix(entry,"PATH=");found=true}} + if !found{env=append(env,"PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"))};p.daemonEnv=env + return func()error{return os.WriteFile(release,[]byte("release owned Git descendants\n"),0o600)},map[string]string{"discovery_git_shim":shim,"discovery_git_events":events,"discovery_git_release":release},nil +} diff --git a/bench/load/discovery_fault_test.go b/bench/load/discovery_fault_test.go new file mode 100644 index 000000000..c0f48a326 --- /dev/null +++ b/bench/load/discovery_fault_test.go @@ -0,0 +1,57 @@ +//go:build linux || darwin + +package main + +import ( + "context" + "bytes" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDiscoveryFaultRetainsOwnedPipeAndDoesNotAffectHarnessGit(t *testing.T){ + first,second:=gitFixture(t),gitFixture(t);second.Label="other" + p,err:=newPrivateRun(t.Context(),t.TempDir(),"unused",[]Corpus{first,second});require.NoError(t,err) + t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) + originalEnv:=append([]string(nil),p.Env...) + release,files,err:=p.installDiscoveryFault();require.NoError(t,err) + t.Cleanup(func(){_=release()}) + require.Equal(t,originalEnv,p.Env,"harness environment remains unchanged") + require.NotEqual(t,p.Env,p.daemonEnv) + require.NoError(t,p.addDiscoveryWorktree(t.Context(),0),"harness worktree creation must never enter its daemon shim") + linked:=p.Repos[len(p.Repos)-1] + ctx,cancel:=context.WithTimeout(t.Context(),10*time.Second);defer cancel() + cmd:=exec.CommandContext(ctx,files["discovery_git_shim"],"-C",linked.Path,"rev-parse","--show-toplevel") + configureOwnedProcess(cmd);cmd.Cancel=func()error{return killOwnedProcess(cmd)} + var output bytes.Buffer + cmd.Stdout,cmd.Stderr=&output,&output + cmd.WaitDelay=3*time.Second + require.NoError(t,cmd.Start()) + done:=make(chan error,1);go func(){done<-cmd.Wait()}() + joined:=false + t.Cleanup(func(){_=release();if !joined{_=killOwnedProcess(cmd);<-done}}) + var child int + require.Eventually(t,func()bool{raw,err:=os.ReadFile(files["discovery_git_events"]);if err!=nil{return false};fields:=strings.Fields(string(raw));if len(fields)!=3{return false};child,err=strconv.Atoi(fields[0]);return err==nil&&fields[1]==linked.Path&&fields[2]=="rev-parse"},3*time.Second,5*time.Millisecond) + require.NoError(t,cmd.Process.Kill(),"cancelled Git parent exits while descendant retains its stdout pipe") + require.NoError(t,syscall.Kill(child,syscall.SIGTERM),"TERM goes only to recorded owned descendant") + select{case err:=<-done:joined=true;t.Fatalf("stall unexpectedly exited after TERM: %v",err);case <-time.After(30*time.Millisecond):} + require.NoError(t,release()) + select{case err:=<-done:joined=true;require.Error(t,err);case <-ctx.Done():t.Fatal("released owned shim did not join")} + _,err=p.git(t.Context(),p.Repos[1].Path,"rev-parse","--show-toplevel");require.NoError(t,err) + _,err=os.Stat(filepath.Join(p.Root,"discovery-git-fault","release"));require.NoError(t,err) +} + +func TestDiscoveryStallOptionsRequireActualAdditions(t *testing.T){ + base:=[]string{"--binary","unused","--manifest","unused","--scratch",t.TempDir(),"--agents","2","--edit-agents","0","--discovery-stall-first"} + _,err:=parseRun(append(base,"--scenario",checkoutDiscoveryScenario,"--worktrees","2"));require.NoError(t,err) + _,err=parseRun(append(base,"--scenario",checkoutDiscoveryScenario,"--worktrees","0"));require.Error(t,err) + _,err=parseRun(append(base,"--scenario","read"));require.Error(t,err) +} diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go index b541a6993..0391b8264 100644 --- a/bench/load/discovery_scope.go +++ b/bench/load/discovery_scope.go @@ -15,6 +15,9 @@ const checkoutDiscoveryScenario = "worktree-discovery" type checkoutDiscoveryObservation struct { Root string `json:"root"` + FamilyRole string `json:"family_role"` + Prefix string `json:"repo_prefix"` + Query string `json:"query"` Added time.Time `json:"added"` FirstRequest time.Time `json:"first_request"` ExactAt time.Time `json:"exact_at"` @@ -34,7 +37,8 @@ type checkoutCWDCounts struct { } type checkoutDiscoveryReport struct { - PendingRefusals int `json:"pending_refusals"` + ForeignCapacityRefusals int `json:"foreign_capacity_refusals"` + PendingRefusals int `json:"pending_refusals"` Checkouts []checkoutDiscoveryObservation `json:"checkouts"` ByCWD map[string]checkoutCWDCounts `json:"by_cwd"` OutsidePendingRefusals int `json:"outside_pending_refusals"` @@ -69,6 +73,7 @@ func summarizeCheckoutDiscovery(attempts []Attempt, checkouts []checkoutDiscover for _, checkout := range checkouts { if inside(checkout.Root, a.CWD) { row.Class = "new_worktree" + if checkout.FamilyRole == "foreign" { row.Class = "foreign_new_worktree"; if a.Code != 0 && (a.ErrorData["reason"] == "family_capacity" || a.ErrorData["reason"] == "stuck_capacity") {out.ForeignCapacityRefusals++} } } } if a.Role == "workload" && !strings.HasPrefix(a.Tool, "edit.") { @@ -136,8 +141,9 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu } report := summarizeCheckoutDiscovery(phase.Attempts[first:], checkouts, frozen.Repos[0].Path, frozen.Repos[1].Path) phase.Gate["checkout_discovery"], _ = json.Marshal(report) - if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 { - resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors", report.OutsidePendingRefusals, report.OutsidePendingErrors)) + if o.Discovery.StallFirst && report.PendingRefusals==0 {resultErr=errors.Join(resultErr,errors.New("Git stall produced no pending refusal exposure"))} + if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 || report.ForeignCapacityRefusals > 0 { + resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors, %d foreign capacity refusals", report.OutsidePendingRefusals, report.OutsidePendingErrors, report.ForeignCapacityRefusals)) } if parent.Err() != nil { resultErr = errors.Join(resultErr, parent.Err(), loadErr) @@ -156,13 +162,14 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu case <-ticker.C: } if len(checkouts) < o.Worktrees && !time.Now().Before(nextAdd) { - if err := p.addWorktree(ctx); err != nil { + family := len(checkouts)%2 + if err := p.addDiscoveryWorktree(ctx, family); err != nil { return err } repo := p.Repos[len(p.Repos)-1] - repo.Prefix = p.Repos[0].Prefix - p.Repos[len(p.Repos)-1].Prefix = repo.Prefix - entry := checkoutDiscoveryObservation{Root: repo.Path, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} + role:="primary" + if family==1 {role="foreign"} + entry := checkoutDiscoveryObservation{Root: repo.Path, FamilyRole:role,Prefix:repo.Prefix,Query:repo.Corpus.Query, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} checkouts = append(checkouts, entry) nextAdd = nextAdd.Add(interval) subdir := filepath.Join(repo.Path, ".discovery-probe") @@ -182,13 +189,13 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu if entry.FirstRequest.IsZero() { entry.FirstRequest = time.Now() } - r := request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": p.Repos[0].Corpus.Query}} + r := request{Tool: "search", Operation: "symbols", Arguments: map[string]any{"operation": "symbols", "query": entry.Query}} _, rows, _ := callTool(ctx, client, r, r.Arguments, "probe", i, len(checkouts), 0, o.Deadline, waitContext) phase.Attempts = append(phase.Attempts, rows...) if i%2 != 0 || !entry.ExactAt.IsZero() || ctx.Err() != nil { continue } - raw, err := control(ctx, p.Socket, "search_symbols", map[string]any{"path": entry.Root, "query": p.Repos[0].Corpus.Query, "limit": 1}) + raw, err := control(ctx, p.Socket, "search_symbols", map[string]any{"path": entry.Root, "query": entry.Query, "limit": 1}) entry.View = append(json.RawMessage(nil), raw...) if err != nil { entry.Error = err.Error() @@ -205,7 +212,7 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu entry.Error = err.Error() continue } - if answer.View.Exact && answer.View.Kind == "worktree" && answer.View.Prefix == p.Repos[0].Prefix { + if answer.View.Exact && answer.View.Kind == "worktree" && answer.View.Prefix == entry.Prefix { entry.ExactAt = time.Now() entry.ExactViewLag = measured(float64(entry.ExactAt.Sub(entry.FirstRequest))/float64(time.Millisecond), "ms") entry.Error = "" diff --git a/bench/load/discovery_scope_test.go b/bench/load/discovery_scope_test.go index 28c9e4ed5..ff9939b10 100644 --- a/bench/load/discovery_scope_test.go +++ b/bench/load/discovery_scope_test.go @@ -116,3 +116,34 @@ func TestDiscoveryAddsWorktreesWhileOriginalSessionsKeepReading(t *testing.T) { require.Positive(t, report.ByCWD[filepath.Join(root, ".discovery-probe")].Calls) require.Zero(t, report.OutsidePendingRefusals) } + +func TestDiscoveryForeignCapacityIsNotExcusedAsPending(t *testing.T) { + start:=time.Unix(100,0);root:=filepath.Join(t.TempDir(),"foreign-new") + entries:=[]checkoutDiscoveryObservation{{Root:root,FamilyRole:"foreign",Added:start}} + rows:=[]Attempt{ + {CWD:root,Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"in_progress"}}, + {CWD:root,Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"family_capacity"}}, + {CWD:filepath.Join(root,"subdir"),Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"stuck_capacity"}}, + } + report:=summarizeCheckoutDiscovery(rows,entries,root+"-primary",root+"-other") + require.Equal(t,3,report.PendingRefusals) + require.Equal(t,2,report.ForeignCapacityRefusals,"foreign family capacity cannot be hidden by the new-root pending window") + require.Equal(t,"foreign_new_worktree",report.ByCWD[root].Class) +} + +func TestDiscoveryNewRootsRetainEachUnrelatedFamily(t *testing.T) { + first,second:=gitFixture(t),gitFixture(t);second.Label="other";second.Query="ForeignOnly" + p,err:=newPrivateRun(t.Context(),t.TempDir(),"unused",[]Corpus{first,second});require.NoError(t,err) + t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) + p.Repos[0].Prefix,p.Repos[1].Prefix="primary","foreign" + for _,family:=range []int{0,1,0,1} { + require.NoError(t,p.addDiscoveryWorktree(t.Context(),family)) + linked:=p.Repos[len(p.Repos)-1];primary:=p.Repos[family] + expected,err:=p.git(t.Context(),primary.Path,"rev-parse","--path-format=absolute","--git-common-dir");require.NoError(t,err) + actual,err:=p.git(t.Context(),linked.Path,"rev-parse","--path-format=absolute","--git-common-dir");require.NoError(t,err) + require.Equal(t,string(expected),string(actual),"linked checkout must share only its selected family") + require.Equal(t,primary.Prefix,linked.Prefix) + require.Equal(t,primary.Corpus.Query,linked.Corpus.Query) + } + require.Error(t,p.addDiscoveryWorktree(t.Context(),2)) +} diff --git a/bench/load/discovery_worktree.go b/bench/load/discovery_worktree.go new file mode 100644 index 000000000..328026b9d --- /dev/null +++ b/bench/load/discovery_worktree.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" +) + +// Only this scenario alternates the two unrelated families. Existing worktree +// scenarios keep their primary-only topology and labels. +func (p *privateRun) addDiscoveryWorktree(ctx context.Context, family int) error { + if family<0 || family>1 || len(p.Repos)<2 {return errors.New("discovery requires one of its two initial families")} + primary:=p.Repos[family] + ordinal:=0 + for _,repo:=range p.Repos {if strings.HasPrefix(repo.Corpus.Label,"discovery-linked-"){ordinal++}} + label:=fmt.Sprintf("discovery-linked-%d-%d",family,ordinal) + path:=filepath.Join(p.Root,"repos",label) + if _,err:=p.git(ctx,primary.Path,"worktree","add","--detach",path,primary.Corpus.Commit);err!=nil{return err} + file,err:=ownedFile(path,primary.Corpus.File);if err!=nil{return err} + corpus:=primary.Corpus;corpus.Label=label + p.Repos=append(p.Repos,privateRepo{Corpus:corpus,Path:path,File:file,Prefix:primary.Prefix}) + return nil +} + +func(p *privateRun)writeDiscoveryBootRepos()([]privateRepo,error){ + if len(p.Repos)<2{return nil,errors.New("discovery requires two configured primaries")} + configured:=*p;configured.Repos=p.Repos[:2] + return configured.writeBootRepos(false) +} diff --git a/bench/load/lifecycle.go b/bench/load/lifecycle.go index c7176643f..3ee2b36a2 100644 --- a/bench/load/lifecycle.go +++ b/bench/load/lifecycle.go @@ -28,6 +28,7 @@ type privateRun struct { Store string Config string Env []string + daemonEnv []string // optional owned scenario shim; harness Git keeps Env Repos []privateRepo Binary string process *exec.Cmd @@ -343,7 +344,9 @@ func (p *privateRun) start() error { } cmd := exec.Command(p.Binary, "--config", p.Config, "daemon", "start", "--backend-path", p.Store, "--tools", "facade-v1", "--tools-mode", "hide") configureOwnedProcess(cmd) - cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, p.Env, p.stdout, p.stderr + env:=p.Env + if p.daemonEnv!=nil {env=p.daemonEnv} + cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, env, p.stdout, p.stderr if err := cmd.Start(); err != nil { _ = p.stdout.Close() _ = p.stderr.Close() diff --git a/bench/load/run.go b/bench/load/run.go index 2aaadf31f..c50bc3f83 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -41,7 +41,8 @@ type runOptions struct { EditCount int EditProbeRole string editProbeBaseline bool - Startup startupOptions + Discovery discoveryOptions + Startup startupOptions pairedDriver func(context.Context, *privateRun, []*socketClient, []*socketClient, *Phase) error smokeState *privateRun // only the fixed smoke entry lends its freshly owned store smokeMarkers []string @@ -85,6 +86,7 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") bindStartupFlags(f, &o.Startup) + bindDiscoveryFlags(f,&o.Discovery) f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } @@ -99,7 +101,8 @@ func validateRunOptions(o runOptions) error { if err := validateAgentCounts(o.Scenario, o.Agents, o.Editors); err != nil { return err } - if err := o.Startup.validate(o.Scenario); err != nil { + if err:=o.Discovery.validate(o.Scenario,o.Worktrees);err!=nil{return err} + if err := o.Startup.validate(o.Scenario); err != nil { return err } if o.Scenario == "idle-stop" && (o.Worktrees != 0 || o.CPUProfile != 0 || o.Profile != "agent") { diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 6be861960..ba54727c7 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -167,11 +167,23 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp } warmCtx, cancelWarm := context.WithTimeout(ctx, o.WarmTimeout) defer cancelWarm() - configured, err := p.writeBootRepos(scenario == "worktrees") + var configured []privateRepo + if scenario==checkoutDiscoveryScenario {configured,err=p.writeDiscoveryBootRepos()}else{configured,err=p.writeBootRepos(scenario == "worktrees")} if err != nil { return phase, nil, artifacts, crashes, "", err } - coldStart := time.Now() + if scenario==checkoutDiscoveryScenario && o.Discovery.StallFirst { + release,files,err:=p.installDiscoveryFault();if err!=nil{return phase,nil,artifacts,crashes,"",err} + for name,path:=range files {artifacts[name]=path} + defer func(){ + resultErr=errors.Join(resultErr,release()) + raw,err:=os.ReadFile(files["discovery_git_events"]) + observed:=err==nil && len(bytes.TrimSpace(raw))>0 + phase.Gate["discovery_git_fault"],_=json.Marshal(map[string]any{"configured":true,"observed":observed,"raw_events":string(raw),"absolute_child_bound_seconds":60}) + if !observed {resultErr=errors.Join(resultErr,errors.New("Git stall never entered the intended new checkout"),err)} + }() + } + coldStart := time.Now() if err := p.start(); err != nil { return phase, nil, artifacts, crashes, "", err } @@ -233,7 +245,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp p.Repos[i].Prefix = tracked.Prefix } } - if i > 0 && scenario == "worktrees" { + if i > 0 && scenario == "worktrees" || i>=2 && scenario==checkoutDiscoveryScenario { p.Repos[i].Prefix = p.Repos[0].Prefix } if p.Repos[i].Prefix == "" { @@ -263,7 +275,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp version = client.version if i < len(p.Repos) { subscribeInvalidations(warmCtx, client, repo.Corpus.Label, o.Deadline, &phase) - if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0, o.Deadline, &phase); err != nil { + if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0 || scenario==checkoutDiscoveryScenario && i>=2, o.Deadline, &phase); err != nil { return phase, nil, artifacts, crashes, version, err } gate, rows, err := warmAnalysis(warmCtx, client, o.Deadline) From 6dd9a3538a776250d2d54fe7a81b2831d1ddd85e Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:02:04 +0100 Subject: [PATCH 51/60] feat(load): resume upgraded discovery at fixed owned roots Source checkpoint; validation is pending shared lane availability. --- bench/load/README.md | 20 ++++ bench/load/discovery_fault.go | 15 ++- bench/load/discovery_seed.go | 163 ++++++++++++++++++++++++++++++ bench/load/discovery_seed_test.go | 64 ++++++++++++ bench/load/discovery_worktree.go | 11 ++ bench/load/run.go | 3 +- bench/load/scenario.go | 3 +- 7 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 bench/load/discovery_seed.go create mode 100644 bench/load/discovery_seed_test.go diff --git a/bench/load/README.md b/bench/load/README.md index 41d18d8a8..2ba04ee2a 100644 --- a/bench/load/README.md +++ b/bench/load/README.md @@ -785,3 +785,23 @@ for the first new primary-family checkout. Its child ignores TERM and retains stdout until explicit release or a 60-second bound. The harness releases it before daemon shutdown and retains child/root/operation events. A missing stall or pending-refusal observation fails this cell. + +For upgraded fixtures, `--discovery-prepare ` prepares owned +roots without opening a store or launching a daemon. Add +`--discovery-existing-worktrees 20 --discovery-existing-nested 5` to create five +in-repository and fifteen sibling checkouts of the first family. The unrelated +control is retained, so descriptor counts are two primaries plus twenty linked +checkouts; selected-family counts are one plus twenty. The descriptor records +actual paths, Git common directories, commits, source and configuration hashes, +and a random owner token. Preparation requires a separate exclusive fixture +producer to populate and upgrade the store at those exact paths. + +`--discovery-resume ` consumes that prepared root once. Lineage +must bind the descriptor, old and upgrade binary/source pins, schema-44 main +and WAL hashes, and structured normal-stop/catalog-identity receipts. Unknown, +live, changed or relocated identities are refused. The stopped immutable +snapshot remains inside the owned root; the destination store must be absent. +The existing seed copier copies only main/WAL files, then both source and +copied hashes are checked again. There is no SQL or catalog path rewriting. +These discovery-only flags preserve the startup scenario's `--seed-store` +contract. A copied fixture or synthetic parser control is not upgrade acceptance. diff --git a/bench/load/discovery_fault.go b/bench/load/discovery_fault.go index f2f376f52..96b25ff44 100644 --- a/bench/load/discovery_fault.go +++ b/bench/load/discovery_fault.go @@ -11,9 +11,20 @@ import ( "strings" ) -type discoveryOptions struct { StallFirst bool } -func bindDiscoveryFlags(f *flag.FlagSet,o *discoveryOptions){f.BoolVar(&o.StallFirst,"discovery-stall-first",false,"worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound")} +type discoveryOptions struct { StallFirst bool; Prepare string; Resume string; ExistingWorktrees int; ExistingNested int } +func bindDiscoveryFlags(f *flag.FlagSet,o *discoveryOptions){ + f.StringVar(&o.Prepare,"discovery-prepare","","worktree-discovery: prepare owned roots and write a descriptor without launching a daemon") + f.StringVar(&o.Resume,"discovery-resume","","worktree-discovery: consume exact owned upgraded-store lineage at unchanged prepared roots") + f.IntVar(&o.ExistingNested,"discovery-existing-nested",0,"discovery preparation: existing linked checkouts placed inside the primary repository") + f.IntVar(&o.ExistingWorktrees,"discovery-existing-worktrees",0,"discovery preparation: existing linked checkouts of the first family") + f.BoolVar(&o.StallFirst,"discovery-stall-first",false,"worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound")} func(o discoveryOptions)validate(scenario string,count int)error{ + if o.ExistingNested<0||o.ExistingNested>o.ExistingWorktrees{return errors.New("discovery existing nested count must fit the existing topology")} + if o.ExistingWorktrees<0{return errors.New("discovery existing-worktree count must be nonnegative")} + if o.Prepare!="" || o.Resume!="" || o.ExistingWorktrees!=0 || o.ExistingNested!=0 { + if scenario!=checkoutDiscoveryScenario||o.Prepare!=""&&o.Resume!=""||o.ExistingWorktrees!=0&&o.Prepare=="" {return errors.New("discovery prepare/resume flags require that scenario, exclusive modes and preparation for existing topology")} + if runtime.GOOS=="windows"{return errors.New("owned discovery seed resume currently requires Unix owner checks")} + } if !o.StallFirst{return nil} if scenario!=checkoutDiscoveryScenario||count<1{return errors.New("--discovery-stall-first requires worktree-discovery and at least one new worktree")} if runtime.GOOS=="windows" {return errors.New("discovery Git stall requires Unix process ownership")} diff --git a/bench/load/discovery_seed.go b/bench/load/discovery_seed.go new file mode 100644 index 000000000..5826dea56 --- /dev/null +++ b/bench/load/discovery_seed.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" +) + +type discoverySeedRepo struct { + Repo privateRepo `json:"repo"` + Role string `json:"role"` + FileSHA256 string `json:"file_sha256"` + CommonDir string `json:"git_common_dir"` +} +type discoverySeedDescriptor struct { + State string `json:"state"` + Root string `json:"owned_root"` + Nonce string `json:"owner_nonce"` + ConfigSHA256 string `json:"config_sha256"` + BootConfigSHA256 string `json:"boot_config_sha256"` + GitConfigSHA256 string `json:"git_config_sha256"` + Repos []discoverySeedRepo `json:"repositories"` + PrimaryCount int `json:"primary_checkout_count"` + LinkedCount int `json:"linked_checkout_count"` + NestedCount int `json:"nested_linked_checkout_count"` + TotalCount int `json:"total_checkout_count"` +} +type discoverySeedLineage struct { + State string `json:"state"` + Descriptor string `json:"descriptor"` + DescriptorSHA256 string `json:"descriptor_sha256"` + OwnedRoot string `json:"owned_root"` + SourceGeneration string `json:"source_generation"` + OldVersion string `json:"old_version"` + OldBinarySHA256 string `json:"old_binary_sha256"` + UpgradeSourceCommit string `json:"upgrade_source_commit"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Snapshot string `json:"snapshot"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + OwnerPID int `json:"owner_pid"` + StopReceipt string `json:"stop_receipt"` + StopReceiptSHA256 string `json:"stop_receipt_sha256"` + IdentityReceipt string `json:"identity_receipt"` + IdentityReceiptSHA256 string `json:"identity_receipt_sha256"` + Repos []discoverySeedRepo `json:"repositories"` +} + +func discoveryFileSHA(path string)(string,error){ + info,err:=os.Lstat(path);if err!=nil{return "",err};if !info.Mode().IsRegular(){return "",errors.New("discovery lineage requires regular files, not symlinks")} + f,err:=os.Open(path);if err!=nil{return "",err};h:=sha256.New();_,copyErr:=io.Copy(h,f);if err:=errors.Join(copyErr,f.Close());err!=nil{return "",err};return hex.EncodeToString(h.Sum(nil)),nil +} +func discoveryReadJSON(path string,out any)error{ + f,err:=os.Open(path);if err!=nil{return err};defer f.Close();d:=json.NewDecoder(io.LimitReader(f,4<<20));d.DisallowUnknownFields();if err:=d.Decode(out);err!=nil{return err};if err:=d.Decode(new(any));err!=io.EOF{return errors.New("discovery descriptor must contain exactly one JSON object")};return nil +} +func discoveryWriteJSON(path string,value any)error{ + b,err:=json.MarshalIndent(value,""," ");if err!=nil{return err};f,err:=os.OpenFile(path,os.O_WRONLY|os.O_CREATE|os.O_EXCL,0o600);if err!=nil{return err};_,err=f.Write(append(b,'\n'));return errors.Join(err,f.Close()) +} + +// Preparation only makes owned clones/topology and metadata. The exclusive +// producer must populate and upgrade this exact root before resume can pass. +func prepareDiscoveryState(ctx context.Context,o runOptions,corpora []Corpus)error{ + if len(corpora)<2{return errors.New("discovery preparation requires two unrelated pinned corpora")} + if err:=validateOutputRoots(corpora,o.Discovery.Prepare);err!=nil{return err} + if _,err:=os.Lstat(o.Discovery.Prepare);!errors.Is(err,os.ErrNotExist){return errors.New("discovery descriptor destination must be absent")} + p,err:=newPrivateRun(ctx,o.Scratch,o.Binary,corpora[:2]);if err!=nil{return err};defer os.RemoveAll(p.SocketDir) + complete:=false;defer func(){if !complete{_=p.removeState()}}() + for i:=range o.Discovery.ExistingWorktrees {if err:=p.addExistingDiscoveryWorktree(ctx,i,id.LinkedCount||d.TotalCount!=2+d.LinkedCount||len(d.Repos)!=d.TotalCount||!reflect.DeepEqual(l.Repos,d.Repos){return nil,errors.New("discovery lineage topology/identity differs from preparation")} + if len(corpora)!=2||!reflect.DeepEqual(d.Repos[0].Repo.Corpus,corpora[0])||!reflect.DeepEqual(d.Repos[1].Repo.Corpus,corpora[1]){return nil,errors.New("discovery seed corpus pins mismatch")} + for path,want:=range map[string]string{filepath.Join(root,"config.yaml"):d.ConfigSHA256,filepath.Join(root,"config","gortex","config.yaml"):d.BootConfigSHA256,filepath.Join(root,"gitconfig"):d.GitConfigSHA256,l.StopReceipt:l.StopReceiptSHA256,l.IdentityReceipt:l.IdentityReceiptSHA256}{if err:=discoveryOwnedFile(root,path);err!=nil{return nil,err};actual,err:=discoveryFileSHA(path);if err!=nil||!discoveryDigestValid(want,32)||actual!=want{return nil,errors.New("discovery configuration/producer proof digest mismatch")}} + for _,proofPath:=range []string{l.StopReceipt,l.IdentityReceipt}{ + var proof discoverySeedProof + if err:=discoveryReadJSON(proofPath,&proof);err!=nil{return nil,err} + if proof.State!="OBSERVED_OWNED_UPGRADE"||proof.ObservedAt.IsZero()||proof.OwnedRoot!=root||proof.OwnerPID!=l.OwnerPID||!proof.NormalStop||!proof.OwnersExited||proof.MainSHA256!=l.MainSHA256||proof.WALSHA256!=l.WALSHA256||proof.SchemaVersion!=44||proof.UpgradeBinarySHA256!=l.UpgradeBinarySHA256||!reflect.DeepEqual(proof.Repos,d.Repos){return nil,errors.New("discovery producer proof is unknown or does not bind this stopped snapshot and identity topology")} + } + process,err:=os.FindProcess(l.OwnerPID);if err!=nil{return nil,err};err=process.Signal(syscall.Signal(0));_=process.Release();if !errors.Is(err,syscall.ESRCH)&&!errors.Is(err,os.ErrProcessDone){return nil,errors.New("discovery upgrade owner is live or cannot be proven exited")} + binaryPath,err:=filepath.Abs(o.Binary);if err!=nil{return nil,err} + p:=&privateRun{Root:root,Store:filepath.Join(root,"store.db"),Config:filepath.Join(root,"config.yaml"),Binary:binaryPath} + for _,entry:=range d.Repos{ + repo:=entry.Repo;if err:=discoveryOwnedFile(root,repo.File);err!=nil{return nil,err};if !inside(root,repo.Path)||repo.File!=filepath.Join(repo.Path,repo.Corpus.File){return nil,errors.New("discovery repository path mismatch")} + actual,err:=discoveryFileSHA(repo.File);if err!=nil||actual!=entry.FileSHA256{return nil,errors.New("discovery prepared source content changed")};p.Repos=append(p.Repos,repo) + } + p.Env=privateEnv(root,"",filepath.Join(root,"gitconfig")) + for i,repo:=range p.Repos{head,err:=p.git(ctx,repo.Path,"rev-parse","HEAD");if err!=nil||strings.TrimSpace(string(head))!=repo.Corpus.Commit{return nil,errors.New("discovery prepared checkout HEAD changed")} + common,err:=p.git(ctx,repo.Path,"rev-parse","--path-format=absolute","--git-common-dir");if err!=nil||strings.TrimSpace(string(common))!=d.Repos[i].CommonDir {return nil,errors.New("discovery prepared Git family changed")}} + if err:=discoveryOwnedFile(root,l.Snapshot);err!=nil{return nil,err} + for _,suffix:=range []string{"","-wal","-shm"}{if _,err:=os.Lstat(p.Store+suffix);!errors.Is(err,os.ErrNotExist){return nil,errors.New("discovery destination store must be absent before one-shot resume")}} + before,err:=discoverySeedPair(l.Snapshot);if err!=nil||before[0]!=l.MainSHA256||before[1]!=l.WALSHA256{return nil,errors.New("discovery quiescent snapshot digest mismatch")} + f,err:=os.Open(l.Snapshot);if err!=nil{return nil,err};header:=make([]byte,100);_,readErr:=io.ReadFull(f,header);err=errors.Join(readErr,f.Close());if err!=nil||string(header[:16])!="SQLite format 3\x00"||binary.BigEndian.Uint32(header[60:64])!=44{return nil,errors.New("discovery snapshot is not a checkpointed schema44 SQLite main")} + if err:=discoveryWriteJSON(filepath.Join(root,"discovery-consumed.json"),map[string]string{"descriptor_sha256":sha,"upgrade_binary_sha256":l.UpgradeBinarySHA256});err!=nil{return nil,err} + if err:=copySeedStore(l.Snapshot,p.Store);err!=nil{return nil,err} + after,err:=discoverySeedPair(l.Snapshot);if err!=nil||before!=after{return nil,errors.New("discovery snapshot changed during copy; partial destination retained")};copied,err:=discoverySeedPair(p.Store);if err!=nil||copied!=before{return nil,errors.New("discovery seed copy differs; partial destination retained")} + p.SocketDir,err=os.MkdirTemp("/tmp","gl-");if err!=nil{return nil,err};p.Socket=filepath.Join(p.SocketDir,"d.sock");p.Env=privateEnv(root,p.Socket,filepath.Join(root,"gitconfig"));return p,nil +} +func discoverySeedPair(path string)([2]string,error){ + var out [2]string;var err error + if _,err:=os.Lstat(path+"-shm");!errors.Is(err,os.ErrNotExist){return out,errors.New("discovery snapshot has shared-memory state or unknown ownership")} + out[0],err=discoveryFileSHA(path);if err!=nil{return out,err};out[1],err=discoveryFileSHA(path+"-wal");if errors.Is(err,os.ErrNotExist){err=nil};return out,err +} + +// Both retained producer receipts must bind observations to this exact stopped +// output; state assertions without snapshot and identity evidence are refused. +type discoverySeedProof struct { + State string `json:"state"` + ObservedAt time.Time `json:"observed_at"` + OwnedRoot string `json:"owned_root"` + OwnerPID int `json:"owner_pid"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Repos []discoverySeedRepo `json:"repositories"` +} diff --git a/bench/load/discovery_seed_test.go b/bench/load/discovery_seed_test.go new file mode 100644 index 000000000..160effdf2 --- /dev/null +++ b/bench/load/discovery_seed_test.go @@ -0,0 +1,64 @@ +//go:build linux || darwin + +package main + +import ( + "encoding/binary" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// This is only a byte-copy/parser fixture, deliberately not a real SQLite +// database or upgrade receipt. It never establishes upgrade acceptance. +func syntheticDiscoverySeed(t *testing.T)(runOptions,[]Corpus,discoverySeedLineage){ + t.Helper();first,second:=gitFixture(t),gitFixture(t);second.Label="other" + scratch:=t.TempDir();descriptor:=filepath.Join(scratch,"prepared.json") + o:=runOptions{Binary:"unused",Scratch:scratch,Discovery:discoveryOptions{Prepare:descriptor,ExistingWorktrees:2,ExistingNested:1}} + corpora:=[]Corpus{first,second};require.NoError(t,prepareDiscoveryState(t.Context(),o,corpora)) + var d discoverySeedDescriptor;require.NoError(t,discoveryReadJSON(descriptor,&d)) + snapshot:=filepath.Join(d.Root,"synthetic-byte-snapshot.db") + header:=make([]byte,100);copy(header,"SQLite format 3\x00");binary.BigEndian.PutUint32(header[60:64],44) + require.NoError(t,os.WriteFile(snapshot,header,0o600));mainSHA,err:=discoveryFileSHA(snapshot);require.NoError(t,err) + child:=exec.Command("sh","-c","exit 0");require.NoError(t,child.Run()) + l:=discoverySeedLineage{State:"VALIDATED_OWNED_UPGRADE",Descriptor:descriptor,OwnedRoot:d.Root,SourceGeneration:"synthetic-test-only",OldVersion:"synthetic",OldBinarySHA256:strings.Repeat("1",64),UpgradeSourceCommit:strings.Repeat("2",40),UpgradeBinarySHA256:strings.Repeat("3",64),Snapshot:snapshot,MainSHA256:mainSHA,SchemaVersion:44,NormalStop:true,OwnersExited:true,OwnerPID:child.Process.Pid,StopReceipt:filepath.Join(d.Root,"synthetic-stop.json"),IdentityReceipt:filepath.Join(d.Root,"synthetic-identity.json"),Repos:d.Repos} + l.DescriptorSHA256,err=discoveryFileSHA(descriptor);require.NoError(t,err) + rewriteSyntheticDiscoveryProofs(t,&l) + o.Discovery=discoveryOptions{Resume:filepath.Join(scratch,"lineage.json")} + return o,corpora,l +} +func rewriteSyntheticDiscoveryProofs(t *testing.T,l *discoverySeedLineage){ + t.Helper();proof:=discoverySeedProof{State:"OBSERVED_OWNED_UPGRADE",ObservedAt:time.Now(),OwnedRoot:l.OwnedRoot,OwnerPID:l.OwnerPID,NormalStop:l.NormalStop,OwnersExited:l.OwnersExited,MainSHA256:l.MainSHA256,WALSHA256:l.WALSHA256,SchemaVersion:l.SchemaVersion,UpgradeBinarySHA256:l.UpgradeBinarySHA256,Repos:l.Repos} + for _,path:=range []string{l.StopReceipt,l.IdentityReceipt}{_=os.Remove(path);require.NoError(t,discoveryWriteJSON(path,proof))} + var err error;l.StopReceiptSHA256,err=discoveryFileSHA(l.StopReceipt);require.NoError(t,err);l.IdentityReceiptSHA256,err=discoveryFileSHA(l.IdentityReceipt);require.NoError(t,err) +} +func TestDiscoverySeedResumeRefusesUnknownLiveAndMismatchedLineage(t *testing.T){ + for _,name:=range []string{"unknown","topology","snapshot","live","shared_memory","changed_git_family"}{t.Run(name,func(t *testing.T){ + o,corpora,l:=syntheticDiscoverySeed(t) + switch name{ + case "unknown":l.State="TEMPLATE" + case "topology":l.Repos=append([]discoverySeedRepo(nil),l.Repos...);l.Repos[2].Repo.Prefix="foreign" + case "snapshot":l.MainSHA256=strings.Repeat("0",64);rewriteSyntheticDiscoveryProofs(t,&l) + case "live":l.OwnerPID=os.Getpid();rewriteSyntheticDiscoveryProofs(t,&l) + case "shared_memory":require.NoError(t,os.WriteFile(l.Snapshot+"-shm",nil,0o600)) + case "changed_git_family":require.NoError(t,os.WriteFile(filepath.Join(l.Repos[2].Repo.Path,".git"),[]byte("gitdir: "+filepath.Join(l.Repos[1].Repo.Path,".git")+"\n"),0o600)) + } + require.NoError(t,discoveryWriteJSON(o.Discovery.Resume,l)) + _,err:=resumeDiscoveryState(t.Context(),o,corpora);require.Error(t,err) + require.NoFileExists(t,filepath.Join(l.OwnedRoot,"store.db"),"refused lineage must not create a destination store") + })} +} +func TestDiscoverySeedCopiesSnapshotWithoutChangingExistingIdentities(t *testing.T){ + o,corpora,l:=syntheticDiscoverySeed(t);require.NoError(t,discoveryWriteJSON(o.Discovery.Resume,l)) + p,err:=resumeDiscoveryState(t.Context(),o,corpora);require.NoError(t,err);t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) + require.Equal(t,l.OwnedRoot,p.Root);require.Len(t,p.Repos,4) + require.True(t,inside(p.Repos[0].Path,p.Repos[2].Path));require.False(t,inside(p.Repos[0].Path,p.Repos[3].Path)) + for i,repo:=range p.Repos{require.Equal(t,l.Repos[i].Repo.Path,repo.Path);require.Equal(t,l.Repos[i].Repo.Prefix,repo.Prefix)} + source,err:=discoveryFileSHA(l.Snapshot);require.NoError(t,err);dest,err:=discoveryFileSHA(p.Store);require.NoError(t,err);require.Equal(t,l.MainSHA256,source);require.Equal(t,source,dest) + _,err=resumeDiscoveryState(t.Context(),o,corpora);require.Error(t,err,"a prepared run is consumed once") +} diff --git a/bench/load/discovery_worktree.go b/bench/load/discovery_worktree.go index 328026b9d..ea68160e2 100644 --- a/bench/load/discovery_worktree.go +++ b/bench/load/discovery_worktree.go @@ -6,6 +6,7 @@ import ( "fmt" "path/filepath" "strings" + "os" ) // Only this scenario alternates the two unrelated families. Existing worktree @@ -29,3 +30,13 @@ func(p *privateRun)writeDiscoveryBootRepos()([]privateRepo,error){ configured:=*p;configured.Repos=p.Repos[:2] return configured.writeBootRepos(false) } + +func(p *privateRun)addExistingDiscoveryWorktree(ctx context.Context,ordinal int,nested bool)error{ + if len(p.Repos)<2{return errors.New("existing discovery topology requires two initial primaries")} + primary:=p.Repos[0];label:=fmt.Sprintf("existing-linked-%d",ordinal) + path:=filepath.Join(p.Root,"repos",label) + if nested {path=filepath.Join(primary.Path,".load-existing",label);if err:=os.MkdirAll(filepath.Dir(path),0o700);err!=nil{return err}} + if _,err:=p.git(ctx,primary.Path,"worktree","add","--detach",path,primary.Corpus.Commit);err!=nil{return err} + file,err:=ownedFile(path,primary.Corpus.File);if err!=nil{return err} + corpus:=primary.Corpus;corpus.Label=label;p.Repos=append(p.Repos,privateRepo{Corpus:corpus,Path:path,File:file,Prefix:primary.Prefix});return nil +} diff --git a/bench/load/run.go b/bench/load/run.go index c50bc3f83..86cdc4df6 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -215,7 +215,8 @@ func runCommand(ctx context.Context, args []string) error { if err := validateOutputRoots(corpora, o.Scratch, o.Output); err != nil { return err } - sha, err := binaryRevision(o.Binary, o.GitSHA) + if o.Discovery.Prepare!="" {return prepareDiscoveryState(ctx,o,corpora)} + sha, err := binaryRevision(o.Binary, o.GitSHA) if err != nil { return err } diff --git a/bench/load/scenario.go b/bench/load/scenario.go index ba54727c7..03b3d0c4a 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -101,7 +101,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp p := o.smokeState var err error if p == nil { - p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected) + if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" {p,err=resumeDiscoveryState(ctx,o,selected)} else {p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected)} } else { err = validateSmokeOwner(p, o.Binary, selected) } @@ -109,6 +109,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp return phase, nil, artifacts, crashes, "", err } artifacts["private_root"] = p.Root + if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" {raw,err:=os.ReadFile(o.Discovery.Resume);if err!=nil{return phase,nil,artifacts,crashes,"",err};phase.Gate["discovery_seed_lineage"]=json.RawMessage(raw)} firstStart := p.startCount + 1 if o.smokeState != nil { info, statErr := os.Stat(p.Store) From df942d3e50e6b2d623e771c1999ca4d702161fe1 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:03:47 +0100 Subject: [PATCH 52/60] fix(load): refuse changed identities after seeded startup Source checkpoint; validation remains pending fair lane rotation. --- bench/load/scenario.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 03b3d0c4a..87a8caa0b 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -243,7 +243,8 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp for i := range p.Repos { for _, tracked := range readinessState.TrackedRepos { if filepath.Clean(tracked.Path) == filepath.Clean(p.Repos[i].Path) { - p.Repos[i].Prefix = tracked.Prefix + if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" && p.Repos[i].Prefix!=tracked.Prefix {return phase,nil,artifacts,crashes,"",errors.New("resumed discovery changed a prepared repository identity prefix")} + p.Repos[i].Prefix = tracked.Prefix } } if i > 0 && scenario == "worktrees" || i>=2 && scenario==checkoutDiscoveryScenario { From 793bed991ded24c3d6a002b74b7bff7c6bcebe55 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:08:09 +0100 Subject: [PATCH 53/60] style(load): format validated discovery adapters --- bench/load/discovery_fault.go | 131 +++++--- bench/load/discovery_fault_test.go | 131 +++++--- bench/load/discovery_scope.go | 43 +-- bench/load/discovery_scope_test.go | 57 ++-- bench/load/discovery_seed.go | 470 +++++++++++++++++++++-------- bench/load/discovery_seed_test.go | 145 ++++++--- bench/load/discovery_worktree.go | 89 ++++-- bench/load/lifecycle.go | 10 +- bench/load/run.go | 18 +- bench/load/scenario.go | 61 ++-- 10 files changed, 785 insertions(+), 370 deletions(-) diff --git a/bench/load/discovery_fault.go b/bench/load/discovery_fault.go index 96b25ff44..20b1585f6 100644 --- a/bench/load/discovery_fault.go +++ b/bench/load/discovery_fault.go @@ -1,48 +1,85 @@ package main import ( - "errors" - "flag" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" + "errors" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" ) -type discoveryOptions struct { StallFirst bool; Prepare string; Resume string; ExistingWorktrees int; ExistingNested int } -func bindDiscoveryFlags(f *flag.FlagSet,o *discoveryOptions){ - f.StringVar(&o.Prepare,"discovery-prepare","","worktree-discovery: prepare owned roots and write a descriptor without launching a daemon") - f.StringVar(&o.Resume,"discovery-resume","","worktree-discovery: consume exact owned upgraded-store lineage at unchanged prepared roots") - f.IntVar(&o.ExistingNested,"discovery-existing-nested",0,"discovery preparation: existing linked checkouts placed inside the primary repository") - f.IntVar(&o.ExistingWorktrees,"discovery-existing-worktrees",0,"discovery preparation: existing linked checkouts of the first family") - f.BoolVar(&o.StallFirst,"discovery-stall-first",false,"worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound")} -func(o discoveryOptions)validate(scenario string,count int)error{ - if o.ExistingNested<0||o.ExistingNested>o.ExistingWorktrees{return errors.New("discovery existing nested count must fit the existing topology")} - if o.ExistingWorktrees<0{return errors.New("discovery existing-worktree count must be nonnegative")} - if o.Prepare!="" || o.Resume!="" || o.ExistingWorktrees!=0 || o.ExistingNested!=0 { - if scenario!=checkoutDiscoveryScenario||o.Prepare!=""&&o.Resume!=""||o.ExistingWorktrees!=0&&o.Prepare=="" {return errors.New("discovery prepare/resume flags require that scenario, exclusive modes and preparation for existing topology")} - if runtime.GOOS=="windows"{return errors.New("owned discovery seed resume currently requires Unix owner checks")} - } - if !o.StallFirst{return nil} - if scenario!=checkoutDiscoveryScenario||count<1{return errors.New("--discovery-stall-first requires worktree-discovery and at least one new worktree")} - if runtime.GOOS=="windows" {return errors.New("discovery Git stall requires Unix process ownership")} - return nil +type discoveryOptions struct { + StallFirst bool + Prepare string + Resume string + ExistingWorktrees int + ExistingNested int } -func discoveryShellLiteral(s string)string{return "'"+strings.ReplaceAll(s,"'","'\"'\"'")+"'"} +func bindDiscoveryFlags(f *flag.FlagSet, o *discoveryOptions) { + f.StringVar(&o.Prepare, "discovery-prepare", "", "worktree-discovery: prepare owned roots and write a descriptor without launching a daemon") + f.StringVar(&o.Resume, "discovery-resume", "", "worktree-discovery: consume exact owned upgraded-store lineage at unchanged prepared roots") + f.IntVar(&o.ExistingNested, "discovery-existing-nested", 0, "discovery preparation: existing linked checkouts placed inside the primary repository") + f.IntVar(&o.ExistingWorktrees, "discovery-existing-worktrees", 0, "discovery preparation: existing linked checkouts of the first family") + f.BoolVar(&o.StallFirst, "discovery-stall-first", false, "worktree-discovery: hold Git stdout for the first new owned checkout, ignoring TERM with a 60-second child bound") +} +func (o discoveryOptions) validate(scenario string, count int) error { + if o.ExistingNested < 0 || o.ExistingNested > o.ExistingWorktrees { + return errors.New("discovery existing nested count must fit the existing topology") + } + if o.ExistingWorktrees < 0 { + return errors.New("discovery existing-worktree count must be nonnegative") + } + if o.Prepare != "" || o.Resume != "" || o.ExistingWorktrees != 0 || o.ExistingNested != 0 { + if scenario != checkoutDiscoveryScenario || o.Prepare != "" && o.Resume != "" || o.ExistingWorktrees != 0 && o.Prepare == "" { + return errors.New("discovery prepare/resume flags require that scenario, exclusive modes and preparation for existing topology") + } + if runtime.GOOS == "windows" { + return errors.New("owned discovery seed resume currently requires Unix owner checks") + } + } + if !o.StallFirst { + return nil + } + if scenario != checkoutDiscoveryScenario || count < 1 { + return errors.New("--discovery-stall-first requires worktree-discovery and at least one new worktree") + } + if runtime.GOOS == "windows" { + return errors.New("discovery Git stall requires Unix process ownership") + } + return nil +} + +func discoveryShellLiteral(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } // No shell runs while preparing this adapter. Only the owned daemon receives // its PATH; worktree creation and all harness Git retain the real executable. -func(p *privateRun)installDiscoveryFault()(func()error,map[string]string,error){ - if runtime.GOOS=="windows"{return nil,nil,errors.New("discovery Git stall requires Unix")} - root,err:=canonicalPath(p.Root);if err!=nil||root!=p.Root{return nil,nil,errors.New("discovery fault requires its canonical private root")} - realGit,err:=exec.LookPath("git");if err!=nil{return nil,nil,err};realGit,err=filepath.Abs(realGit);if err!=nil{return nil,nil,err} - dir:=filepath.Join(p.Root,"discovery-git-fault");if err:=os.Mkdir(dir,0o700);err!=nil{return nil,nil,err} - target:=filepath.Join(p.Root,"repos","discovery-linked-0-0") - release,events:=filepath.Join(dir,"release"),filepath.Join(dir,"events.tsv") - script:=fmt.Sprintf(`#!/bin/sh +func (p *privateRun) installDiscoveryFault() (func() error, map[string]string, error) { + if runtime.GOOS == "windows" { + return nil, nil, errors.New("discovery Git stall requires Unix") + } + root, err := canonicalPath(p.Root) + if err != nil || root != p.Root { + return nil, nil, errors.New("discovery fault requires its canonical private root") + } + realGit, err := exec.LookPath("git") + if err != nil { + return nil, nil, err + } + realGit, err = filepath.Abs(realGit) + if err != nil { + return nil, nil, err + } + dir := filepath.Join(p.Root, "discovery-git-fault") + if err := os.Mkdir(dir, 0o700); err != nil { + return nil, nil, err + } + target := filepath.Join(p.Root, "repos", "discovery-linked-0-0") + release, events := filepath.Join(dir, "release"), filepath.Join(dir, "events.tsv") + script := fmt.Sprintf(`#!/bin/sh cwd=$(pwd -P) want_cwd=0 operation= @@ -66,10 +103,22 @@ case "$cwd" in %s|%s/*) fi;; esac exec %s "$@" -`,discoveryShellLiteral(target),discoveryShellLiteral(target),discoveryShellLiteral(events),discoveryShellLiteral(release),discoveryShellLiteral(events),discoveryShellLiteral(realGit)) - shim:=filepath.Join(dir,"git");if err:=os.WriteFile(shim,[]byte(script),0o700);err!=nil{return nil,nil,err} - env:=append([]string(nil),p.Env...);found:=false - for i,entry:=range env{if strings.HasPrefix(entry,"PATH="){env[i]="PATH="+dir+string(os.PathListSeparator)+strings.TrimPrefix(entry,"PATH=");found=true}} - if !found{env=append(env,"PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"))};p.daemonEnv=env - return func()error{return os.WriteFile(release,[]byte("release owned Git descendants\n"),0o600)},map[string]string{"discovery_git_shim":shim,"discovery_git_events":events,"discovery_git_release":release},nil +`, discoveryShellLiteral(target), discoveryShellLiteral(target), discoveryShellLiteral(events), discoveryShellLiteral(release), discoveryShellLiteral(events), discoveryShellLiteral(realGit)) + shim := filepath.Join(dir, "git") + if err := os.WriteFile(shim, []byte(script), 0o700); err != nil { + return nil, nil, err + } + env := append([]string(nil), p.Env...) + found := false + for i, entry := range env { + if strings.HasPrefix(entry, "PATH=") { + env[i] = "PATH=" + dir + string(os.PathListSeparator) + strings.TrimPrefix(entry, "PATH=") + found = true + } + } + if !found { + env = append(env, "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH")) + } + p.daemonEnv = env + return func() error { return os.WriteFile(release, []byte("release owned Git descendants\n"), 0o600) }, map[string]string{"discovery_git_shim": shim, "discovery_git_events": events, "discovery_git_release": release}, nil } diff --git a/bench/load/discovery_fault_test.go b/bench/load/discovery_fault_test.go index c0f48a326..468576607 100644 --- a/bench/load/discovery_fault_test.go +++ b/bench/load/discovery_fault_test.go @@ -3,55 +3,94 @@ package main import ( - "context" - "bytes" - "os" - "os/exec" - "path/filepath" - "strconv" - "strings" - "syscall" - "testing" - "time" + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/require" ) -func TestDiscoveryFaultRetainsOwnedPipeAndDoesNotAffectHarnessGit(t *testing.T){ - first,second:=gitFixture(t),gitFixture(t);second.Label="other" - p,err:=newPrivateRun(t.Context(),t.TempDir(),"unused",[]Corpus{first,second});require.NoError(t,err) - t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) - originalEnv:=append([]string(nil),p.Env...) - release,files,err:=p.installDiscoveryFault();require.NoError(t,err) - t.Cleanup(func(){_=release()}) - require.Equal(t,originalEnv,p.Env,"harness environment remains unchanged") - require.NotEqual(t,p.Env,p.daemonEnv) - require.NoError(t,p.addDiscoveryWorktree(t.Context(),0),"harness worktree creation must never enter its daemon shim") - linked:=p.Repos[len(p.Repos)-1] - ctx,cancel:=context.WithTimeout(t.Context(),10*time.Second);defer cancel() - cmd:=exec.CommandContext(ctx,files["discovery_git_shim"],"-C",linked.Path,"rev-parse","--show-toplevel") - configureOwnedProcess(cmd);cmd.Cancel=func()error{return killOwnedProcess(cmd)} - var output bytes.Buffer - cmd.Stdout,cmd.Stderr=&output,&output - cmd.WaitDelay=3*time.Second - require.NoError(t,cmd.Start()) - done:=make(chan error,1);go func(){done<-cmd.Wait()}() - joined:=false - t.Cleanup(func(){_=release();if !joined{_=killOwnedProcess(cmd);<-done}}) - var child int - require.Eventually(t,func()bool{raw,err:=os.ReadFile(files["discovery_git_events"]);if err!=nil{return false};fields:=strings.Fields(string(raw));if len(fields)!=3{return false};child,err=strconv.Atoi(fields[0]);return err==nil&&fields[1]==linked.Path&&fields[2]=="rev-parse"},3*time.Second,5*time.Millisecond) - require.NoError(t,cmd.Process.Kill(),"cancelled Git parent exits while descendant retains its stdout pipe") - require.NoError(t,syscall.Kill(child,syscall.SIGTERM),"TERM goes only to recorded owned descendant") - select{case err:=<-done:joined=true;t.Fatalf("stall unexpectedly exited after TERM: %v",err);case <-time.After(30*time.Millisecond):} - require.NoError(t,release()) - select{case err:=<-done:joined=true;require.Error(t,err);case <-ctx.Done():t.Fatal("released owned shim did not join")} - _,err=p.git(t.Context(),p.Repos[1].Path,"rev-parse","--show-toplevel");require.NoError(t,err) - _,err=os.Stat(filepath.Join(p.Root,"discovery-git-fault","release"));require.NoError(t,err) +func TestDiscoveryFaultRetainsOwnedPipeAndDoesNotAffectHarnessGit(t *testing.T) { + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + originalEnv := append([]string(nil), p.Env...) + release, files, err := p.installDiscoveryFault() + require.NoError(t, err) + t.Cleanup(func() { _ = release() }) + require.Equal(t, originalEnv, p.Env, "harness environment remains unchanged") + require.NotEqual(t, p.Env, p.daemonEnv) + require.NoError(t, p.addDiscoveryWorktree(t.Context(), 0), "harness worktree creation must never enter its daemon shim") + linked := p.Repos[len(p.Repos)-1] + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, files["discovery_git_shim"], "-C", linked.Path, "rev-parse", "--show-toplevel") + configureOwnedProcess(cmd) + cmd.Cancel = func() error { return killOwnedProcess(cmd) } + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + cmd.WaitDelay = 3 * time.Second + require.NoError(t, cmd.Start()) + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + joined := false + t.Cleanup(func() { + _ = release() + if !joined { + _ = killOwnedProcess(cmd) + <-done + } + }) + var child int + require.Eventually(t, func() bool { + raw, err := os.ReadFile(files["discovery_git_events"]) + if err != nil { + return false + } + fields := strings.Fields(string(raw)) + if len(fields) != 3 { + return false + } + child, err = strconv.Atoi(fields[0]) + return err == nil && fields[1] == linked.Path && fields[2] == "rev-parse" + }, 3*time.Second, 5*time.Millisecond) + require.NoError(t, cmd.Process.Kill(), "cancelled Git parent exits while descendant retains its stdout pipe") + require.NoError(t, syscall.Kill(child, syscall.SIGTERM), "TERM goes only to recorded owned descendant") + select { + case err := <-done: + joined = true + t.Fatalf("stall unexpectedly exited after TERM: %v", err) + case <-time.After(30 * time.Millisecond): + } + require.NoError(t, release()) + select { + case err := <-done: + joined = true + require.Error(t, err) + case <-ctx.Done(): + t.Fatal("released owned shim did not join") + } + _, err = p.git(t.Context(), p.Repos[1].Path, "rev-parse", "--show-toplevel") + require.NoError(t, err) + _, err = os.Stat(filepath.Join(p.Root, "discovery-git-fault", "release")) + require.NoError(t, err) } -func TestDiscoveryStallOptionsRequireActualAdditions(t *testing.T){ - base:=[]string{"--binary","unused","--manifest","unused","--scratch",t.TempDir(),"--agents","2","--edit-agents","0","--discovery-stall-first"} - _,err:=parseRun(append(base,"--scenario",checkoutDiscoveryScenario,"--worktrees","2"));require.NoError(t,err) - _,err=parseRun(append(base,"--scenario",checkoutDiscoveryScenario,"--worktrees","0"));require.Error(t,err) - _,err=parseRun(append(base,"--scenario","read"));require.Error(t,err) +func TestDiscoveryStallOptionsRequireActualAdditions(t *testing.T) { + base := []string{"--binary", "unused", "--manifest", "unused", "--scratch", t.TempDir(), "--agents", "2", "--edit-agents", "0", "--discovery-stall-first"} + _, err := parseRun(append(base, "--scenario", checkoutDiscoveryScenario, "--worktrees", "2")) + require.NoError(t, err) + _, err = parseRun(append(base, "--scenario", checkoutDiscoveryScenario, "--worktrees", "0")) + require.Error(t, err) + _, err = parseRun(append(base, "--scenario", "read")) + require.Error(t, err) } diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go index 0391b8264..2ed5b1cd0 100644 --- a/bench/load/discovery_scope.go +++ b/bench/load/discovery_scope.go @@ -15,9 +15,9 @@ const checkoutDiscoveryScenario = "worktree-discovery" type checkoutDiscoveryObservation struct { Root string `json:"root"` - FamilyRole string `json:"family_role"` - Prefix string `json:"repo_prefix"` - Query string `json:"query"` + FamilyRole string `json:"family_role"` + Prefix string `json:"repo_prefix"` + Query string `json:"query"` Added time.Time `json:"added"` FirstRequest time.Time `json:"first_request"` ExactAt time.Time `json:"exact_at"` @@ -37,12 +37,12 @@ type checkoutCWDCounts struct { } type checkoutDiscoveryReport struct { - ForeignCapacityRefusals int `json:"foreign_capacity_refusals"` - PendingRefusals int `json:"pending_refusals"` - Checkouts []checkoutDiscoveryObservation `json:"checkouts"` - ByCWD map[string]checkoutCWDCounts `json:"by_cwd"` - OutsidePendingRefusals int `json:"outside_pending_refusals"` - OutsidePendingErrors int `json:"outside_pending_errors"` + ForeignCapacityRefusals int `json:"foreign_capacity_refusals"` + PendingRefusals int `json:"pending_refusals"` + Checkouts []checkoutDiscoveryObservation `json:"checkouts"` + ByCWD map[string]checkoutCWDCounts `json:"by_cwd"` + OutsidePendingRefusals int `json:"outside_pending_refusals"` + OutsidePendingErrors int `json:"outside_pending_errors"` } // A refusal belongs to a pending checkout only while its discovery is @@ -73,7 +73,12 @@ func summarizeCheckoutDiscovery(attempts []Attempt, checkouts []checkoutDiscover for _, checkout := range checkouts { if inside(checkout.Root, a.CWD) { row.Class = "new_worktree" - if checkout.FamilyRole == "foreign" { row.Class = "foreign_new_worktree"; if a.Code != 0 && (a.ErrorData["reason"] == "family_capacity" || a.ErrorData["reason"] == "stuck_capacity") {out.ForeignCapacityRefusals++} } + if checkout.FamilyRole == "foreign" { + row.Class = "foreign_new_worktree" + if a.Code != 0 && (a.ErrorData["reason"] == "family_capacity" || a.ErrorData["reason"] == "stuck_capacity") { + out.ForeignCapacityRefusals++ + } + } } } if a.Role == "workload" && !strings.HasPrefix(a.Tool, "edit.") { @@ -141,8 +146,10 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu } report := summarizeCheckoutDiscovery(phase.Attempts[first:], checkouts, frozen.Repos[0].Path, frozen.Repos[1].Path) phase.Gate["checkout_discovery"], _ = json.Marshal(report) - if o.Discovery.StallFirst && report.PendingRefusals==0 {resultErr=errors.Join(resultErr,errors.New("Git stall produced no pending refusal exposure"))} - if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 || report.ForeignCapacityRefusals > 0 { + if o.Discovery.StallFirst && report.PendingRefusals == 0 { + resultErr = errors.Join(resultErr, errors.New("Git stall produced no pending refusal exposure")) + } + if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 || report.ForeignCapacityRefusals > 0 { resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors, %d foreign capacity refusals", report.OutsidePendingRefusals, report.OutsidePendingErrors, report.ForeignCapacityRefusals)) } if parent.Err() != nil { @@ -162,14 +169,16 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu case <-ticker.C: } if len(checkouts) < o.Worktrees && !time.Now().Before(nextAdd) { - family := len(checkouts)%2 - if err := p.addDiscoveryWorktree(ctx, family); err != nil { + family := len(checkouts) % 2 + if err := p.addDiscoveryWorktree(ctx, family); err != nil { return err } repo := p.Repos[len(p.Repos)-1] - role:="primary" - if family==1 {role="foreign"} - entry := checkoutDiscoveryObservation{Root: repo.Path, FamilyRole:role,Prefix:repo.Prefix,Query:repo.Corpus.Query, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} + role := "primary" + if family == 1 { + role = "foreign" + } + entry := checkoutDiscoveryObservation{Root: repo.Path, FamilyRole: role, Prefix: repo.Prefix, Query: repo.Corpus.Query, Added: time.Now(), ExactViewLag: unavailable("ms", "no exact worktree view observed before phase end")} checkouts = append(checkouts, entry) nextAdd = nextAdd.Add(interval) subdir := filepath.Join(repo.Path, ".discovery-probe") diff --git a/bench/load/discovery_scope_test.go b/bench/load/discovery_scope_test.go index ff9939b10..2aee33958 100644 --- a/bench/load/discovery_scope_test.go +++ b/bench/load/discovery_scope_test.go @@ -118,32 +118,39 @@ func TestDiscoveryAddsWorktreesWhileOriginalSessionsKeepReading(t *testing.T) { } func TestDiscoveryForeignCapacityIsNotExcusedAsPending(t *testing.T) { - start:=time.Unix(100,0);root:=filepath.Join(t.TempDir(),"foreign-new") - entries:=[]checkoutDiscoveryObservation{{Root:root,FamilyRole:"foreign",Added:start}} - rows:=[]Attempt{ - {CWD:root,Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"in_progress"}}, - {CWD:root,Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"family_capacity"}}, - {CWD:filepath.Join(root,"subdir"),Started:start.Add(time.Millisecond),Code:-32000,Outcome:"error",ErrorData:map[string]any{"reason":"stuck_capacity"}}, - } - report:=summarizeCheckoutDiscovery(rows,entries,root+"-primary",root+"-other") - require.Equal(t,3,report.PendingRefusals) - require.Equal(t,2,report.ForeignCapacityRefusals,"foreign family capacity cannot be hidden by the new-root pending window") - require.Equal(t,"foreign_new_worktree",report.ByCWD[root].Class) + start := time.Unix(100, 0) + root := filepath.Join(t.TempDir(), "foreign-new") + entries := []checkoutDiscoveryObservation{{Root: root, FamilyRole: "foreign", Added: start}} + rows := []Attempt{ + {CWD: root, Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "in_progress"}}, + {CWD: root, Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "family_capacity"}}, + {CWD: filepath.Join(root, "subdir"), Started: start.Add(time.Millisecond), Code: -32000, Outcome: "error", ErrorData: map[string]any{"reason": "stuck_capacity"}}, + } + report := summarizeCheckoutDiscovery(rows, entries, root+"-primary", root+"-other") + require.Equal(t, 3, report.PendingRefusals) + require.Equal(t, 2, report.ForeignCapacityRefusals, "foreign family capacity cannot be hidden by the new-root pending window") + require.Equal(t, "foreign_new_worktree", report.ByCWD[root].Class) } func TestDiscoveryNewRootsRetainEachUnrelatedFamily(t *testing.T) { - first,second:=gitFixture(t),gitFixture(t);second.Label="other";second.Query="ForeignOnly" - p,err:=newPrivateRun(t.Context(),t.TempDir(),"unused",[]Corpus{first,second});require.NoError(t,err) - t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) - p.Repos[0].Prefix,p.Repos[1].Prefix="primary","foreign" - for _,family:=range []int{0,1,0,1} { - require.NoError(t,p.addDiscoveryWorktree(t.Context(),family)) - linked:=p.Repos[len(p.Repos)-1];primary:=p.Repos[family] - expected,err:=p.git(t.Context(),primary.Path,"rev-parse","--path-format=absolute","--git-common-dir");require.NoError(t,err) - actual,err:=p.git(t.Context(),linked.Path,"rev-parse","--path-format=absolute","--git-common-dir");require.NoError(t,err) - require.Equal(t,string(expected),string(actual),"linked checkout must share only its selected family") - require.Equal(t,primary.Prefix,linked.Prefix) - require.Equal(t,primary.Corpus.Query,linked.Corpus.Query) - } - require.Error(t,p.addDiscoveryWorktree(t.Context(),2)) + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + second.Query = "ForeignOnly" + p, err := newPrivateRun(t.Context(), t.TempDir(), "unused", []Corpus{first, second}) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + p.Repos[0].Prefix, p.Repos[1].Prefix = "primary", "foreign" + for _, family := range []int{0, 1, 0, 1} { + require.NoError(t, p.addDiscoveryWorktree(t.Context(), family)) + linked := p.Repos[len(p.Repos)-1] + primary := p.Repos[family] + expected, err := p.git(t.Context(), primary.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + require.NoError(t, err) + actual, err := p.git(t.Context(), linked.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + require.NoError(t, err) + require.Equal(t, string(expected), string(actual), "linked checkout must share only its selected family") + require.Equal(t, primary.Prefix, linked.Prefix) + require.Equal(t, primary.Corpus.Query, linked.Corpus.Query) + } + require.Error(t, p.addDiscoveryWorktree(t.Context(), 2)) } diff --git a/bench/load/discovery_seed.go b/bench/load/discovery_seed.go index 5826dea56..359e78fb3 100644 --- a/bench/load/discovery_seed.go +++ b/bench/load/discovery_seed.go @@ -1,163 +1,369 @@ package main import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "reflect" - "strings" - "syscall" - "time" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "time" ) type discoverySeedRepo struct { - Repo privateRepo `json:"repo"` - Role string `json:"role"` - FileSHA256 string `json:"file_sha256"` - CommonDir string `json:"git_common_dir"` + Repo privateRepo `json:"repo"` + Role string `json:"role"` + FileSHA256 string `json:"file_sha256"` + CommonDir string `json:"git_common_dir"` } type discoverySeedDescriptor struct { - State string `json:"state"` - Root string `json:"owned_root"` - Nonce string `json:"owner_nonce"` - ConfigSHA256 string `json:"config_sha256"` - BootConfigSHA256 string `json:"boot_config_sha256"` - GitConfigSHA256 string `json:"git_config_sha256"` - Repos []discoverySeedRepo `json:"repositories"` - PrimaryCount int `json:"primary_checkout_count"` - LinkedCount int `json:"linked_checkout_count"` - NestedCount int `json:"nested_linked_checkout_count"` - TotalCount int `json:"total_checkout_count"` + State string `json:"state"` + Root string `json:"owned_root"` + Nonce string `json:"owner_nonce"` + ConfigSHA256 string `json:"config_sha256"` + BootConfigSHA256 string `json:"boot_config_sha256"` + GitConfigSHA256 string `json:"git_config_sha256"` + Repos []discoverySeedRepo `json:"repositories"` + PrimaryCount int `json:"primary_checkout_count"` + LinkedCount int `json:"linked_checkout_count"` + NestedCount int `json:"nested_linked_checkout_count"` + TotalCount int `json:"total_checkout_count"` } type discoverySeedLineage struct { - State string `json:"state"` - Descriptor string `json:"descriptor"` - DescriptorSHA256 string `json:"descriptor_sha256"` - OwnedRoot string `json:"owned_root"` - SourceGeneration string `json:"source_generation"` - OldVersion string `json:"old_version"` - OldBinarySHA256 string `json:"old_binary_sha256"` - UpgradeSourceCommit string `json:"upgrade_source_commit"` - UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` - Snapshot string `json:"snapshot"` - MainSHA256 string `json:"main_sha256"` - WALSHA256 string `json:"wal_sha256"` - SchemaVersion int `json:"schema_version"` - NormalStop bool `json:"normal_stop"` - OwnersExited bool `json:"owners_exited"` - OwnerPID int `json:"owner_pid"` - StopReceipt string `json:"stop_receipt"` - StopReceiptSHA256 string `json:"stop_receipt_sha256"` - IdentityReceipt string `json:"identity_receipt"` - IdentityReceiptSHA256 string `json:"identity_receipt_sha256"` - Repos []discoverySeedRepo `json:"repositories"` + State string `json:"state"` + Descriptor string `json:"descriptor"` + DescriptorSHA256 string `json:"descriptor_sha256"` + OwnedRoot string `json:"owned_root"` + SourceGeneration string `json:"source_generation"` + OldVersion string `json:"old_version"` + OldBinarySHA256 string `json:"old_binary_sha256"` + UpgradeSourceCommit string `json:"upgrade_source_commit"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Snapshot string `json:"snapshot"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + OwnerPID int `json:"owner_pid"` + StopReceipt string `json:"stop_receipt"` + StopReceiptSHA256 string `json:"stop_receipt_sha256"` + IdentityReceipt string `json:"identity_receipt"` + IdentityReceiptSHA256 string `json:"identity_receipt_sha256"` + Repos []discoverySeedRepo `json:"repositories"` } -func discoveryFileSHA(path string)(string,error){ - info,err:=os.Lstat(path);if err!=nil{return "",err};if !info.Mode().IsRegular(){return "",errors.New("discovery lineage requires regular files, not symlinks")} - f,err:=os.Open(path);if err!=nil{return "",err};h:=sha256.New();_,copyErr:=io.Copy(h,f);if err:=errors.Join(copyErr,f.Close());err!=nil{return "",err};return hex.EncodeToString(h.Sum(nil)),nil +func discoveryFileSHA(path string) (string, error) { + info, err := os.Lstat(path) + if err != nil { + return "", err + } + if !info.Mode().IsRegular() { + return "", errors.New("discovery lineage requires regular files, not symlinks") + } + f, err := os.Open(path) + if err != nil { + return "", err + } + h := sha256.New() + _, copyErr := io.Copy(h, f) + if err := errors.Join(copyErr, f.Close()); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil } -func discoveryReadJSON(path string,out any)error{ - f,err:=os.Open(path);if err!=nil{return err};defer f.Close();d:=json.NewDecoder(io.LimitReader(f,4<<20));d.DisallowUnknownFields();if err:=d.Decode(out);err!=nil{return err};if err:=d.Decode(new(any));err!=io.EOF{return errors.New("discovery descriptor must contain exactly one JSON object")};return nil +func discoveryReadJSON(path string, out any) error { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + d := json.NewDecoder(io.LimitReader(f, 4<<20)) + d.DisallowUnknownFields() + if err := d.Decode(out); err != nil { + return err + } + if err := d.Decode(new(any)); err != io.EOF { + return errors.New("discovery descriptor must contain exactly one JSON object") + } + return nil } -func discoveryWriteJSON(path string,value any)error{ - b,err:=json.MarshalIndent(value,""," ");if err!=nil{return err};f,err:=os.OpenFile(path,os.O_WRONLY|os.O_CREATE|os.O_EXCL,0o600);if err!=nil{return err};_,err=f.Write(append(b,'\n'));return errors.Join(err,f.Close()) +func discoveryWriteJSON(path string, value any) error { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + _, err = f.Write(append(b, '\n')) + return errors.Join(err, f.Close()) } // Preparation only makes owned clones/topology and metadata. The exclusive // producer must populate and upgrade this exact root before resume can pass. -func prepareDiscoveryState(ctx context.Context,o runOptions,corpora []Corpus)error{ - if len(corpora)<2{return errors.New("discovery preparation requires two unrelated pinned corpora")} - if err:=validateOutputRoots(corpora,o.Discovery.Prepare);err!=nil{return err} - if _,err:=os.Lstat(o.Discovery.Prepare);!errors.Is(err,os.ErrNotExist){return errors.New("discovery descriptor destination must be absent")} - p,err:=newPrivateRun(ctx,o.Scratch,o.Binary,corpora[:2]);if err!=nil{return err};defer os.RemoveAll(p.SocketDir) - complete:=false;defer func(){if !complete{_=p.removeState()}}() - for i:=range o.Discovery.ExistingWorktrees {if err:=p.addExistingDiscoveryWorktree(ctx,i,id.LinkedCount||d.TotalCount!=2+d.LinkedCount||len(d.Repos)!=d.TotalCount||!reflect.DeepEqual(l.Repos,d.Repos){return nil,errors.New("discovery lineage topology/identity differs from preparation")} - if len(corpora)!=2||!reflect.DeepEqual(d.Repos[0].Repo.Corpus,corpora[0])||!reflect.DeepEqual(d.Repos[1].Repo.Corpus,corpora[1]){return nil,errors.New("discovery seed corpus pins mismatch")} - for path,want:=range map[string]string{filepath.Join(root,"config.yaml"):d.ConfigSHA256,filepath.Join(root,"config","gortex","config.yaml"):d.BootConfigSHA256,filepath.Join(root,"gitconfig"):d.GitConfigSHA256,l.StopReceipt:l.StopReceiptSHA256,l.IdentityReceipt:l.IdentityReceiptSHA256}{if err:=discoveryOwnedFile(root,path);err!=nil{return nil,err};actual,err:=discoveryFileSHA(path);if err!=nil||!discoveryDigestValid(want,32)||actual!=want{return nil,errors.New("discovery configuration/producer proof digest mismatch")}} - for _,proofPath:=range []string{l.StopReceipt,l.IdentityReceipt}{ - var proof discoverySeedProof - if err:=discoveryReadJSON(proofPath,&proof);err!=nil{return nil,err} - if proof.State!="OBSERVED_OWNED_UPGRADE"||proof.ObservedAt.IsZero()||proof.OwnedRoot!=root||proof.OwnerPID!=l.OwnerPID||!proof.NormalStop||!proof.OwnersExited||proof.MainSHA256!=l.MainSHA256||proof.WALSHA256!=l.WALSHA256||proof.SchemaVersion!=44||proof.UpgradeBinarySHA256!=l.UpgradeBinarySHA256||!reflect.DeepEqual(proof.Repos,d.Repos){return nil,errors.New("discovery producer proof is unknown or does not bind this stopped snapshot and identity topology")} - } - process,err:=os.FindProcess(l.OwnerPID);if err!=nil{return nil,err};err=process.Signal(syscall.Signal(0));_=process.Release();if !errors.Is(err,syscall.ESRCH)&&!errors.Is(err,os.ErrProcessDone){return nil,errors.New("discovery upgrade owner is live or cannot be proven exited")} - binaryPath,err:=filepath.Abs(o.Binary);if err!=nil{return nil,err} - p:=&privateRun{Root:root,Store:filepath.Join(root,"store.db"),Config:filepath.Join(root,"config.yaml"),Binary:binaryPath} - for _,entry:=range d.Repos{ - repo:=entry.Repo;if err:=discoveryOwnedFile(root,repo.File);err!=nil{return nil,err};if !inside(root,repo.Path)||repo.File!=filepath.Join(repo.Path,repo.Corpus.File){return nil,errors.New("discovery repository path mismatch")} - actual,err:=discoveryFileSHA(repo.File);if err!=nil||actual!=entry.FileSHA256{return nil,errors.New("discovery prepared source content changed")};p.Repos=append(p.Repos,repo) - } - p.Env=privateEnv(root,"",filepath.Join(root,"gitconfig")) - for i,repo:=range p.Repos{head,err:=p.git(ctx,repo.Path,"rev-parse","HEAD");if err!=nil||strings.TrimSpace(string(head))!=repo.Corpus.Commit{return nil,errors.New("discovery prepared checkout HEAD changed")} - common,err:=p.git(ctx,repo.Path,"rev-parse","--path-format=absolute","--git-common-dir");if err!=nil||strings.TrimSpace(string(common))!=d.Repos[i].CommonDir {return nil,errors.New("discovery prepared Git family changed")}} - if err:=discoveryOwnedFile(root,l.Snapshot);err!=nil{return nil,err} - for _,suffix:=range []string{"","-wal","-shm"}{if _,err:=os.Lstat(p.Store+suffix);!errors.Is(err,os.ErrNotExist){return nil,errors.New("discovery destination store must be absent before one-shot resume")}} - before,err:=discoverySeedPair(l.Snapshot);if err!=nil||before[0]!=l.MainSHA256||before[1]!=l.WALSHA256{return nil,errors.New("discovery quiescent snapshot digest mismatch")} - f,err:=os.Open(l.Snapshot);if err!=nil{return nil,err};header:=make([]byte,100);_,readErr:=io.ReadFull(f,header);err=errors.Join(readErr,f.Close());if err!=nil||string(header[:16])!="SQLite format 3\x00"||binary.BigEndian.Uint32(header[60:64])!=44{return nil,errors.New("discovery snapshot is not a checkpointed schema44 SQLite main")} - if err:=discoveryWriteJSON(filepath.Join(root,"discovery-consumed.json"),map[string]string{"descriptor_sha256":sha,"upgrade_binary_sha256":l.UpgradeBinarySHA256});err!=nil{return nil,err} - if err:=copySeedStore(l.Snapshot,p.Store);err!=nil{return nil,err} - after,err:=discoverySeedPair(l.Snapshot);if err!=nil||before!=after{return nil,errors.New("discovery snapshot changed during copy; partial destination retained")};copied,err:=discoverySeedPair(p.Store);if err!=nil||copied!=before{return nil,errors.New("discovery seed copy differs; partial destination retained")} - p.SocketDir,err=os.MkdirTemp("/tmp","gl-");if err!=nil{return nil,err};p.Socket=filepath.Join(p.SocketDir,"d.sock");p.Env=privateEnv(root,p.Socket,filepath.Join(root,"gitconfig"));return p,nil +func resumeDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) (*privateRun, error) { + var l discoverySeedLineage + if err := discoveryReadJSON(o.Discovery.Resume, &l); err != nil { + return nil, err + } + if l.State != "VALIDATED_OWNED_UPGRADE" || l.SourceGeneration == "" || l.OldVersion == "" || l.SchemaVersion != 44 || !l.NormalStop || !l.OwnersExited || l.OwnerPID < 1 || !discoveryDigestValid(l.OldBinarySHA256, 32) || !discoveryDigestValid(l.UpgradeBinarySHA256, 32) || !discoveryDigestValid(l.UpgradeSourceCommit, 20) { + return nil, errors.New("discovery seed lacks complete validated upgrade lineage") + } + sha, err := discoveryFileSHA(l.Descriptor) + if err != nil || sha != l.DescriptorSHA256 { + return nil, errors.New("discovery descriptor digest mismatch") + } + var d discoverySeedDescriptor + if err := discoveryReadJSON(l.Descriptor, &d); err != nil { + return nil, err + } + scratch, err := canonicalPath(o.Scratch) + if err != nil { + return nil, err + } + root, err := canonicalPath(d.Root) + if err != nil || root != d.Root || root != l.OwnedRoot || !inside(scratch, root) || root == scratch || !strings.HasPrefix(filepath.Base(root), "load-") { + return nil, errors.New("discovery seed is outside its exact owned scratch root") + } + var owner discoverySeedDescriptor + if err := discoveryReadJSON(filepath.Join(root, "discovery-owner.json"), &owner); err != nil || !reflect.DeepEqual(owner, d) { + return nil, errors.New("discovery root ownership does not match descriptor") + } + if d.State != "PREPARED_ROOTS_NO_STORE" || !discoveryDigestValid(d.Nonce, 32) || d.PrimaryCount != 2 || d.LinkedCount < 0 || d.NestedCount < 0 || d.NestedCount > d.LinkedCount || d.TotalCount != 2+d.LinkedCount || len(d.Repos) != d.TotalCount || !reflect.DeepEqual(l.Repos, d.Repos) { + return nil, errors.New("discovery lineage topology/identity differs from preparation") + } + if len(corpora) != 2 || !reflect.DeepEqual(d.Repos[0].Repo.Corpus, corpora[0]) || !reflect.DeepEqual(d.Repos[1].Repo.Corpus, corpora[1]) { + return nil, errors.New("discovery seed corpus pins mismatch") + } + for path, want := range map[string]string{filepath.Join(root, "config.yaml"): d.ConfigSHA256, filepath.Join(root, "config", "gortex", "config.yaml"): d.BootConfigSHA256, filepath.Join(root, "gitconfig"): d.GitConfigSHA256, l.StopReceipt: l.StopReceiptSHA256, l.IdentityReceipt: l.IdentityReceiptSHA256} { + if err := discoveryOwnedFile(root, path); err != nil { + return nil, err + } + actual, err := discoveryFileSHA(path) + if err != nil || !discoveryDigestValid(want, 32) || actual != want { + return nil, errors.New("discovery configuration/producer proof digest mismatch") + } + } + for _, proofPath := range []string{l.StopReceipt, l.IdentityReceipt} { + var proof discoverySeedProof + if err := discoveryReadJSON(proofPath, &proof); err != nil { + return nil, err + } + if proof.State != "OBSERVED_OWNED_UPGRADE" || proof.ObservedAt.IsZero() || proof.OwnedRoot != root || proof.OwnerPID != l.OwnerPID || !proof.NormalStop || !proof.OwnersExited || proof.MainSHA256 != l.MainSHA256 || proof.WALSHA256 != l.WALSHA256 || proof.SchemaVersion != 44 || proof.UpgradeBinarySHA256 != l.UpgradeBinarySHA256 || !reflect.DeepEqual(proof.Repos, d.Repos) { + return nil, errors.New("discovery producer proof is unknown or does not bind this stopped snapshot and identity topology") + } + } + process, err := os.FindProcess(l.OwnerPID) + if err != nil { + return nil, err + } + err = process.Signal(syscall.Signal(0)) + _ = process.Release() + if !errors.Is(err, syscall.ESRCH) && !errors.Is(err, os.ErrProcessDone) { + return nil, errors.New("discovery upgrade owner is live or cannot be proven exited") + } + binaryPath, err := filepath.Abs(o.Binary) + if err != nil { + return nil, err + } + p := &privateRun{Root: root, Store: filepath.Join(root, "store.db"), Config: filepath.Join(root, "config.yaml"), Binary: binaryPath} + for _, entry := range d.Repos { + repo := entry.Repo + if err := discoveryOwnedFile(root, repo.File); err != nil { + return nil, err + } + if !inside(root, repo.Path) || repo.File != filepath.Join(repo.Path, repo.Corpus.File) { + return nil, errors.New("discovery repository path mismatch") + } + actual, err := discoveryFileSHA(repo.File) + if err != nil || actual != entry.FileSHA256 { + return nil, errors.New("discovery prepared source content changed") + } + p.Repos = append(p.Repos, repo) + } + p.Env = privateEnv(root, "", filepath.Join(root, "gitconfig")) + for i, repo := range p.Repos { + head, err := p.git(ctx, repo.Path, "rev-parse", "HEAD") + if err != nil || strings.TrimSpace(string(head)) != repo.Corpus.Commit { + return nil, errors.New("discovery prepared checkout HEAD changed") + } + common, err := p.git(ctx, repo.Path, "rev-parse", "--path-format=absolute", "--git-common-dir") + if err != nil || strings.TrimSpace(string(common)) != d.Repos[i].CommonDir { + return nil, errors.New("discovery prepared Git family changed") + } + } + if err := discoveryOwnedFile(root, l.Snapshot); err != nil { + return nil, err + } + for _, suffix := range []string{"", "-wal", "-shm"} { + if _, err := os.Lstat(p.Store + suffix); !errors.Is(err, os.ErrNotExist) { + return nil, errors.New("discovery destination store must be absent before one-shot resume") + } + } + before, err := discoverySeedPair(l.Snapshot) + if err != nil || before[0] != l.MainSHA256 || before[1] != l.WALSHA256 { + return nil, errors.New("discovery quiescent snapshot digest mismatch") + } + f, err := os.Open(l.Snapshot) + if err != nil { + return nil, err + } + header := make([]byte, 100) + _, readErr := io.ReadFull(f, header) + err = errors.Join(readErr, f.Close()) + if err != nil || string(header[:16]) != "SQLite format 3\x00" || binary.BigEndian.Uint32(header[60:64]) != 44 { + return nil, errors.New("discovery snapshot is not a checkpointed schema44 SQLite main") + } + if err := discoveryWriteJSON(filepath.Join(root, "discovery-consumed.json"), map[string]string{"descriptor_sha256": sha, "upgrade_binary_sha256": l.UpgradeBinarySHA256}); err != nil { + return nil, err + } + if err := copySeedStore(l.Snapshot, p.Store); err != nil { + return nil, err + } + after, err := discoverySeedPair(l.Snapshot) + if err != nil || before != after { + return nil, errors.New("discovery snapshot changed during copy; partial destination retained") + } + copied, err := discoverySeedPair(p.Store) + if err != nil || copied != before { + return nil, errors.New("discovery seed copy differs; partial destination retained") + } + p.SocketDir, err = os.MkdirTemp("/tmp", "gl-") + if err != nil { + return nil, err + } + p.Socket = filepath.Join(p.SocketDir, "d.sock") + p.Env = privateEnv(root, p.Socket, filepath.Join(root, "gitconfig")) + return p, nil } -func discoverySeedPair(path string)([2]string,error){ - var out [2]string;var err error - if _,err:=os.Lstat(path+"-shm");!errors.Is(err,os.ErrNotExist){return out,errors.New("discovery snapshot has shared-memory state or unknown ownership")} - out[0],err=discoveryFileSHA(path);if err!=nil{return out,err};out[1],err=discoveryFileSHA(path+"-wal");if errors.Is(err,os.ErrNotExist){err=nil};return out,err +func discoverySeedPair(path string) ([2]string, error) { + var out [2]string + var err error + if _, err := os.Lstat(path + "-shm"); !errors.Is(err, os.ErrNotExist) { + return out, errors.New("discovery snapshot has shared-memory state or unknown ownership") + } + out[0], err = discoveryFileSHA(path) + if err != nil { + return out, err + } + out[1], err = discoveryFileSHA(path + "-wal") + if errors.Is(err, os.ErrNotExist) { + err = nil + } + return out, err } // Both retained producer receipts must bind observations to this exact stopped // output; state assertions without snapshot and identity evidence are refused. type discoverySeedProof struct { - State string `json:"state"` - ObservedAt time.Time `json:"observed_at"` - OwnedRoot string `json:"owned_root"` - OwnerPID int `json:"owner_pid"` - NormalStop bool `json:"normal_stop"` - OwnersExited bool `json:"owners_exited"` - MainSHA256 string `json:"main_sha256"` - WALSHA256 string `json:"wal_sha256"` - SchemaVersion int `json:"schema_version"` - UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` - Repos []discoverySeedRepo `json:"repositories"` + State string `json:"state"` + ObservedAt time.Time `json:"observed_at"` + OwnedRoot string `json:"owned_root"` + OwnerPID int `json:"owner_pid"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Repos []discoverySeedRepo `json:"repositories"` } diff --git a/bench/load/discovery_seed_test.go b/bench/load/discovery_seed_test.go index 160effdf2..81a9ae6a6 100644 --- a/bench/load/discovery_seed_test.go +++ b/bench/load/discovery_seed_test.go @@ -3,62 +3,107 @@ package main import ( - "encoding/binary" - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "time" + "encoding/binary" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" - "github.com/stretchr/testify/require" + "github.com/stretchr/testify/require" ) // This is only a byte-copy/parser fixture, deliberately not a real SQLite // database or upgrade receipt. It never establishes upgrade acceptance. -func syntheticDiscoverySeed(t *testing.T)(runOptions,[]Corpus,discoverySeedLineage){ - t.Helper();first,second:=gitFixture(t),gitFixture(t);second.Label="other" - scratch:=t.TempDir();descriptor:=filepath.Join(scratch,"prepared.json") - o:=runOptions{Binary:"unused",Scratch:scratch,Discovery:discoveryOptions{Prepare:descriptor,ExistingWorktrees:2,ExistingNested:1}} - corpora:=[]Corpus{first,second};require.NoError(t,prepareDiscoveryState(t.Context(),o,corpora)) - var d discoverySeedDescriptor;require.NoError(t,discoveryReadJSON(descriptor,&d)) - snapshot:=filepath.Join(d.Root,"synthetic-byte-snapshot.db") - header:=make([]byte,100);copy(header,"SQLite format 3\x00");binary.BigEndian.PutUint32(header[60:64],44) - require.NoError(t,os.WriteFile(snapshot,header,0o600));mainSHA,err:=discoveryFileSHA(snapshot);require.NoError(t,err) - child:=exec.Command("sh","-c","exit 0");require.NoError(t,child.Run()) - l:=discoverySeedLineage{State:"VALIDATED_OWNED_UPGRADE",Descriptor:descriptor,OwnedRoot:d.Root,SourceGeneration:"synthetic-test-only",OldVersion:"synthetic",OldBinarySHA256:strings.Repeat("1",64),UpgradeSourceCommit:strings.Repeat("2",40),UpgradeBinarySHA256:strings.Repeat("3",64),Snapshot:snapshot,MainSHA256:mainSHA,SchemaVersion:44,NormalStop:true,OwnersExited:true,OwnerPID:child.Process.Pid,StopReceipt:filepath.Join(d.Root,"synthetic-stop.json"),IdentityReceipt:filepath.Join(d.Root,"synthetic-identity.json"),Repos:d.Repos} - l.DescriptorSHA256,err=discoveryFileSHA(descriptor);require.NoError(t,err) - rewriteSyntheticDiscoveryProofs(t,&l) - o.Discovery=discoveryOptions{Resume:filepath.Join(scratch,"lineage.json")} - return o,corpora,l +func syntheticDiscoverySeed(t *testing.T) (runOptions, []Corpus, discoverySeedLineage) { + t.Helper() + first, second := gitFixture(t), gitFixture(t) + second.Label = "other" + scratch := t.TempDir() + descriptor := filepath.Join(scratch, "prepared.json") + o := runOptions{Binary: "unused", Scratch: scratch, Discovery: discoveryOptions{Prepare: descriptor, ExistingWorktrees: 2, ExistingNested: 1}} + corpora := []Corpus{first, second} + require.NoError(t, prepareDiscoveryState(t.Context(), o, corpora)) + var d discoverySeedDescriptor + require.NoError(t, discoveryReadJSON(descriptor, &d)) + snapshot := filepath.Join(d.Root, "synthetic-byte-snapshot.db") + header := make([]byte, 100) + copy(header, "SQLite format 3\x00") + binary.BigEndian.PutUint32(header[60:64], 44) + require.NoError(t, os.WriteFile(snapshot, header, 0o600)) + mainSHA, err := discoveryFileSHA(snapshot) + require.NoError(t, err) + child := exec.Command("sh", "-c", "exit 0") + require.NoError(t, child.Run()) + l := discoverySeedLineage{State: "VALIDATED_OWNED_UPGRADE", Descriptor: descriptor, OwnedRoot: d.Root, SourceGeneration: "synthetic-test-only", OldVersion: "synthetic", OldBinarySHA256: strings.Repeat("1", 64), UpgradeSourceCommit: strings.Repeat("2", 40), UpgradeBinarySHA256: strings.Repeat("3", 64), Snapshot: snapshot, MainSHA256: mainSHA, SchemaVersion: 44, NormalStop: true, OwnersExited: true, OwnerPID: child.Process.Pid, StopReceipt: filepath.Join(d.Root, "synthetic-stop.json"), IdentityReceipt: filepath.Join(d.Root, "synthetic-identity.json"), Repos: d.Repos} + l.DescriptorSHA256, err = discoveryFileSHA(descriptor) + require.NoError(t, err) + rewriteSyntheticDiscoveryProofs(t, &l) + o.Discovery = discoveryOptions{Resume: filepath.Join(scratch, "lineage.json")} + return o, corpora, l } -func rewriteSyntheticDiscoveryProofs(t *testing.T,l *discoverySeedLineage){ - t.Helper();proof:=discoverySeedProof{State:"OBSERVED_OWNED_UPGRADE",ObservedAt:time.Now(),OwnedRoot:l.OwnedRoot,OwnerPID:l.OwnerPID,NormalStop:l.NormalStop,OwnersExited:l.OwnersExited,MainSHA256:l.MainSHA256,WALSHA256:l.WALSHA256,SchemaVersion:l.SchemaVersion,UpgradeBinarySHA256:l.UpgradeBinarySHA256,Repos:l.Repos} - for _,path:=range []string{l.StopReceipt,l.IdentityReceipt}{_=os.Remove(path);require.NoError(t,discoveryWriteJSON(path,proof))} - var err error;l.StopReceiptSHA256,err=discoveryFileSHA(l.StopReceipt);require.NoError(t,err);l.IdentityReceiptSHA256,err=discoveryFileSHA(l.IdentityReceipt);require.NoError(t,err) +func rewriteSyntheticDiscoveryProofs(t *testing.T, l *discoverySeedLineage) { + t.Helper() + proof := discoverySeedProof{State: "OBSERVED_OWNED_UPGRADE", ObservedAt: time.Now(), OwnedRoot: l.OwnedRoot, OwnerPID: l.OwnerPID, NormalStop: l.NormalStop, OwnersExited: l.OwnersExited, MainSHA256: l.MainSHA256, WALSHA256: l.WALSHA256, SchemaVersion: l.SchemaVersion, UpgradeBinarySHA256: l.UpgradeBinarySHA256, Repos: l.Repos} + for _, path := range []string{l.StopReceipt, l.IdentityReceipt} { + _ = os.Remove(path) + require.NoError(t, discoveryWriteJSON(path, proof)) + } + var err error + l.StopReceiptSHA256, err = discoveryFileSHA(l.StopReceipt) + require.NoError(t, err) + l.IdentityReceiptSHA256, err = discoveryFileSHA(l.IdentityReceipt) + require.NoError(t, err) } -func TestDiscoverySeedResumeRefusesUnknownLiveAndMismatchedLineage(t *testing.T){ - for _,name:=range []string{"unknown","topology","snapshot","live","shared_memory","changed_git_family"}{t.Run(name,func(t *testing.T){ - o,corpora,l:=syntheticDiscoverySeed(t) - switch name{ - case "unknown":l.State="TEMPLATE" - case "topology":l.Repos=append([]discoverySeedRepo(nil),l.Repos...);l.Repos[2].Repo.Prefix="foreign" - case "snapshot":l.MainSHA256=strings.Repeat("0",64);rewriteSyntheticDiscoveryProofs(t,&l) - case "live":l.OwnerPID=os.Getpid();rewriteSyntheticDiscoveryProofs(t,&l) - case "shared_memory":require.NoError(t,os.WriteFile(l.Snapshot+"-shm",nil,0o600)) - case "changed_git_family":require.NoError(t,os.WriteFile(filepath.Join(l.Repos[2].Repo.Path,".git"),[]byte("gitdir: "+filepath.Join(l.Repos[1].Repo.Path,".git")+"\n"),0o600)) - } - require.NoError(t,discoveryWriteJSON(o.Discovery.Resume,l)) - _,err:=resumeDiscoveryState(t.Context(),o,corpora);require.Error(t,err) - require.NoFileExists(t,filepath.Join(l.OwnedRoot,"store.db"),"refused lineage must not create a destination store") - })} +func TestDiscoverySeedResumeRefusesUnknownLiveAndMismatchedLineage(t *testing.T) { + for _, name := range []string{"unknown", "topology", "snapshot", "live", "shared_memory", "changed_git_family"} { + t.Run(name, func(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + switch name { + case "unknown": + l.State = "TEMPLATE" + case "topology": + l.Repos = append([]discoverySeedRepo(nil), l.Repos...) + l.Repos[2].Repo.Prefix = "foreign" + case "snapshot": + l.MainSHA256 = strings.Repeat("0", 64) + rewriteSyntheticDiscoveryProofs(t, &l) + case "live": + l.OwnerPID = os.Getpid() + rewriteSyntheticDiscoveryProofs(t, &l) + case "shared_memory": + require.NoError(t, os.WriteFile(l.Snapshot+"-shm", nil, 0o600)) + case "changed_git_family": + require.NoError(t, os.WriteFile(filepath.Join(l.Repos[2].Repo.Path, ".git"), []byte("gitdir: "+filepath.Join(l.Repos[1].Repo.Path, ".git")+"\n"), 0o600)) + } + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + _, err := resumeDiscoveryState(t.Context(), o, corpora) + require.Error(t, err) + require.NoFileExists(t, filepath.Join(l.OwnedRoot, "store.db"), "refused lineage must not create a destination store") + }) + } } -func TestDiscoverySeedCopiesSnapshotWithoutChangingExistingIdentities(t *testing.T){ - o,corpora,l:=syntheticDiscoverySeed(t);require.NoError(t,discoveryWriteJSON(o.Discovery.Resume,l)) - p,err:=resumeDiscoveryState(t.Context(),o,corpora);require.NoError(t,err);t.Cleanup(func(){_=os.RemoveAll(p.SocketDir)}) - require.Equal(t,l.OwnedRoot,p.Root);require.Len(t,p.Repos,4) - require.True(t,inside(p.Repos[0].Path,p.Repos[2].Path));require.False(t,inside(p.Repos[0].Path,p.Repos[3].Path)) - for i,repo:=range p.Repos{require.Equal(t,l.Repos[i].Repo.Path,repo.Path);require.Equal(t,l.Repos[i].Repo.Prefix,repo.Prefix)} - source,err:=discoveryFileSHA(l.Snapshot);require.NoError(t,err);dest,err:=discoveryFileSHA(p.Store);require.NoError(t,err);require.Equal(t,l.MainSHA256,source);require.Equal(t,source,dest) - _,err=resumeDiscoveryState(t.Context(),o,corpora);require.Error(t,err,"a prepared run is consumed once") +func TestDiscoverySeedCopiesSnapshotWithoutChangingExistingIdentities(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + p, err := resumeDiscoveryState(t.Context(), o, corpora) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + require.Equal(t, l.OwnedRoot, p.Root) + require.Len(t, p.Repos, 4) + require.True(t, inside(p.Repos[0].Path, p.Repos[2].Path)) + require.False(t, inside(p.Repos[0].Path, p.Repos[3].Path)) + for i, repo := range p.Repos { + require.Equal(t, l.Repos[i].Repo.Path, repo.Path) + require.Equal(t, l.Repos[i].Repo.Prefix, repo.Prefix) + } + source, err := discoveryFileSHA(l.Snapshot) + require.NoError(t, err) + dest, err := discoveryFileSHA(p.Store) + require.NoError(t, err) + require.Equal(t, l.MainSHA256, source) + require.Equal(t, source, dest) + _, err = resumeDiscoveryState(t.Context(), o, corpora) + require.Error(t, err, "a prepared run is consumed once") } diff --git a/bench/load/discovery_worktree.go b/bench/load/discovery_worktree.go index ea68160e2..872fb3174 100644 --- a/bench/load/discovery_worktree.go +++ b/bench/load/discovery_worktree.go @@ -1,42 +1,73 @@ package main import ( - "context" - "errors" - "fmt" - "path/filepath" - "strings" - "os" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" ) // Only this scenario alternates the two unrelated families. Existing worktree // scenarios keep their primary-only topology and labels. func (p *privateRun) addDiscoveryWorktree(ctx context.Context, family int) error { - if family<0 || family>1 || len(p.Repos)<2 {return errors.New("discovery requires one of its two initial families")} - primary:=p.Repos[family] - ordinal:=0 - for _,repo:=range p.Repos {if strings.HasPrefix(repo.Corpus.Label,"discovery-linked-"){ordinal++}} - label:=fmt.Sprintf("discovery-linked-%d-%d",family,ordinal) - path:=filepath.Join(p.Root,"repos",label) - if _,err:=p.git(ctx,primary.Path,"worktree","add","--detach",path,primary.Corpus.Commit);err!=nil{return err} - file,err:=ownedFile(path,primary.Corpus.File);if err!=nil{return err} - corpus:=primary.Corpus;corpus.Label=label - p.Repos=append(p.Repos,privateRepo{Corpus:corpus,Path:path,File:file,Prefix:primary.Prefix}) - return nil + if family < 0 || family > 1 || len(p.Repos) < 2 { + return errors.New("discovery requires one of its two initial families") + } + primary := p.Repos[family] + ordinal := 0 + for _, repo := range p.Repos { + if strings.HasPrefix(repo.Corpus.Label, "discovery-linked-") { + ordinal++ + } + } + label := fmt.Sprintf("discovery-linked-%d-%d", family, ordinal) + path := filepath.Join(p.Root, "repos", label) + if _, err := p.git(ctx, primary.Path, "worktree", "add", "--detach", path, primary.Corpus.Commit); err != nil { + return err + } + file, err := ownedFile(path, primary.Corpus.File) + if err != nil { + return err + } + corpus := primary.Corpus + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Corpus: corpus, Path: path, File: file, Prefix: primary.Prefix}) + return nil } -func(p *privateRun)writeDiscoveryBootRepos()([]privateRepo,error){ - if len(p.Repos)<2{return nil,errors.New("discovery requires two configured primaries")} - configured:=*p;configured.Repos=p.Repos[:2] - return configured.writeBootRepos(false) +func (p *privateRun) writeDiscoveryBootRepos() ([]privateRepo, error) { + if len(p.Repos) < 2 { + return nil, errors.New("discovery requires two configured primaries") + } + configured := *p + configured.Repos = p.Repos[:2] + return configured.writeBootRepos(false) } -func(p *privateRun)addExistingDiscoveryWorktree(ctx context.Context,ordinal int,nested bool)error{ - if len(p.Repos)<2{return errors.New("existing discovery topology requires two initial primaries")} - primary:=p.Repos[0];label:=fmt.Sprintf("existing-linked-%d",ordinal) - path:=filepath.Join(p.Root,"repos",label) - if nested {path=filepath.Join(primary.Path,".load-existing",label);if err:=os.MkdirAll(filepath.Dir(path),0o700);err!=nil{return err}} - if _,err:=p.git(ctx,primary.Path,"worktree","add","--detach",path,primary.Corpus.Commit);err!=nil{return err} - file,err:=ownedFile(path,primary.Corpus.File);if err!=nil{return err} - corpus:=primary.Corpus;corpus.Label=label;p.Repos=append(p.Repos,privateRepo{Corpus:corpus,Path:path,File:file,Prefix:primary.Prefix});return nil +func (p *privateRun) addExistingDiscoveryWorktree(ctx context.Context, ordinal int, nested bool) error { + if len(p.Repos) < 2 { + return errors.New("existing discovery topology requires two initial primaries") + } + primary := p.Repos[0] + label := fmt.Sprintf("existing-linked-%d", ordinal) + path := filepath.Join(p.Root, "repos", label) + if nested { + path = filepath.Join(primary.Path, ".load-existing", label) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + } + if _, err := p.git(ctx, primary.Path, "worktree", "add", "--detach", path, primary.Corpus.Commit); err != nil { + return err + } + file, err := ownedFile(path, primary.Corpus.File) + if err != nil { + return err + } + corpus := primary.Corpus + corpus.Label = label + p.Repos = append(p.Repos, privateRepo{Corpus: corpus, Path: path, File: file, Prefix: primary.Prefix}) + return nil } diff --git a/bench/load/lifecycle.go b/bench/load/lifecycle.go index 3ee2b36a2..c846073cc 100644 --- a/bench/load/lifecycle.go +++ b/bench/load/lifecycle.go @@ -28,7 +28,7 @@ type privateRun struct { Store string Config string Env []string - daemonEnv []string // optional owned scenario shim; harness Git keeps Env + daemonEnv []string // optional owned scenario shim; harness Git keeps Env Repos []privateRepo Binary string process *exec.Cmd @@ -344,9 +344,11 @@ func (p *privateRun) start() error { } cmd := exec.Command(p.Binary, "--config", p.Config, "daemon", "start", "--backend-path", p.Store, "--tools", "facade-v1", "--tools-mode", "hide") configureOwnedProcess(cmd) - env:=p.Env - if p.daemonEnv!=nil {env=p.daemonEnv} - cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, env, p.stdout, p.stderr + env := p.Env + if p.daemonEnv != nil { + env = p.daemonEnv + } + cmd.Dir, cmd.Env, cmd.Stdout, cmd.Stderr = p.Root, env, p.stdout, p.stderr if err := cmd.Start(); err != nil { _ = p.stdout.Close() _ = p.stderr.Close() diff --git a/bench/load/run.go b/bench/load/run.go index 86cdc4df6..92ded901d 100644 --- a/bench/load/run.go +++ b/bench/load/run.go @@ -41,8 +41,8 @@ type runOptions struct { EditCount int EditProbeRole string editProbeBaseline bool - Discovery discoveryOptions - Startup startupOptions + Discovery discoveryOptions + Startup startupOptions pairedDriver func(context.Context, *privateRun, []*socketClient, []*socketClient, *Phase) error smokeState *privateRun // only the fixed smoke entry lends its freshly owned store smokeMarkers []string @@ -86,7 +86,7 @@ func bindRunFlags(f *flag.FlagSet, o *runOptions) { f.IntVar(&o.SweepRepetitions, "sweep-repetitions", 15, "samples per benchmark operation (smoke may explicitly use 1)") f.DurationVar(&o.SweepBudget, "sweep-duration", 10*time.Minute, "separate total budget for the repeated paced sweep") bindStartupFlags(f, &o.Startup) - bindDiscoveryFlags(f,&o.Discovery) + bindDiscoveryFlags(f, &o.Discovery) f.IntVar(&o.EditCount, "edit-count", 100, "edit-and-probe: 100 logical edits, or explicit 1-edit diagnostic") f.StringVar(&o.EditProbeRole, "edit-probe-role", "", "edit-and-probe standalone role: candidate (default; phase accounting required) or baseline (absent phase timings reported unavailable)") } @@ -101,8 +101,10 @@ func validateRunOptions(o runOptions) error { if err := validateAgentCounts(o.Scenario, o.Agents, o.Editors); err != nil { return err } - if err:=o.Discovery.validate(o.Scenario,o.Worktrees);err!=nil{return err} - if err := o.Startup.validate(o.Scenario); err != nil { + if err := o.Discovery.validate(o.Scenario, o.Worktrees); err != nil { + return err + } + if err := o.Startup.validate(o.Scenario); err != nil { return err } if o.Scenario == "idle-stop" && (o.Worktrees != 0 || o.CPUProfile != 0 || o.Profile != "agent") { @@ -215,8 +217,10 @@ func runCommand(ctx context.Context, args []string) error { if err := validateOutputRoots(corpora, o.Scratch, o.Output); err != nil { return err } - if o.Discovery.Prepare!="" {return prepareDiscoveryState(ctx,o,corpora)} - sha, err := binaryRevision(o.Binary, o.GitSHA) + if o.Discovery.Prepare != "" { + return prepareDiscoveryState(ctx, o, corpora) + } + sha, err := binaryRevision(o.Binary, o.GitSHA) if err != nil { return err } diff --git a/bench/load/scenario.go b/bench/load/scenario.go index 87a8caa0b..d6b4abe50 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -101,7 +101,11 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp p := o.smokeState var err error if p == nil { - if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" {p,err=resumeDiscoveryState(ctx,o,selected)} else {p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected)} + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" { + p, err = resumeDiscoveryState(ctx, o, selected) + } else { + p, err = newPrivateRun(ctx, outputRoot, o.Binary, selected) + } } else { err = validateSmokeOwner(p, o.Binary, selected) } @@ -109,7 +113,13 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp return phase, nil, artifacts, crashes, "", err } artifacts["private_root"] = p.Root - if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" {raw,err:=os.ReadFile(o.Discovery.Resume);if err!=nil{return phase,nil,artifacts,crashes,"",err};phase.Gate["discovery_seed_lineage"]=json.RawMessage(raw)} + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" { + raw, err := os.ReadFile(o.Discovery.Resume) + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + phase.Gate["discovery_seed_lineage"] = json.RawMessage(raw) + } firstStart := p.startCount + 1 if o.smokeState != nil { info, statErr := os.Stat(p.Store) @@ -169,22 +179,33 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp warmCtx, cancelWarm := context.WithTimeout(ctx, o.WarmTimeout) defer cancelWarm() var configured []privateRepo - if scenario==checkoutDiscoveryScenario {configured,err=p.writeDiscoveryBootRepos()}else{configured,err=p.writeBootRepos(scenario == "worktrees")} + if scenario == checkoutDiscoveryScenario { + configured, err = p.writeDiscoveryBootRepos() + } else { + configured, err = p.writeBootRepos(scenario == "worktrees") + } if err != nil { return phase, nil, artifacts, crashes, "", err } - if scenario==checkoutDiscoveryScenario && o.Discovery.StallFirst { - release,files,err:=p.installDiscoveryFault();if err!=nil{return phase,nil,artifacts,crashes,"",err} - for name,path:=range files {artifacts[name]=path} - defer func(){ - resultErr=errors.Join(resultErr,release()) - raw,err:=os.ReadFile(files["discovery_git_events"]) - observed:=err==nil && len(bytes.TrimSpace(raw))>0 - phase.Gate["discovery_git_fault"],_=json.Marshal(map[string]any{"configured":true,"observed":observed,"raw_events":string(raw),"absolute_child_bound_seconds":60}) - if !observed {resultErr=errors.Join(resultErr,errors.New("Git stall never entered the intended new checkout"),err)} - }() - } - coldStart := time.Now() + if scenario == checkoutDiscoveryScenario && o.Discovery.StallFirst { + release, files, err := p.installDiscoveryFault() + if err != nil { + return phase, nil, artifacts, crashes, "", err + } + for name, path := range files { + artifacts[name] = path + } + defer func() { + resultErr = errors.Join(resultErr, release()) + raw, err := os.ReadFile(files["discovery_git_events"]) + observed := err == nil && len(bytes.TrimSpace(raw)) > 0 + phase.Gate["discovery_git_fault"], _ = json.Marshal(map[string]any{"configured": true, "observed": observed, "raw_events": string(raw), "absolute_child_bound_seconds": 60}) + if !observed { + resultErr = errors.Join(resultErr, errors.New("Git stall never entered the intended new checkout"), err) + } + }() + } + coldStart := time.Now() if err := p.start(); err != nil { return phase, nil, artifacts, crashes, "", err } @@ -243,11 +264,13 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp for i := range p.Repos { for _, tracked := range readinessState.TrackedRepos { if filepath.Clean(tracked.Path) == filepath.Clean(p.Repos[i].Path) { - if scenario==checkoutDiscoveryScenario && o.Discovery.Resume!="" && p.Repos[i].Prefix!=tracked.Prefix {return phase,nil,artifacts,crashes,"",errors.New("resumed discovery changed a prepared repository identity prefix")} - p.Repos[i].Prefix = tracked.Prefix + if scenario == checkoutDiscoveryScenario && o.Discovery.Resume != "" && p.Repos[i].Prefix != tracked.Prefix { + return phase, nil, artifacts, crashes, "", errors.New("resumed discovery changed a prepared repository identity prefix") + } + p.Repos[i].Prefix = tracked.Prefix } } - if i > 0 && scenario == "worktrees" || i>=2 && scenario==checkoutDiscoveryScenario { + if i > 0 && scenario == "worktrees" || i >= 2 && scenario == checkoutDiscoveryScenario { p.Repos[i].Prefix = p.Repos[0].Prefix } if p.Repos[i].Prefix == "" { @@ -277,7 +300,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp version = client.version if i < len(p.Repos) { subscribeInvalidations(warmCtx, client, repo.Corpus.Label, o.Deadline, &phase) - if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0 || scenario==checkoutDiscoveryScenario && i>=2, o.Deadline, &phase); err != nil { + if err := preflightRoot(warmCtx, p, client, repo, scenario == "worktrees" && i > 0 || scenario == checkoutDiscoveryScenario && i >= 2, o.Deadline, &phase); err != nil { return phase, nil, artifacts, crashes, version, err } gate, rows, err := warmAnalysis(warmCtx, client, o.Deadline) From 2b9f35f7eb07668fcc07caf67d09f428eebd1533 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:10:54 +0100 Subject: [PATCH 54/60] fix(load): use lowercase discovery stall errors --- bench/load/discovery_scope.go | 2 +- bench/load/scenario.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/load/discovery_scope.go b/bench/load/discovery_scope.go index 2ed5b1cd0..f42021c4b 100644 --- a/bench/load/discovery_scope.go +++ b/bench/load/discovery_scope.go @@ -147,7 +147,7 @@ func measureCheckoutDiscovery(parent context.Context, o runOptions, p *privateRu report := summarizeCheckoutDiscovery(phase.Attempts[first:], checkouts, frozen.Repos[0].Path, frozen.Repos[1].Path) phase.Gate["checkout_discovery"], _ = json.Marshal(report) if o.Discovery.StallFirst && report.PendingRefusals == 0 { - resultErr = errors.Join(resultErr, errors.New("Git stall produced no pending refusal exposure")) + resultErr = errors.Join(resultErr, errors.New("git stall produced no pending refusal exposure")) } if report.OutsidePendingRefusals > 0 || report.OutsidePendingErrors > 0 || report.ForeignCapacityRefusals > 0 { resultErr = errors.Join(resultErr, fmt.Errorf("checkout discovery affected other cwds: %d refusals, %d errors, %d foreign capacity refusals", report.OutsidePendingRefusals, report.OutsidePendingErrors, report.ForeignCapacityRefusals)) diff --git a/bench/load/scenario.go b/bench/load/scenario.go index d6b4abe50..a0469dea2 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -201,7 +201,7 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp observed := err == nil && len(bytes.TrimSpace(raw)) > 0 phase.Gate["discovery_git_fault"], _ = json.Marshal(map[string]any{"configured": true, "observed": observed, "raw_events": string(raw), "absolute_child_bound_seconds": 60}) if !observed { - resultErr = errors.Join(resultErr, errors.New("Git stall never entered the intended new checkout"), err) + resultErr = errors.Join(resultErr, errors.New("git stall never entered the intended new checkout"), err) } }() } From 361d7bd7f223b8a55a49f6381dd0187deafc4d85 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 20:57:55 +0100 Subject: [PATCH 55/60] test: reproduce missing daemon warning for CLI prefix refusal --- cmd/gortex/track_catalog_test.go | 19 ++++--- cmd/gortex/track_collision_authority_test.go | 60 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 cmd/gortex/track_collision_authority_test.go diff --git a/cmd/gortex/track_catalog_test.go b/cmd/gortex/track_catalog_test.go index abfb213ed..ca84d8e1d 100644 --- a/cmd/gortex/track_catalog_test.go +++ b/cmd/gortex/track_catalog_test.go @@ -28,6 +28,11 @@ import ( ) func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) (*config.ConfigManager, *indexer.MultiIndexer) { + t.Helper() + return startCatalogTrackDaemonWithLogger(t, entries, loaded, zap.NewNop()) +} + +func startCatalogTrackDaemonWithLogger(t *testing.T, entries, loaded []config.RepoEntry, logger *zap.Logger) (*config.ConfigManager, *indexer.MultiIndexer) { t.Helper() dir := testenv.ShortTempDir(t) socket := filepath.Join(dir, "s") @@ -43,9 +48,9 @@ func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) ( languages.RegisterAll(reg) cm, err := config.NewConfigManager(gc.ConfigPath()) require.NoError(t, err) - idx := indexer.New(store, reg, config.Default().Index, zap.NewNop()) - mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, zap.NewNop()) - lc, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: zap.NewNop()}) + idx := indexer.New(store, reg, config.Default().Index, logger) + mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, logger) + lc, err := indexer.NewCheckoutLifecycle(indexer.CheckoutLifecycleConfig{MultiIndexer: mi, ConfigManager: cm, Graph: store, Logger: logger}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, lc.Close()) @@ -58,10 +63,10 @@ func startCatalogTrackDaemon(t *testing.T, entries, loaded []config.RepoEntry) ( require.NoError(t, err) require.NoError(t, result.CatalogErr) } - srv := gortexmcp.NewServer(query.NewEngine(store), store, idx, nil, zap.NewNop(), nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm}) - d := daemon.New(socket, "test", zap.NewNop()) - d.Controller = &realController{graph: store, multiIndexer: mi, configManager: cm, lifecycle: lc, logger: zap.NewNop()} - d.MCPDispatcher = newMCPDispatcher(srv, mi, zap.NewNop()) + srv := gortexmcp.NewServer(query.NewEngine(store), store, idx, nil, logger, nil, gortexmcp.MultiRepoOptions{MultiIndexer: mi, ConfigManager: cm}) + d := daemon.New(socket, "test", logger) + d.Controller = &realController{graph: store, multiIndexer: mi, configManager: cm, lifecycle: lc, logger: logger} + d.MCPDispatcher = newMCPDispatcher(srv, mi, logger) require.NoError(t, d.Listen()) done := make(chan error, 1) go func() { done <- d.Serve() }() diff --git a/cmd/gortex/track_collision_authority_test.go b/cmd/gortex/track_collision_authority_test.go new file mode 100644 index 000000000..7a0181ce0 --- /dev/null +++ b/cmd/gortex/track_collision_authority_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/zzet/gortex/internal/config" +) + +// The private in-process control server runs the real ownership lifecycle. +// No child daemon, timing measurement, or application store is involved. +func TestRunTrackConfiguredCollisionReachesDaemonAuthority(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + owner, loser := filepath.Join(t.TempDir(), "src"), filepath.Join(t.TempDir(), "src") + for _, root := range []string{owner, loser} { + require.NoError(t, os.MkdirAll(root, 0o755)) + wtGit(t, root, "init", "-q", "-b", "main") + require.NoError(t, os.WriteFile(filepath.Join(root, "main.go"), []byte("package sample\nfunc Alpha() {}\n"), 0o644)) + wtGit(t, root, "add", ".") + wtGit(t, root, "commit", "-q", "-m", "isolated ownership fixture") + } + core, logs := observer.New(zap.WarnLevel) + cm, mi := startCatalogTrackDaemonWithLogger(t, []config.RepoEntry{{Path: owner, Name: "src"}}, []config.RepoEntry{{Path: owner, Name: "src"}}, zap.New(core)) + before, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + ownerMetadata := mi.GetMetadata("src") + ownerConfig := cm.GetRepoConfig("src") + oldWait, oldName, oldAsWorktree := trackWait, trackName, trackAsWorktree + t.Cleanup(func() { trackWait, trackName, trackAsWorktree = oldWait, oldName, oldAsWorktree }) + trackName, trackAsWorktree = "", false + for _, wait := range []bool{false, true, false} { + trackWait = wait + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + err := runTrack(cmd, []string{loser}) + require.ErrorContains(t, err, owner) + after, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + require.Equal(t, before, after, "collision must never save the loser") + require.Same(t, ownerMetadata, mi.GetMetadata("src")) + require.Equal(t, ownerConfig, cm.GetRepoConfig("src")) + warnings := logs.FilterMessage("repository not loaded").All() + require.Len(t, warnings, 1, "CLI refusal must reach the daemon ownership decision; retries must deduplicate") + require.Contains(t, warnings[0].ContextMap()["error"], owner) + require.Contains(t, warnings[0].ContextMap()["error"], loser) + require.Contains(t, warnings[0].ContextMap()["error"], "--name") + } + trackWait = false + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + require.NoError(t, runTrack(cmd, []string{owner})) + require.Len(t, logs.FilterMessage("repository not loaded").All(), 1, "same-root no-op adds no collision warning") +} From dd82c856f0eeb84f1bfdce0d5419a42c2a80af3a Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 21:00:03 +0100 Subject: [PATCH 56/60] fix: check daemon ownership before CLI collision refusal --- cmd/gortex/daemon_controller.go | 11 ++++ cmd/gortex/track.go | 58 ++++++++++++++++- cmd/gortex/track_collision_authority_test.go | 23 +++++++ .../daemon/control_track_preflight_test.go | 63 +++++++++++++++++++ internal/daemon/proto.go | 8 ++- internal/daemon/server.go | 25 ++++++++ 6 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 internal/daemon/control_track_preflight_test.go diff --git a/cmd/gortex/daemon_controller.go b/cmd/gortex/daemon_controller.go index df8ec5892..7e33a3650 100644 --- a/cmd/gortex/daemon_controller.go +++ b/cmd/gortex/daemon_controller.go @@ -163,6 +163,17 @@ type daemonWatcherSlot struct { ready bool } +// PreflightTrack checks authoritative ownership without indexing, persisting, +// or taking the controller mutation lock. A clear answer grants no registration. +func (c *realController) PreflightTrack(ctx context.Context, p daemon.TrackParams) error { + if c.multiIndexer == nil || c.lifecycle == nil { + return fmt.Errorf("checkout lifecycle not initialized") + } + return c.lifecycle.PreflightTrack(ctx, config.RepoEntry{ + Path: p.Path, Name: p.Name, Ref: p.Ref, AsWorktree: p.AsWorktree, + }) +} + // Track indexes a new repository and persists it to the global config. // Path is resolved to an absolute form before the MultiIndexer sees it. func (c *realController) Track(ctx context.Context, p daemon.TrackParams) (json.RawMessage, error) { diff --git a/cmd/gortex/track.go b/cmd/gortex/track.go index 4d2a1b42a..c067375c1 100644 --- a/cmd/gortex/track.go +++ b/cmd/gortex/track.go @@ -247,7 +247,17 @@ func runTrack(cmd *cobra.Command, args []string) error { } } if owner, held := owners[prefix]; held && !pathkey.SamePathIdentity(pathkey.CanonicalPath(owner), absPath) { - return indexer.NewPrefixCollisionError(prefix, owner, absPath) + collision := indexer.NewPrefixCollisionError(prefix, owner, absPath) + // Let a running daemon make its own read-only ownership decision, + // including its deduplicated warning, before refusing locally. + // Never send a mutating Track or persist this rejected entry. + if err := preflightDaemonTrack(cmd.Context(), absPath, daemonName); err != nil { + var authoritative *trackPrefixCollisionError + if errors.As(err, &authoritative) { + return authoritative + } + } + return collision } if err := gc.AddRepo(entry); err != nil { return err @@ -795,3 +805,49 @@ func emitUntrackSummary(w io.Writer, target string, r untrackResult) { fmt.Fprintln(w, " "+progress.StatStrip(stats...)) fmt.Fprintln(w) } + +// preflightDaemonTrack never starts a daemon or authorizes a mutation. Its +// deadline includes dialing/handshake, and stays below the CLI refusal budget. +// Older daemons reject the distinct verb instead of treating it as Track. +func preflightDaemonTrack(ctx context.Context, path, name string) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + deadline := time.Now().Add(750 * time.Millisecond) + if callerDeadline, ok := ctx.Deadline(); ok && callerDeadline.Before(deadline) { + deadline = callerDeadline + } + if trackWait && trackWaitTimeout > 0 { + if waitDeadline := time.Now().Add(trackWaitTimeout); waitDeadline.Before(deadline) { + deadline = waitDeadline + } + } + client, err := daemon.DialToWithDeadline(daemon.SocketPath(), daemon.Handshake{Mode: daemon.ModeControl, ClientName: "cli"}, deadline) + if err != nil { + return err + } + defer client.Close() + stop := context.AfterFunc(ctx, func() { _ = client.Close() }) + defer stop() + if err := ctx.Err(); err != nil { + return err + } + remaining := time.Until(deadline) + if remaining <= 0 { + return context.DeadlineExceeded + } + response, err := client.ControlWithTimeout(daemon.ControlTrackPreflight, daemon.TrackParams{Path: path, Name: name, AsWorktree: trackAsWorktree}, remaining) + if err != nil { + return err + } + if !response.OK { + if response.ErrorCode == daemon.ErrPrefixCollision { + return &trackPrefixCollisionError{message: response.ErrorMsg} + } + return fmt.Errorf("track preflight rejected: %s %s", response.ErrorCode, response.ErrorMsg) + } + return nil +} diff --git a/cmd/gortex/track_collision_authority_test.go b/cmd/gortex/track_collision_authority_test.go index 7a0181ce0..8916a7c2b 100644 --- a/cmd/gortex/track_collision_authority_test.go +++ b/cmd/gortex/track_collision_authority_test.go @@ -41,6 +41,8 @@ func TestRunTrackConfiguredCollisionReachesDaemonAuthority(t *testing.T) { cmd.SetErr(&bytes.Buffer{}) err := runTrack(cmd, []string{loser}) require.ErrorContains(t, err, owner) + var authoritative *trackPrefixCollisionError + require.ErrorAs(t, err, &authoritative) after, err := os.ReadFile(cm.Global().ConfigPath()) require.NoError(t, err) require.Equal(t, before, after, "collision must never save the loser") @@ -58,3 +60,24 @@ func TestRunTrackConfiguredCollisionReachesDaemonAuthority(t *testing.T) { require.NoError(t, runTrack(cmd, []string{owner})) require.Len(t, logs.FilterMessage("repository not loaded").All(), 1, "same-root no-op adds no collision warning") } + +func TestRunTrackLocalCollisionCannotRegisterWhenDaemonHasNoOwner(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + owner, loser := filepath.Join(t.TempDir(), "src"), filepath.Join(t.TempDir(), "src") + for _, root := range []string{owner, loser} { + require.NoError(t, os.MkdirAll(root, 0o755)) + } + cm, mi := startCatalogTrackDaemon(t, []config.RepoEntry{{Path: owner, Name: "src"}}, nil) + before, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + oldWait, oldName, oldAsWorktree := trackWait, trackName, trackAsWorktree + t.Cleanup(func() { trackWait, trackName, trackAsWorktree = oldWait, oldName, oldAsWorktree }) + trackWait, trackName, trackAsWorktree = false, "", false + cmd := &cobra.Command{} + cmd.SetErr(&bytes.Buffer{}) + require.ErrorContains(t, runTrack(cmd, []string{loser}), owner) + after, err := os.ReadFile(cm.Global().ConfigPath()) + require.NoError(t, err) + require.Equal(t, before, after) + require.Nil(t, mi.GetMetadata("src"), "read-only preflight must never register the loser when daemon ownership is unresolved") +} diff --git a/internal/daemon/control_track_preflight_test.go b/internal/daemon/control_track_preflight_test.go new file mode 100644 index 000000000..082e09341 --- /dev/null +++ b/internal/daemon/control_track_preflight_test.go @@ -0,0 +1,63 @@ +package daemon + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" + "go.uber.org/zap" +) + +type preflightOnlyController struct { + fakeController + preflightCalls int + preflightErr error +} + +func (c *preflightOnlyController) PreflightTrack(ctx context.Context, _ TrackParams) error { + c.preflightCalls++ + if err := ctx.Err(); err != nil { + return err + } + return c.preflightErr +} + +func TestControlTrackPreflightNeverMutates(t *testing.T) { + params, err := json.Marshal(TrackParams{Path: "candidate", Name: "src"}) + require.NoError(t, err) + for _, collision := range []bool{false, true} { + controller := &preflightOnlyController{} + if collision { + controller.preflightErr = &indexer.PrefixCollisionError{Prefix: "src", ExistingPath: "owner", Path: "candidate"} + } + server := New("", "test", zap.NewNop()) + server.Controller = controller + response := server.handleControl(t.Context(), nil, ControlRequest{Kind: ControlTrackPreflight, Params: params}) + require.Equal(t, !collision, response.OK) + if collision { + require.Equal(t, ErrPrefixCollision, response.ErrorCode) + } + require.Equal(t, 1, controller.preflightCalls) + require.Empty(t, controller.trackCalls) + } + // Old controllers cannot accidentally reinterpret preflight as a track. + old := &fakeController{} + server := New("", "test", zap.NewNop()) + server.Controller = old + response := server.handleControl(t.Context(), nil, ControlRequest{Kind: ControlTrackPreflight, Params: params}) + require.False(t, response.OK) + require.Empty(t, old.trackCalls) +} + +func TestControlTrackPreflightCanceledNeverMutates(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + controller := &preflightOnlyController{} + server := New("", "test", zap.NewNop()) + server.Controller = controller + response := server.handleControl(ctx, nil, ControlRequest{Kind: ControlTrackPreflight}) + require.False(t, response.OK) + require.Empty(t, controller.trackCalls) +} diff --git a/internal/daemon/proto.go b/internal/daemon/proto.go index 1e02423bc..b45c475b0 100644 --- a/internal/daemon/proto.go +++ b/internal/daemon/proto.go @@ -128,9 +128,11 @@ type ControlResponse struct { // Control operation kinds. One constant per kind so callers can't typo. const ( - ControlTrack = "track" - ControlUntrack = "untrack" - ControlReload = "reload" + ControlTrack = "track" + // ControlTrackPreflight observes ownership without registering or saving. + ControlTrackPreflight = "track_preflight" + ControlUntrack = "untrack" + ControlReload = "reload" // ControlProxy reloads servers.toml and rebuilds + atomically swaps // the daemon's multi-server Router, then invalidates the roster // cache — so `gortex proxy on/off/add/remove` apply to a running diff --git a/internal/daemon/server.go b/internal/daemon/server.go index 0d5a95680..abfffa527 100644 --- a/internal/daemon/server.go +++ b/internal/daemon/server.go @@ -777,6 +777,31 @@ func (s *Server) handleControlBounded(sess *Session, req ControlRequest) Control func (s *Server) handleControl(ctx context.Context, _ *Session, req ControlRequest) ControlResponse { switch req.Kind { + case ControlTrackPreflight: + // Separate from Track so an unsupported/older controller cannot + // accidentally register a root for a read-only ownership observation. + preflight, ok := s.Controller.(interface { + PreflightTrack(context.Context, TrackParams) error + }) + if !ok { + return controlErr(ErrInternal, "track preflight is not supported") + } + var p TrackParams + if err := unmarshalParams(req.Params, &p); err != nil { + return controlErr(ErrInternal, err.Error()) + } + if err := preflight.PreflightTrack(ctx, p); err != nil { + var collision *indexer.PrefixCollisionError + if errors.As(err, &collision) { + return controlErr(ErrPrefixCollision, err.Error()) + } + if errors.Is(err, indexer.ErrRepoIdentityPending) { + return ControlResponse{ErrorCode: ErrRepoLoading, ErrorMsg: err.Error(), Retryable: true} + } + return controlErr(ErrInternal, err.Error()) + } + return ControlResponse{OK: true} + case ControlTrack: var p TrackParams if err := unmarshalParams(req.Params, &p); err != nil { From 9404a6beaead7b8f179c0e3bf0c806f7864964c0 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 21:13:06 +0100 Subject: [PATCH 57/60] test(bench): observe concurrent native deletes in disk fixture --- bench/load/edit_probe_lifecycle_test.go | 109 ++++++++++++++++++++++++ bench/load/edit_probe_native_test.go | 16 +++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/bench/load/edit_probe_lifecycle_test.go b/bench/load/edit_probe_lifecycle_test.go index 3a45131e5..fc192e2a3 100644 --- a/bench/load/edit_probe_lifecycle_test.go +++ b/bench/load/edit_probe_lifecycle_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "encoding/json" + "errors" "os" "path/filepath" "slices" @@ -121,6 +123,113 @@ func TestEditProbeDeleteActivatedBeforeItsCreateCompletes(t *testing.T) { } } +// A probe can enumerate the created file just before the native delete +// removes it. Its later read must observe absence, not invent a fixture IO +// failure. Prove presence first and force that exact ordering without sleeps. +func TestEditProbeDiskSnapshotFollowsCreateThenConcurrentDelete(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + plan, err := makeEditProbePlan(c, 42, 100) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + created := filepath.Join(dir, plan.Steps[0].File) + removeOnRead := false + var reported []error + handle := editProbeDiskRepliesWithReadFile(c, dir, func(err error) { reported = append(reported, err) }, func(path string) ([]byte, error) { + if path == created && removeOnRead { + removeOnRead = false + if err := os.Remove(path); err != nil { + return nil, err + } + } + return os.ReadFile(path) + }) + invoke := func(oracle editProbeOracle) json.RawMessage { + t.Helper() + answer, rpcErr := handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": oracle.call.Tool, "arguments": oracle.call.Arguments}}) + if rpcErr != nil { + t.Fatal(rpcErr) + } + raw, err := json.Marshal(answer) + if err != nil { + t.Fatal(err) + } + return raw + } + if _, found := editProbeExactNotFound(invoke(editProbeOracles(c, plan.Steps[0])[0]), "fixture/"+plan.Steps[1].BeforeSymbol); !found { + t.Fatal("not-yet-created file was not exactly absent") + } + content := plan.Steps[0].Request.Arguments["content"].(string) + if err := os.WriteFile(created, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + callee := c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] + for _, oracle := range editProbeOracles(c, plan.Steps[0]) { + body, err := decodeEditProbeResult(invoke(oracle)) + if err != nil || !editProbeCorrect(oracle.name, body, c, plan.Steps[0], callee) { + t.Fatalf("create presence was not proved by %s: %v %+v", oracle.name, err, body) + } + } + removeOnRead = true + for _, oracle := range editProbeOracles(c, plan.Steps[1]) { + raw := invoke(oracle) + if oracle.name == "source" { + if _, found := editProbeExactNotFound(raw, "fixture/"+plan.Steps[1].BeforeSymbol); !found { + t.Fatal("deleted file lacks its exact not-found identity") + } + continue + } + body, err := decodeEditProbeResult(raw) + if err != nil || !editProbeCorrect(oracle.name, body, c, plan.Steps[1], callee) { + t.Fatalf("delete absence was not proved by %s: %v %+v", oracle.name, err, body) + } + } + if removeOnRead { + t.Fatal("delete did not occur between enumeration and read") + } + if _, err := os.Stat(created); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("created file remains after causal delete: %v", err) + } + if len(reported) != 0 { + t.Fatalf("causal disappearance was reported as a fixture failure: %v", reported) + } +} + +func TestEditProbeDiskSnapshotKeepsOtherReadFailures(t *testing.T) { + for _, tc := range []struct { + name, operation string + err error + }{ + {"snapshot_permission", "summary", os.ErrPermission}, + {"direct_file_missing", "file", os.ErrNotExist}, + } { + t.Run(tc.name, func(t *testing.T) { + c := editProbeCreateDeleteCorpus(t) + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, c.File), []byte(c.EditInitialContent), 0o600); err != nil { + t.Fatal(err) + } + var reported []error + handle := editProbeDiskRepliesWithReadFile(c, dir, func(err error) { reported = append(reported, err) }, func(path string) ([]byte, error) { + return nil, &os.PathError{Op: "open", Path: path, Err: tc.err} + }) + handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": "read", "arguments": map[string]any{"operation": tc.operation, "target": map[string]any{"file": c.File}}}}) + if len(reported) == 0 { + t.Fatal("fixture read failure was silently accepted") + } + for _, err := range reported { + if !errors.Is(err, tc.err) { + t.Fatalf("wrong retained read failure: %v", err) + } + } + }) + } +} + func TestEditProbeCreatedFilesRestoredByOneTimedBatch(t *testing.T) { c := editProbeCreateDeleteCorpus(t) plan, err := makeEditProbePlan(c, 42, 100) diff --git a/bench/load/edit_probe_native_test.go b/bench/load/edit_probe_native_test.go index cac71e48d..980d1aac8 100644 --- a/bench/load/edit_probe_native_test.go +++ b/bench/load/edit_probe_native_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -32,6 +33,10 @@ func editProbeDiskStub(t *testing.T, c Corpus, dir string) func(stubFrame) (any, } func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFrame) (any, *rpcError) { + return editProbeDiskRepliesWithReadFile(c, dir, report, os.ReadFile) +} + +func editProbeDiskRepliesWithReadFile(c Corpus, dir string, report func(error), readFile func(string) ([]byte, error)) func(stubFrame) (any, *rpcError) { declared := regexp.MustCompile(`func (GortexEditProbe\w+)\(`) callee := c.Symbol[strings.LastIndex(c.Symbol, "::")+2:] return func(frame stubFrame) (any, *rpcError) { @@ -60,9 +65,16 @@ func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFra graph, edges := []any{map[string]any{"id": "fixture/" + c.Symbol}}, []any{} paths, _ := filepath.Glob(filepath.Join(dir, "*.go")) for _, path := range paths { - data, err := os.ReadFile(path) + data, err := readFile(path) + // Native delete can remove an enumerated file before this read. + // That file is absent in this reply's snapshot. Direct read.file + // calls and every other IO error still fail the fixture. + if errors.Is(err, os.ErrNotExist) { + continue + } if err != nil { report(err) + continue } file := filepath.Base(path) for _, match := range declared.FindAllStringSubmatch(string(data), -1) { @@ -82,7 +94,7 @@ func editProbeDiskReplies(c Corpus, dir string, report func(error)) func(stubFra rows := []any{} switch frame.Params["name"].(string) + "." + args["operation"].(string) { case "read.file": - data, err := os.ReadFile(filepath.Join(dir, file)) + data, err := readFile(filepath.Join(dir, file)) if err != nil { report(err) } From 0847c6fdf2b07659236320dc3d9f4db242e2e9b9 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 21:17:16 +0100 Subject: [PATCH 58/60] test(bench): check negative fixture reply errors --- bench/load/edit_probe_lifecycle_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bench/load/edit_probe_lifecycle_test.go b/bench/load/edit_probe_lifecycle_test.go index fc192e2a3..052ae6dd0 100644 --- a/bench/load/edit_probe_lifecycle_test.go +++ b/bench/load/edit_probe_lifecycle_test.go @@ -217,7 +217,10 @@ func TestEditProbeDiskSnapshotKeepsOtherReadFailures(t *testing.T) { handle := editProbeDiskRepliesWithReadFile(c, dir, func(err error) { reported = append(reported, err) }, func(path string) ([]byte, error) { return nil, &os.PathError{Op: "open", Path: path, Err: tc.err} }) - handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": "read", "arguments": map[string]any{"operation": tc.operation, "target": map[string]any{"file": c.File}}}}) + _, rpcErr := handle(stubFrame{Method: "tools/call", Params: map[string]any{"name": "read", "arguments": map[string]any{"operation": tc.operation, "target": map[string]any{"file": c.File}}}}) + if rpcErr != nil { + t.Fatal(rpcErr) + } if len(reported) == 0 { t.Fatal("fixture read failure was silently accepted") } From 0a642e3fb3285b52295852a411701b0f8564b574 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 21:56:02 +0100 Subject: [PATCH 59/60] test(load): bind resumed discovery to observed checkout catalog --- bench/load/discovery_catalog.go | 170 ++++++++++++++++++++++++++++++ bench/load/discovery_seed.go | 34 ++++-- bench/load/discovery_seed_test.go | 53 +++++++++- bench/load/lifecycle.go | 31 +++--- bench/load/scenario.go | 5 + 5 files changed, 265 insertions(+), 28 deletions(-) create mode 100644 bench/load/discovery_catalog.go diff --git a/bench/load/discovery_catalog.go b/bench/load/discovery_catalog.go new file mode 100644 index 000000000..c1651c1a3 --- /dev/null +++ b/bench/load/discovery_catalog.go @@ -0,0 +1,170 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "reflect" + "time" + + "github.com/zzet/gortex/internal/indexer" +) + +// These fields come from actual list_checkouts observations, never prepared +// topology declarations. View riders are intentionally not catalog identity. +type discoveryCatalogIdentity struct { + Root string `json:"root"` + CheckoutID string `json:"checkout_id"` + Incarnation string `json:"incarnation"` + FamilyID string `json:"family_id"` + RepoPrefix string `json:"repo_prefix"` + CommonDir string `json:"common_dir"` +} +type discoveryCatalogScope struct { + CWD string `json:"cwd"` + Overview indexer.FamiliesOverview `json:"overview"` +} +type discoveryCatalogReceipt struct { + Before []discoveryCatalogScope `json:"before"` + After []discoveryCatalogScope `json:"after"` +} + +func discoveryCatalogIdentities(scopes []discoveryCatalogScope, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) { + expected := make(map[string]discoverySeedRepo, len(repos)) + for _, repo := range repos { + expected[repo.Repo.Path] = repo + } + result := make(map[string]discoveryCatalogIdentity, len(repos)) + ids := make(map[string]string) + owners := make(map[string]string) + for _, scope := range scopes { + for _, family := range scope.Overview.Families { + if family.FamilyID == "" || family.CommonDir == "" || family.PrimaryRepoPrefix == "" { + return nil, errors.New("discovery catalog has incomplete family identity") + } + graphs := make(map[string]indexer.GraphOverview) + for _, graph := range family.Graphs { + if graph.GraphID == "" || graph.RepoPrefix == "" || graph.OwnerCheckoutID == "" { + return nil, errors.New("discovery catalog has incomplete graph identity") + } + if old, ok := graphs[graph.GraphID]; ok && old != graph { + return nil, errors.New("discovery catalog graph has conflicting identities") + } + graphs[graph.GraphID] = graph + owners[family.FamilyID+"\x00"+graph.RepoPrefix] = graph.OwnerCheckoutID + } + for _, checkout := range family.Checkouts { + repo, ok := expected[checkout.RootPath] + prefix := family.PrimaryRepoPrefix + if checkout.GraphID != "" { + prefix = graphs[checkout.GraphID].RepoPrefix + } + if !ok || checkout.CheckoutID == "" || checkout.Incarnation == "" || prefix != repo.Repo.Prefix || family.CommonDir != repo.CommonDir { + return nil, errors.New("discovery catalog checkout does not bind prepared root and family") + } + row := discoveryCatalogIdentity{Root: checkout.RootPath, CheckoutID: checkout.CheckoutID, Incarnation: checkout.Incarnation, FamilyID: family.FamilyID, RepoPrefix: prefix, CommonDir: family.CommonDir} + if old, ok := result[row.Root]; ok && old != row { + return nil, errors.New("discovery catalog root has conflicting identity") + } + if root, ok := ids[row.CheckoutID]; ok && root != row.Root { + return nil, errors.New("discovery catalog ID names multiple roots") + } + result[row.Root] = row + ids[row.CheckoutID] = row.Root + } + } + } + if len(result) != len(expected) { + return nil, errors.New("discovery catalog does not cover every prepared root") + } + for _, repo := range repos[:2] { + row := result[repo.Repo.Path] + if owners[row.FamilyID+"\x00"+row.RepoPrefix] != row.CheckoutID { + return nil, errors.New("discovery catalog primary graph has wrong checkout owner") + } + } + return result, nil +} + +func discoveryReadCatalogReceipt(root string, proof discoverySeedProof, repos []discoverySeedRepo) (map[string]discoveryCatalogIdentity, error) { + if err := discoveryOwnedFile(root, proof.CatalogReceipt); err != nil { + return nil, err + } + hash, err := discoveryFileSHA(proof.CatalogReceipt) + if err != nil || !discoveryDigestValid(proof.CatalogReceiptSHA256, 32) || hash != proof.CatalogReceiptSHA256 { + return nil, errors.New("discovery raw catalog receipt digest mismatch") + } + var raw discoveryCatalogReceipt + if err := discoveryReadJSON(proof.CatalogReceipt, &raw); err != nil { + return nil, err + } + before, err := discoveryCatalogIdentities(raw.Before, repos) + if err != nil { + return nil, err + } + after, err := discoveryCatalogIdentities(raw.After, repos) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(before, after) { + return nil, errors.New("discovery old-to-upgraded catalog identity mapping changed") + } + return after, nil +} + +func (p *privateRun) verifyDiscoveryCatalog(ctx context.Context) (json.RawMessage, error) { + if p.discoveryCatalog == nil { + return nil, nil + } + var scopes []discoveryCatalogScope + for _, repo := range p.Repos[:2] { + callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + conn, reader, version, err := dialSocketSurface(callCtx, p.Socket, repo.Path, "mcp", "full", "hide") + if err != nil { + cancel() + return nil, err + } + client := &socketClient{conn: conn, reader: reader, pending: make(map[int64]chan wireReply), done: make(chan struct{}), version: version, cwd: repo.Path, writeGate: make(chan struct{}, 1)} + go client.readLoop() + _, err = client.call(callCtx, "initialize", map[string]any{"protocolVersion": "2024-11-05", "capabilities": map[string]any{}, "clientInfo": map[string]string{"name": "discovery-lineage", "version": "1"}}) + if err == nil { + err = client.send(callCtx, map[string]any{"jsonrpc": "2.0", "method": "notifications/initialized"}, false) + } + var raw json.RawMessage + if err == nil { + raw, err = client.call(callCtx, "tools/call", map[string]any{"name": "list_checkouts", "arguments": map[string]any{"format": "json", "max_bytes": 0}}) + } + _ = client.Close() + cancel() + if err != nil { + return nil, err + } + payload, err := textPayload(raw, "content") + if err != nil { + return nil, err + } + var overview indexer.FamiliesOverview + if err = json.Unmarshal(payload, &overview); err != nil { + return nil, err + } + scopes = append(scopes, discoveryCatalogScope{CWD: repo.Path, Overview: overview}) + } + repos := make([]discoverySeedRepo, len(p.Repos)) + for i, repo := range p.Repos { + identity := p.discoveryCatalog[repo.Path] + repos[i] = discoverySeedRepo{Repo: repo, CommonDir: identity.CommonDir} + } + actual, err := discoveryCatalogIdentities(scopes, repos) + if err != nil { + return nil, err + } + if !reflect.DeepEqual(actual, p.discoveryCatalog) { + return nil, errors.New("resumed discovery changed observed catalog checkout identity") + } + raw, err := json.Marshal(scopes) + if err == nil { + err = discoveryWriteJSON(filepath.Join(p.Root, "resumed-catalog-observed.json"), scopes) + } + return raw, err +} diff --git a/bench/load/discovery_seed.go b/bench/load/discovery_seed.go index 359e78fb3..cb1c6d807 100644 --- a/bench/load/discovery_seed.go +++ b/bench/load/discovery_seed.go @@ -243,6 +243,7 @@ func resumeDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) ( return nil, errors.New("discovery configuration/producer proof digest mismatch") } } + var catalog map[string]discoveryCatalogIdentity for _, proofPath := range []string{l.StopReceipt, l.IdentityReceipt} { var proof discoverySeedProof if err := discoveryReadJSON(proofPath, &proof); err != nil { @@ -251,6 +252,14 @@ func resumeDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) ( if proof.State != "OBSERVED_OWNED_UPGRADE" || proof.ObservedAt.IsZero() || proof.OwnedRoot != root || proof.OwnerPID != l.OwnerPID || !proof.NormalStop || !proof.OwnersExited || proof.MainSHA256 != l.MainSHA256 || proof.WALSHA256 != l.WALSHA256 || proof.SchemaVersion != 44 || proof.UpgradeBinarySHA256 != l.UpgradeBinarySHA256 || !reflect.DeepEqual(proof.Repos, d.Repos) { return nil, errors.New("discovery producer proof is unknown or does not bind this stopped snapshot and identity topology") } + observed, err := discoveryReadCatalogReceipt(root, proof, d.Repos) + if err != nil { + return nil, err + } + if catalog != nil && !reflect.DeepEqual(catalog, observed) { + return nil, errors.New("discovery producer proofs disagree on catalog identities") + } + catalog = observed } process, err := os.FindProcess(l.OwnerPID) if err != nil { @@ -327,6 +336,7 @@ func resumeDiscoveryState(ctx context.Context, o runOptions, corpora []Corpus) ( if err != nil || copied != before { return nil, errors.New("discovery seed copy differs; partial destination retained") } + p.discoveryCatalog = catalog p.SocketDir, err = os.MkdirTemp("/tmp", "gl-") if err != nil { return nil, err @@ -355,15 +365,17 @@ func discoverySeedPair(path string) ([2]string, error) { // Both retained producer receipts must bind observations to this exact stopped // output; state assertions without snapshot and identity evidence are refused. type discoverySeedProof struct { - State string `json:"state"` - ObservedAt time.Time `json:"observed_at"` - OwnedRoot string `json:"owned_root"` - OwnerPID int `json:"owner_pid"` - NormalStop bool `json:"normal_stop"` - OwnersExited bool `json:"owners_exited"` - MainSHA256 string `json:"main_sha256"` - WALSHA256 string `json:"wal_sha256"` - SchemaVersion int `json:"schema_version"` - UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` - Repos []discoverySeedRepo `json:"repositories"` + CatalogReceipt string `json:"catalog_receipt"` + CatalogReceiptSHA256 string `json:"catalog_receipt_sha256"` + State string `json:"state"` + ObservedAt time.Time `json:"observed_at"` + OwnedRoot string `json:"owned_root"` + OwnerPID int `json:"owner_pid"` + NormalStop bool `json:"normal_stop"` + OwnersExited bool `json:"owners_exited"` + MainSHA256 string `json:"main_sha256"` + WALSHA256 string `json:"wal_sha256"` + SchemaVersion int `json:"schema_version"` + UpgradeBinarySHA256 string `json:"upgrade_binary_sha256"` + Repos []discoverySeedRepo `json:"repositories"` } diff --git a/bench/load/discovery_seed_test.go b/bench/load/discovery_seed_test.go index 81a9ae6a6..bbb44cec4 100644 --- a/bench/load/discovery_seed_test.go +++ b/bench/load/discovery_seed_test.go @@ -4,6 +4,7 @@ package main import ( "encoding/binary" + "fmt" "os" "os/exec" "path/filepath" @@ -12,6 +13,7 @@ import ( "time" "github.com/stretchr/testify/require" + "github.com/zzet/gortex/internal/indexer" ) // This is only a byte-copy/parser fixture, deliberately not a real SQLite @@ -45,12 +47,33 @@ func syntheticDiscoverySeed(t *testing.T) (runOptions, []Corpus, discoverySeedLi } func rewriteSyntheticDiscoveryProofs(t *testing.T, l *discoverySeedLineage) { t.Helper() - proof := discoverySeedProof{State: "OBSERVED_OWNED_UPGRADE", ObservedAt: time.Now(), OwnedRoot: l.OwnedRoot, OwnerPID: l.OwnerPID, NormalStop: l.NormalStop, OwnersExited: l.OwnersExited, MainSHA256: l.MainSHA256, WALSHA256: l.WALSHA256, SchemaVersion: l.SchemaVersion, UpgradeBinarySHA256: l.UpgradeBinarySHA256, Repos: l.Repos} + catalogPath := filepath.Join(l.OwnedRoot, "synthetic-catalog.json") + var scopes []discoveryCatalogScope + for i := 0; i < 2; i++ { + repo := l.Repos[i] + family := indexer.FamilyOverview{FamilyID: fmt.Sprintf("synthetic-family-%d", i), CommonDir: repo.CommonDir, PrimaryRepoPrefix: repo.Repo.Prefix} + family.Graphs = []indexer.GraphOverview{{GraphID: fmt.Sprintf("synthetic-graph-%d", i), RepoPrefix: repo.Repo.Prefix, OwnerCheckoutID: fmt.Sprintf("synthetic-checkout-%d", i)}} + for j, row := range l.Repos { + if row.CommonDir != repo.CommonDir { + continue + } + checkout := indexer.CheckoutOverview{CheckoutID: fmt.Sprintf("synthetic-checkout-%d", j), Incarnation: fmt.Sprintf("synthetic-incarnation-%d", j), RootPath: row.Repo.Path} + if j == i { + checkout.GraphID = family.Graphs[0].GraphID + } + family.Checkouts = append(family.Checkouts, checkout) + } + scopes = append(scopes, discoveryCatalogScope{CWD: repo.Repo.Path, Overview: indexer.FamiliesOverview{Families: []indexer.FamilyOverview{family}}}) + } + _ = os.Remove(catalogPath) + require.NoError(t, discoveryWriteJSON(catalogPath, discoveryCatalogReceipt{Before: scopes, After: scopes})) + catalogSHA, err := discoveryFileSHA(catalogPath) + require.NoError(t, err) + proof := discoverySeedProof{CatalogReceipt: catalogPath, CatalogReceiptSHA256: catalogSHA, State: "OBSERVED_OWNED_UPGRADE", ObservedAt: time.Now(), OwnedRoot: l.OwnedRoot, OwnerPID: l.OwnerPID, NormalStop: l.NormalStop, OwnersExited: l.OwnersExited, MainSHA256: l.MainSHA256, WALSHA256: l.WALSHA256, SchemaVersion: l.SchemaVersion, UpgradeBinarySHA256: l.UpgradeBinarySHA256, Repos: l.Repos} for _, path := range []string{l.StopReceipt, l.IdentityReceipt} { _ = os.Remove(path) require.NoError(t, discoveryWriteJSON(path, proof)) } - var err error l.StopReceiptSHA256, err = discoveryFileSHA(l.StopReceipt) require.NoError(t, err) l.IdentityReceiptSHA256, err = discoveryFileSHA(l.IdentityReceipt) @@ -107,3 +130,29 @@ func TestDiscoverySeedCopiesSnapshotWithoutChangingExistingIdentities(t *testing _, err = resumeDiscoveryState(t.Context(), o, corpora) require.Error(t, err, "a prepared run is consumed once") } + +func TestDiscoverySeedRefusesRekeyedCatalogBeforeCopy(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + var proof discoverySeedProof + require.NoError(t, discoveryReadJSON(l.IdentityReceipt, &proof)) + var raw discoveryCatalogReceipt + require.NoError(t, discoveryReadJSON(proof.CatalogReceipt, &raw)) + raw.After[0].Overview.Families[0].Checkouts[1].Incarnation = "synthetic-rekeyed-incarnation" + require.NoError(t, os.Remove(proof.CatalogReceipt)) + require.NoError(t, discoveryWriteJSON(proof.CatalogReceipt, raw)) + var err error + proof.CatalogReceiptSHA256, err = discoveryFileSHA(proof.CatalogReceipt) + require.NoError(t, err) + for _, path := range []string{l.StopReceipt, l.IdentityReceipt} { + require.NoError(t, os.Remove(path)) + require.NoError(t, discoveryWriteJSON(path, proof)) + } + l.StopReceiptSHA256, err = discoveryFileSHA(l.StopReceipt) + require.NoError(t, err) + l.IdentityReceiptSHA256, err = discoveryFileSHA(l.IdentityReceipt) + require.NoError(t, err) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + _, err = resumeDiscoveryState(t.Context(), o, corpora) + require.ErrorContains(t, err, "catalog identity mapping changed") + require.NoFileExists(t, filepath.Join(l.OwnedRoot, "store.db")) +} diff --git a/bench/load/lifecycle.go b/bench/load/lifecycle.go index c846073cc..8c592b1c2 100644 --- a/bench/load/lifecycle.go +++ b/bench/load/lifecycle.go @@ -22,21 +22,22 @@ type privateRepo struct { } type privateRun struct { - Root string - SocketDir string - Socket string - Store string - Config string - Env []string - daemonEnv []string // optional owned scenario shim; harness Git keeps Env - Repos []privateRepo - Binary string - process *exec.Cmd - done chan struct{} - exitErr error - stdout *os.File - stderr *os.File - startCount int + discoveryCatalog map[string]discoveryCatalogIdentity // discovery-only observed seed identities + Root string + SocketDir string + Socket string + Store string + Config string + Env []string + daemonEnv []string // optional owned scenario shim; harness Git keeps Env + Repos []privateRepo + Binary string + process *exec.Cmd + done chan struct{} + exitErr error + stdout *os.File + stderr *os.File + startCount int } func inside(root, path string) bool { diff --git a/bench/load/scenario.go b/bench/load/scenario.go index a0469dea2..993d919c0 100644 --- a/bench/load/scenario.go +++ b/bench/load/scenario.go @@ -277,6 +277,11 @@ func runPhase(ctx context.Context, o runOptions, scenario string, corpora []Corp return phase, nil, artifacts, crashes, "", errors.New("readiness has no actual repo prefix") } } + if observed, err := p.verifyDiscoveryCatalog(ctx); err != nil { + return phase, nil, artifacts, crashes, "", err + } else if observed != nil { + phase.Gate["resumed_catalog"] = observed + } var clients []*socketClient defer func() { for _, client := range clients { From c25eb9d3168af41e0103f04d1a64a82dc37889c7 Mon Sep 17 00:00:00 2001 From: Andrey Kumanyaev Date: Sun, 1 Nov 2026 21:58:20 +0100 Subject: [PATCH 60/60] test(load): reject live catalog rekeys after seed resume --- bench/load/discovery_seed_test.go | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/bench/load/discovery_seed_test.go b/bench/load/discovery_seed_test.go index bbb44cec4..b638bcc87 100644 --- a/bench/load/discovery_seed_test.go +++ b/bench/load/discovery_seed_test.go @@ -9,6 +9,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -156,3 +157,41 @@ func TestDiscoverySeedRefusesRekeyedCatalogBeforeCopy(t *testing.T) { require.ErrorContains(t, err, "catalog identity mapping changed") require.NoFileExists(t, filepath.Join(l.OwnedRoot, "store.db")) } + +func TestDiscoverySeedVerifiesLiveCatalogAfterResume(t *testing.T) { + for _, rekey := range []bool{false, true} { + t.Run(fmt.Sprintf("rekey_%t", rekey), func(t *testing.T) { + o, corpora, l := syntheticDiscoverySeed(t) + var proof discoverySeedProof + require.NoError(t, discoveryReadJSON(l.IdentityReceipt, &proof)) + var raw discoveryCatalogReceipt + require.NoError(t, discoveryReadJSON(proof.CatalogReceipt, &raw)) + require.NoError(t, discoveryWriteJSON(o.Discovery.Resume, l)) + p, err := resumeDiscoveryState(t.Context(), o, corpora) + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(p.SocketDir) }) + var calls atomic.Int32 + p.Socket = startStub(t, func(frame stubFrame) (any, *rpcError) { + calls.Add(1) + if frame.Method != "tools/call" || frame.Params["name"] != "list_checkouts" { + return nil, &rpcError{Code: -32601, Message: "unexpected synthetic call"} + } + overview := raw.After[frame.Connection-1].Overview + if rekey && frame.Connection == 1 { + overview.Families[0].Checkouts[1].CheckoutID = "unique-live-rekey" + } + return stubContent(overview), nil + }) + observed, err := p.verifyDiscoveryCatalog(t.Context()) + require.Equal(t, int32(2), calls.Load(), "both primary family scopes must be observed") + if rekey { + require.ErrorContains(t, err, "changed observed catalog checkout identity") + require.NoFileExists(t, filepath.Join(p.Root, "resumed-catalog-observed.json")) + } else { + require.NoError(t, err) + require.NotEmpty(t, observed) + require.FileExists(t, filepath.Join(p.Root, "resumed-catalog-observed.json")) + } + }) + } +}