fix(sqlite): make the bulk window fail-safe on panics and a full disk

cmd/gortex/daemon_state.go

@@ -743,7 +743,7 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z
 		coordinatedBulkActive := coordinatedBulk != nil && coordinatedBulk.BeginCoordinatedBulkLoad()
 		defer func() {
 			if coordinatedBulkActive {
-				if err := coordinatedBulk.EndCoordinatedBulkLoad(); err != nil {
+				if err := graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk); err != nil {
 					logger.Error("daemon: coordinated cold bulk-load cleanup failed", zap.Error(err))
 				}
 			}
@@ -916,9 +916,14 @@ func warmupDaemonStateContext(ctx context.Context, state *daemonState, logger *z
 		// final seal includes them in bulk index creation, checkpointing, and
 		// planner statistics instead of reopening ordinary write transactions
 		// immediately after a large cold load.
+		// The finalize is the window's last call: the deferred end above is
+		// skipped once it has run, and nothing retries a failed seal. So a
+		// failed end is followed by an abort, which closes the window and
+		// leaves the indexes it could not rebuild to the store's repair, rather
+		// than keeping the warmup's relaxed-durability writer until shutdown.
 		var finalizeBulk func() error
 		if coordinatedBulkActive {
-			finalizeBulk = coordinatedBulk.EndCoordinatedBulkLoad
+			finalizeBulk = func() error { return graph.EndOrAbortCoordinatedBulkLoad(coordinatedBulk) }
 		}
 		drain := state.multiIndexer.RunDeferredGoModAll
 		if batchCtx.Err() != nil {

cmd/gortex/daemon_warmup_bulk_test.go

@@ -1,8 +1,23 @@
 package main
 
 import (
+	"context"
 	"errors"
+	"os"
+	"path/filepath"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest/observer"
+
+	"github.com/zzet/gortex/internal/config"
+	"github.com/zzet/gortex/internal/graph/store_sqlite"
+	"github.com/zzet/gortex/internal/indexer"
+	"github.com/zzet/gortex/internal/parser"
+	"github.com/zzet/gortex/internal/parser/languages"
 )
 
 func TestDrainGoModAndFinalizeWarmupBulkOrdersDrainBeforeFinalize(t *testing.T) {
@@ -38,3 +53,63 @@ func TestDrainGoModAndFinalizeWarmupBulkWithoutBulkFinalizer(t *testing.T) {
 		t.Fatalf("finalize elapsed = %s, want 0", finalizeElapsed)
 	}
 }
+
+// TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow: a cold warmup whose
+// final seal fails logs the failure and closes the coordinated window, rather
+// than keeping its relaxed-durability writer until the daemon shuts down. The
+// writer is back at its own synchronous level, the planner probe no longer
+// sees a live window, and the store's repair rebuilds the index the seal
+// could not.
+func TestWarmupFailedFinalSealGivesUpTheCoordinatedWindow(t *testing.T) {
+	t.Setenv("GORTEX_PARSER_ISOLATION", "0")
+	root := t.TempDir()
+	t.Setenv("HOME", filepath.Join(root, "home"))
+	t.Setenv("XDG_CONFIG_HOME", filepath.Join(root, "config"))
+	t.Setenv("XDG_DATA_HOME", filepath.Join(root, "data"))
+	t.Setenv("XDG_CACHE_HOME", filepath.Join(root, "cache"))
+	repo := filepath.Join(root, "repo")
+	require.NoError(t, os.Mkdir(repo, 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(repo, "main.go"), []byte("package main\nfunc Main() {}\n"), 0o600))
+	store, err := store_sqlite.Open(filepath.Join(root, "store.sqlite"))
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, store.Close()) })
+	ctx := context.Background()
+	writerSync, err := store.WriterPragmaForTest(ctx, "synchronous")
+	require.NoError(t, err)
+	cm, err := config.NewConfigManager(filepath.Join(root, "config.yaml"))
+	require.NoError(t, err)
+	require.NoError(t, cm.Global().AddRepo(config.RepoEntry{Path: repo}))
+	reg := parser.NewRegistry()
+	reg.Register(languages.NewGoExtractor())
+	logs, observed := observer.New(zap.InfoLevel)
+	logger := zap.New(logs)
+	idx := indexer.New(store, reg, config.Default().Index, logger)
+	mi := indexer.NewMultiIndexer(store, reg, idx.Search(), cm, logger)
+	t.Cleanup(func() { require.NoError(t, mi.Close(context.Background())) })
+	state := &daemonState{graph: store, indexer: idx, multiIndexer: mi, configManager: cm}
+	injected := errors.New("injected final seal failure")
+	restore := store.SetBulkSealHookForTest(func(index string) error {
+		if index == "nodes_by_kind" {
+			return injected
+		}
+		return nil
+	})
+
+	_, _ = warmupDaemonState(state, logger, func() {})
+	restore()
+	require.NotEmpty(t, observed.FilterMessage("daemon: coordinated cold bulk-load finalize failed").All(),
+		"the fixture must fail the coordinated window's final seal")
+	gotSync, err := store.WriterPragmaForTest(ctx, "synchronous")
+	require.NoError(t, err)
+	assert.Equal(t, writerSync, gotSync, "writer durability must be restored")
+	health, err := store.PlannerStatsHealth(ctx)
+	require.NoError(t, err)
+	assert.NotEqual(t, "bulk_window_active", health.Reason, "the failed window must not stay open")
+	deadline := time.Now().Add(10 * time.Second)
+	for health.Reason == "dense_indexes_missing" && time.Now().Before(deadline) {
+		time.Sleep(10 * time.Millisecond)
+		health, err = store.PlannerStatsHealth(ctx)
+		require.NoError(t, err)
+	}
+	assert.NotEqual(t, "dense_indexes_missing", health.Reason, "the repair must rebuild the index the seal left out")
+}

docs/mcp.md

@@ -479,7 +479,7 @@ Gortex captures every large tool response into a bounded per-session ring; these
 |------|-------------|
 | `analyze` | Unified graph analysis dispatcher. `kind` ∈ `dead_code`, `hotspots`, `cycles`, `would_create_cycle`, `connectivity_health`, `todos`, `blame`, `coverage`, `coverage_gaps`, `coverage_summary`, `stale_code`, `stale_flags`, `ownership`, `releases`, `cgo_users`, `wasm_users`, `orphan_tables`, `unreferenced_tables`, `channel_ops`, `goroutine_spawns`, `field_writers`, `race_writes`, `unclosed_channels`, `unsafe_patterns`, `health_score`, `impact`, `annotation_users`, `config_readers`, `env_var_users`, `sql_call_sites`, `fixes_history`, `edge_audit`, `domain`, `named`, `tests_as_edges`, `clusters`, `event_emitters`, `pubsub`, `string_emitters`, `error_surface`, `log_events`, `sql_rebuild`, `external_calls`, `routes`, `models`, `components`, `k8s_resources`, `images`, `kustomize`, `cross_repo`, `dbt_models`, `synthesizers`, `resolution_outcomes`. `clusters` takes an `algorithm` arg (`leiden` / `louvain` / `spectral`). `impact` takes an optional `target` (`{symbol}` or `{file}`): with one it ranks that target's blast radius — the target row first, then its transitive dependents with their `depth` — and reports the closure width plus whether it is exact; without one it ranks the whole repo. An unresolvable target is a structured error, never a silent fall-back to the repo-wide ranking. `synthesizers` rolls up every framework-dispatch-synthesized edge by the pass that produced it; `resolution_outcomes` classifies unresolved call/reference edges by why the resolver gave up (`ambiguous_multi_match` / `candidate_out_of_scope` / `cross_language_only` / `stub_only` / `no_definition`) |
 | `find_clones` | Near-duplicate function/method clusters from the MinHash + LSH `similar_to` layer; `dead_only: true` finds dead duplicates of live code |
-| `index_health` | Health score, parse failures, stale files, language coverage, tracked-repo path liveness (`tracked_repo_paths_ok` + `missing_repo_paths` — a repo whose directory was deleted still holds its registration and silently drops out of workspace-wide answers), per-(repo, provider) semantic-enrichment lifecycle (`semantic_enrichment`: running / completed / partial / abandoned / failed with edge counts, plus a `semantic_enrichment_ok` rollup) — a green file count with a `partial` enrichment state means LSP-tier edges are incomplete. `path_liveness` asks the same question one level down, per file: it stats the paths the graph itself claims and reports how many indexed files no longer exist on disk (`orphan_files` / `orphan_rate` / `orphans_by_repo`, sampled with `truncated: true` past 20k files). `stale_files` only covers files the daemon still tracks, so a deletion it never witnessed shows up here and nowhere else; a non-zero `orphan_files` caps `health_score`. On a SQLite-backed daemon a `planner_stats` object reports the query planner's own view of the store: `nodes` / `edges` carry `believed` (the leading token of the index's `sqlite_stat1` row) against `actual_from_counters` (the `repo_index_state` counter sum across ALL view generations, flagged by `counters_known`; no counter row describes the external-call bucket or nodes carrying no repo prefix, and a leftover empty-prefix row on a store that later became multi-repo is summed on top of the per-repo rows, so the pair is an approximation of index cardinality and a difference between the two is not a measured discrepancy), `receivers` carries a `bounded: true` LIMIT-capped probe of the Go receiver index plus `complete` (false means the probe stopped at its cap, so `actual` is a lower bound and must not be compared with `believed`) and is omitted entirely when the receiver index is not in the schema — a bulk load has dropped it, or this store never had it, and a `believed: 0` there would read as the poisoned near-zero row rather than as an absent index. `stale` says whether the graph has outgrown what the planner believes — which can invert join order on receiver/edge queries — while `reason` names every verdict, staleness or not: `bulk_window_active` is why a cold load in progress reports zeros. `last_refresh_at` / `last_refresh_reason` appear once this process has rebuilt them. The field is read-only: statistics refresh at the next repository index, commit/HEAD move, whole-graph resolve or generation build, never from reading this payload — and that refresh is cooperative, so a busy writer, a spent per-pass budget or a per-index timeout defers the rest of the work to the following boundary (which resumes where the last one stopped) rather than stalling the pipeline that noticed |
+| `index_health` | Health score, parse failures, stale files, language coverage, tracked-repo path liveness (`tracked_repo_paths_ok` + `missing_repo_paths` — a repo whose directory was deleted still holds its registration and silently drops out of workspace-wide answers), per-(repo, provider) semantic-enrichment lifecycle (`semantic_enrichment`: running / completed / partial / abandoned / failed with edge counts, plus a `semantic_enrichment_ok` rollup) — a green file count with a `partial` enrichment state means LSP-tier edges are incomplete. `path_liveness` asks the same question one level down, per file: it stats the paths the graph itself claims and reports how many indexed files no longer exist on disk (`orphan_files` / `orphan_rate` / `orphans_by_repo`, sampled with `truncated: true` past 20k files). `stale_files` only covers files the daemon still tracks, so a deletion it never witnessed shows up here and nowhere else; a non-zero `orphan_files` caps `health_score`. On a SQLite-backed daemon a `planner_stats` object reports the query planner's own view of the store: `nodes` / `edges` carry `believed` (the leading token of the index's `sqlite_stat1` row) against `actual_from_counters` (the `repo_index_state` counter sum across ALL view generations, flagged by `counters_known`; no counter row describes the external-call bucket or nodes carrying no repo prefix, and a leftover empty-prefix row on a store that later became multi-repo is summed on top of the per-repo rows, so the pair is an approximation of index cardinality and a difference between the two is not a measured discrepancy), `receivers` carries a `bounded: true` LIMIT-capped probe of the Go receiver index plus `complete` (false means the probe stopped at its cap, so `actual` is a lower bound and must not be compared with `believed`) and is omitted entirely when the receiver index is not in the schema — a bulk load has dropped it, or this store never had it, and a `believed: 0` there would read as the poisoned near-zero row rather than as an absent index. `stale` says whether the graph has outgrown what the planner believes — which can invert join order on receiver/edge queries — while `reason` names every verdict, staleness or not: `bulk_window_active` is why a cold load in progress reports zeros, and `dense_indexes_missing` means a cold load closed without rebuilding some of its dense indexes (typically because the volume filled up during the final index rebuild): `dense_indexes_missing` in the same object names them, lookups that use them scan until they are back, and the store rebuilds them in the background once its volume has free space at least the size of the store (a daemon restart also rebuilds them). `last_refresh_at` / `last_refresh_reason` appear once this process has rebuilt them. The field is read-only: statistics refresh at the next repository index, commit/HEAD move, whole-graph resolve or generation build, never from reading this payload — and that refresh is cooperative, so a busy writer, a spent per-pass budget or a per-index timeout defers the rest of the work to the following boundary (which resumes where the last one stopped) rather than stalling the pipeline that noticed |
 | `get_symbol_history` | Symbols modified this session with counts; flags churning (3+ edits) |
 The `analyze` dispatcher also accepts a set of **facade-aliased kinds** that route to the captured legacy handler instead of the dispatcher switch: `processes` → `get_processes`, `communities` → `get_communities`, `contracts` → `contracts`, `architecture` → `get_architecture`, `clones` → `find_clones`, `health` → `audit_health`, `inspections` → `run_inspections`, `recent_changes` → `get_recent_changes`, and the other entries of the facade analyze migration table (see `mcp-facade-v1.md`). These aliases are **surface-independent**: they work for client (facade-v1), explicitly legacy-preset, and session-less HTTP callers alike, with no `tools_search` promotion — the HTTP dashboard endpoints depend on this under the `core`/`defer` default.
 

internal/graph/planner_stats.go

@@ -55,6 +55,11 @@ type PlannerStatsFreshness struct {
 	Refreshed bool
 	Reason    string
 
+	// DenseIndexesMissing names secondary indexes the backend knows are absent
+	// and owes a rebuild for (an interrupted bulk load), reported with its own
+	// Reason instead of a verdict. Empty in the ordinary case.
+	DenseIndexesMissing []string
+
 	LastRefreshAt     time.Time
 	LastRefreshReason string
 

internal/graph/store.go

@@ -2,6 +2,7 @@ package graph
 
 import (
 	"context"
+	"errors"
 	"iter"
 	"sync"
 )
@@ -525,9 +526,27 @@ type BulkLoader interface {
 // multi-repository cold load. Implementations engage only on an empty store;
 // nested per-repository BulkLoader calls must leave the outer window open so
 // secondary indexes are rebuilt once after every repository drains.
+//
+// A failed EndCoordinatedBulkLoad may leave the window open so its owner can
+// retry it. AbortCoordinatedBulkLoad is how an owner that will not retry gives
+// it up: the window closes unfinished, ordinary durability comes back, and
+// whatever the end could not rebuild is left to the backend to repair.
 type CoordinatedBulkLoader interface {
 	BeginCoordinatedBulkLoad() bool
 	EndCoordinatedBulkLoad() error
+	AbortCoordinatedBulkLoad() error
+}
+
+// EndOrAbortCoordinatedBulkLoad ends l's coordinated window and, when the end
+// fails, aborts it. It is for an owner's last call on the window: an end that
+// fails with the window still open would otherwise leave it open until the
+// store closes, with every later write at the window's relaxed durability.
+func EndOrAbortCoordinatedBulkLoad(l CoordinatedBulkLoader) error {
+	err := l.EndCoordinatedBulkLoad()
+	if err == nil {
+		return nil
+	}
+	return errors.Join(err, l.AbortCoordinatedBulkLoad())
 }
 
 // SymbolHit is a single full-text-search result: the matched node ID

internal/graph/store_sqlite/add_batch_set.go

@@ -551,6 +551,26 @@ func insertEdgeChunksTxLimitedContext(ctx context.Context, tx *sql.Tx, viewGen i
 	return rowsInserted, statements, insertedKeys, nil
 }
 
+// SetBatchWriteHookForTest installs a hook every AddBatch-family write on this
+// store calls inside its transaction, under the write gate and before any row
+// is written, with the batch's node and edge counts. A returned error fails the
+// batch exactly as a failed insert would (the transaction rolls back and the
+// error reaches the caller the way that caller reports storage errors); a panic
+// unwinds the batch the same way. The hook runs under the write gate, so it
+// must not call the store's write methods. restore puts the previous hook back.
+func (s *Store) SetBatchWriteHookForTest(hook func(nodeRows, edgeRows int) error) (restore func()) {
+	previous := s.batchWriteHook.Swap(&hook)
+	return func() { s.batchWriteHook.Store(previous) }
+}
+
+func (s *Store) runBatchWriteHook(nodeRows, edgeRows int) error {
+	hook := s.batchWriteHook.Load()
+	if hook == nil || *hook == nil {
+		return nil
+	}
+	return (*hook)(nodeRows, edgeRows)
+}
+
 func (s *Store) addBatchSetOriented(nodes []*graph.Node, edges []*graph.Edge) (sqliteAddBatchStats, error) {
 	return s.addBatchSetOrientedContext(context.Background(), nodes, edges)
 }
@@ -708,6 +728,9 @@ func (s *Store) addBatchSetOrientedContext(ctx context.Context, nodes []*graph.N
 	}
 	defer owner.finish(&err)
 	tx := owner.tx
+	if err := s.runBatchWriteHook(len(nodes), len(edges)); err != nil {
+		return stats, err
+	}
 
 	var (
 		receiptDelta  *sqliteMutationReceiptAccumulator